From: Christoph Lameter <clameter@sgi.com>
Changelog
* Atomic pte operations for i386 in regular and PAE modes
Signed-off-by: Christoph Lameter <clameter@sgi.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
---
25-akpm/include/asm-i386/pgalloc.h | 17 +++++++++++++
25-akpm/include/asm-i386/pgtable-2level.h | 3 ++
25-akpm/include/asm-i386/pgtable-3level.h | 39 ++++++++++++++++++------------
25-akpm/include/asm-i386/pgtable.h | 1
4 files changed, 45 insertions(+), 15 deletions(-)
diff -puN include/asm-i386/pgalloc.h~page-fault-scalability-patch-v11-i386-atomic-pte-operations include/asm-i386/pgalloc.h
--- 25/include/asm-i386/pgalloc.h~page-fault-scalability-patch-v11-i386-atomic-pte-operations Thu Dec 2 12:39:42 2004
+++ 25-akpm/include/asm-i386/pgalloc.h Thu Dec 2 12:39:42 2004
@@ -4,9 +4,12 @@
#include <linux/config.h>
#include <asm/processor.h>
#include <asm/fixmap.h>
+#include <asm/system.h>
#include <linux/threads.h>
#include <linux/mm.h> /* for struct page */
+#define PMD_NONE 0L
+
#define pmd_populate_kernel(mm, pmd, pte) \
set_pmd(pmd, __pmd(_PAGE_TABLE + __pa(pte)))
@@ -16,6 +19,19 @@ static inline void pmd_populate(struct m
((unsigned long long)page_to_pfn(pte) <<
(unsigned long long) PAGE_SHIFT)));
}
+
+/* Atomic version */
+static inline int pmd_test_and_populate(struct mm_struct *mm, pmd_t *pmd, struct page *pte)
+{
+#ifdef CONFIG_X86_PAE
+ return cmpxchg8b( ((unsigned long long *)pmd), PMD_NONE, _PAGE_TABLE +
+ ((unsigned long long)page_to_pfn(pte) <<
+ (unsigned long long) PAGE_SHIFT) ) == PMD_NONE;
+#else
+ return cmpxchg( (unsigned long *)pmd, PMD_NONE, _PAGE_TABLE + (page_to_pfn(pte) << PAGE_SHIFT)) == PMD_NONE;
+#endif
+}
+
/*
* Allocate and free page tables.
*/
@@ -46,6 +62,7 @@ static inline void pte_free(struct page
#define pmd_free(x) do { } while (0)
#define __pmd_free_tlb(tlb,x) do { } while (0)
#define pgd_populate(mm, pmd, pte) BUG()
+#define pgd_test_and_populate(mm, pmd, pte) ({ BUG(); 1; })
#define check_pgt_cache() do { } while (0)
diff -puN include/asm-i386/pgtable-2level.h~page-fault-scalability-patch-v11-i386-atomic-pte-operations include/asm-i386/pgtable-2level.h
--- 25/include/asm-i386/pgtable-2level.h~page-fault-scalability-patch-v11-i386-atomic-pte-operations Thu Dec 2 12:39:42 2004
+++ 25-akpm/include/asm-i386/pgtable-2level.h Thu Dec 2 12:39:42 2004
@@ -82,4 +82,7 @@ static inline int pte_exec_kernel(pte_t
#define __pte_to_swp_entry(pte) ((swp_entry_t) { (pte).pte_low })
#define __swp_entry_to_pte(x) ((pte_t) { (x).val })
+/* Atomic PTE operations */
+#define ptep_cmpxchg(__vma,__a,__xp,__oldpte,__newpte) (cmpxchg(&(__xp)->pte_low, (__oldpte).pte_low, (__newpte).pte_low)==(__oldpte).pte_low)
+
#endif /* _I386_PGTABLE_2LEVEL_H */
diff -puN include/asm-i386/pgtable-3level.h~page-fault-scalability-patch-v11-i386-atomic-pte-operations include/asm-i386/pgtable-3level.h
--- 25/include/asm-i386/pgtable-3level.h~page-fault-scalability-patch-v11-i386-atomic-pte-operations Thu Dec 2 12:39:42 2004
+++ 25-akpm/include/asm-i386/pgtable-3level.h Thu Dec 2 12:39:42 2004
@@ -6,7 +6,8 @@
* tables on PPro+ CPUs.
*
* Copyright (C) 1999 Ingo Molnar <mingo@redhat.com>
- */
+ * August 26, 2004 added ptep_cmpxchg <christoph@lameter.com>
+*/
#define pte_ERROR(e) \
printk("%s:%d: bad pte %p(%08lx%08lx).\n", __FILE__, __LINE__, &(e), (e).pte_high, (e).pte_low)
@@ -42,26 +43,15 @@ static inline int pte_exec_kernel(pte_t
return pte_x(pte);
}
-/* Rules for using set_pte: the pte being assigned *must* be
- * either not present or in a state where the hardware will
- * not attempt to update the pte. In places where this is
- * not possible, use pte_get_and_clear to obtain the old pte
- * value and then use set_pte to update it. -ben
- */
-static inline void set_pte(pte_t *ptep, pte_t pte)
-{
- ptep->pte_high = pte.pte_high;
- smp_wmb();
- ptep->pte_low = pte.pte_low;
-}
-#define __HAVE_ARCH_SET_PTE_ATOMIC
-#define set_pte_atomic(pteptr,pteval) \
+#define set_pte(pteptr,pteval) \
set_64bit((unsigned long long *)(pteptr),pte_val(pteval))
#define set_pmd(pmdptr,pmdval) \
set_64bit((unsigned long long *)(pmdptr),pmd_val(pmdval))
#define set_pgd(pgdptr,pgdval) \
set_64bit((unsigned long long *)(pgdptr),pgd_val(pgdval))
+#define set_pte_atomic set_pte
+
/*
* Pentium-II erratum A13: in PAE mode we explicitly have to flush
* the TLB via cr3 if the top-level pgd is changed...
@@ -142,4 +132,23 @@ static inline pmd_t pfn_pmd(unsigned lon
#define __pte_to_swp_entry(pte) ((swp_entry_t){ (pte).pte_high })
#define __swp_entry_to_pte(x) ((pte_t){ 0, (x).val })
+/* Atomic PTE operations */
+#define ptep_xchg_flush(__vma, __addr, __ptep, __newval) \
+({ pte_t __r; \
+ /* xchg acts as a barrier before the setting of the high bits. */\
+ __r.pte_low = xchg(&(__ptep)->pte_low, (__newval).pte_low); \
+ __r.pte_high = (__ptep)->pte_high; \
+ (__ptep)->pte_high = (__newval).pte_high; \
+ flush_tlb_page(__vma, __addr); \
+ (__r); \
+})
+
+#define __HAVE_ARCH_PTEP_XCHG_FLUSH
+
+static inline int ptep_cmpxchg(struct vm_area_struct *vma, unsigned long address, pte_t *ptep, pte_t oldval, pte_t newval)
+{
+ return cmpxchg((unsigned int *)ptep, pte_val(oldval), pte_val(newval)) == pte_val(oldval);
+}
+
+
#endif /* _I386_PGTABLE_3LEVEL_H */
diff -puN include/asm-i386/pgtable.h~page-fault-scalability-patch-v11-i386-atomic-pte-operations include/asm-i386/pgtable.h
--- 25/include/asm-i386/pgtable.h~page-fault-scalability-patch-v11-i386-atomic-pte-operations Thu Dec 2 12:39:42 2004
+++ 25-akpm/include/asm-i386/pgtable.h Thu Dec 2 12:39:42 2004
@@ -403,6 +403,7 @@ extern pte_t *lookup_address(unsigned lo
#define __HAVE_ARCH_PTEP_SET_WRPROTECT
#define __HAVE_ARCH_PTEP_MKDIRTY
#define __HAVE_ARCH_PTE_SAME
+#define __HAVE_ARCH_ATOMIC_TABLE_OPS
#include <asm-generic/pgtable.h>
#include <asm-generic/nopml4-pgtable.h>
_