From: Christoph Lameter <clameter@sgi.com>
Changelog
* Provide atomic pte operations for ia64
* Enhanced parallelism in page fault handler if applied together
with the generic patch
Signed-off-by: Christoph Lameter <clameter@sgi.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
---
25-akpm/include/asm-ia64/pgalloc.h | 18 ++++++++++++++++++
25-akpm/include/asm-ia64/pgtable.h | 29 ++++++++++++++++++++++-------
2 files changed, 40 insertions(+), 7 deletions(-)
diff -puN include/asm-ia64/pgalloc.h~page-fault-scalability-patch-v11-ia64-atomic-pte-operations include/asm-ia64/pgalloc.h
--- 25/include/asm-ia64/pgalloc.h~page-fault-scalability-patch-v11-ia64-atomic-pte-operations Thu Dec 2 12:39:13 2004
+++ 25-akpm/include/asm-ia64/pgalloc.h Thu Dec 2 12:39:13 2004
@@ -34,6 +34,10 @@
#define pmd_quicklist (local_cpu_data->pmd_quick)
#define pgtable_cache_size (local_cpu_data->pgtable_cache_sz)
+/* Empty entries of PMD and PGD */
+#define PMD_NONE 0
+#define PGD_NONE 0
+
static inline pgd_t*
pgd_alloc_one_fast (struct mm_struct *mm)
{
@@ -64,12 +68,19 @@ pgd_free (pgd_t *pgd)
preempt_enable();
}
+
static inline void
pgd_populate (struct mm_struct *mm, pgd_t *pgd_entry, pmd_t *pmd)
{
pgd_val(*pgd_entry) = __pa(pmd);
}
+/* Atomic populate */
+static inline int
+pgd_test_and_populate (struct mm_struct *mm, pgd_t *pgd_entry, pmd_t *pmd)
+{
+ return ia64_cmpxchg8_acq(pgd_entry,__pa(pmd), PGD_NONE) == PGD_NONE;
+}
static inline pmd_t*
pmd_alloc_one_fast (struct mm_struct *mm, unsigned long addr)
@@ -118,6 +129,13 @@ pmd_populate (struct mm_struct *mm, pmd_
pmd_val(*pmd_entry) = page_to_phys(pte);
}
+/* Atomic populate */
+static inline int
+pmd_test_and_populate (struct mm_struct *mm, pmd_t *pmd_entry, struct page *pte)
+{
+ return ia64_cmpxchg8_acq(pmd_entry, page_to_phys(pte), PMD_NONE) == PMD_NONE;
+}
+
static inline void
pmd_populate_kernel (struct mm_struct *mm, pmd_t *pmd_entry, pte_t *pte)
{
diff -puN include/asm-ia64/pgtable.h~page-fault-scalability-patch-v11-ia64-atomic-pte-operations include/asm-ia64/pgtable.h
--- 25/include/asm-ia64/pgtable.h~page-fault-scalability-patch-v11-ia64-atomic-pte-operations Thu Dec 2 12:39:13 2004
+++ 25-akpm/include/asm-ia64/pgtable.h Thu Dec 2 12:39:13 2004
@@ -407,6 +407,26 @@ ptep_mkdirty (pte_t *ptep)
#endif
}
+/*
+ * IA-64 doesn't have any external MMU info: the page tables contain all the necessary
+ * information. However, we use this routine to take care of any (delayed) i-cache
+ * flushing that may be necessary.
+ */
+extern void update_mmu_cache (struct vm_area_struct *vma, unsigned long vaddr, pte_t pte);
+
+static inline int
+ptep_cmpxchg (struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t oldval, pte_t newval)
+{
+ /*
+ * IA64 defers icache flushes. If the new pte is executable we may
+ * have to flush the icache to insure cache coherency immediately
+ * after the cmpxchg.
+ */
+ if (pte_exec(newval))
+ update_mmu_cache(vma, addr, newval);
+ return ia64_cmpxchg8_acq(&ptep->pte, newval.pte, oldval.pte) == oldval.pte;
+}
+
static inline int
pte_same (pte_t a, pte_t b)
{
@@ -469,13 +489,6 @@ extern void hugetlb_free_pgtables(struct
struct vm_area_struct * prev, unsigned long start, unsigned long end);
#endif
-/*
- * IA-64 doesn't have any external MMU info: the page tables contain all the necessary
- * information. However, we use this routine to take care of any (delayed) i-cache
- * flushing that may be necessary.
- */
-extern void update_mmu_cache (struct vm_area_struct *vma, unsigned long vaddr, pte_t pte);
-
#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
/*
* Update PTEP with ENTRY, which is guaranteed to be a less
@@ -553,6 +566,8 @@ do { \
#define __HAVE_ARCH_PTEP_MKDIRTY
#define __HAVE_ARCH_PTE_SAME
#define __HAVE_ARCH_PGD_OFFSET_GATE
+#define __HAVE_ARCH_ATOMIC_TABLE_OPS
+#define __HAVE_ARCH_LOCK_TABLE_OPS
#include <asm-generic/pgtable.h>
#include <asm-generic/nopml4-pgtable.h>
_