From: Hugh Dickins <hugh@veritas.com>
Convert unuse_process pagetable walkers to loops using p?d_addr_end; but
correct its name to unuse_mm, rename its levels to _range as elsewhere.
Leave unuse_pte out-of-line since it's so rarely called; but move the
funny activate_page inside it. foundaddr was a leftover from before: we
still want to break out once page is found, but no need to pass addr up.
And we need not comment on the page_table_lock at every level.
Whereas most objects shrink ~200 bytes text, swapfile.o grows slightly:
it had earlier been converted to the addr,end style to fix a 4level bug.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
---
25-akpm/mm/swapfile.c | 148 ++++++++++++++++++--------------------------------
1 files changed, 56 insertions(+), 92 deletions(-)
diff -puN mm/swapfile.c~ptwalk-unuse_mm mm/swapfile.c
--- 25/mm/swapfile.c~ptwalk-unuse_mm 2005-03-09 16:34:09.000000000 -0800
+++ 25-akpm/mm/swapfile.c 2005-03-09 16:34:09.000000000 -0800
@@ -394,154 +394,121 @@ void free_swap_and_cache(swp_entry_t ent
}
/*
- * The swap entry has been read in advance, and we return 1 to indicate
- * that the page has been used or is no longer needed.
- *
* Always set the resulting pte to be nowrite (the same as COW pages
* after one process has exited). We don't know just how many PTEs will
* share this swap entry, so be cautious and let do_wp_page work out
* what to do if a write is requested later.
+ *
+ * vma->vm_mm->page_table_lock is held.
*/
-/* vma->vm_mm->page_table_lock is held */
-static void
-unuse_pte(struct vm_area_struct *vma, unsigned long address, pte_t *dir,
- swp_entry_t entry, struct page *page)
+static void unuse_pte(struct vm_area_struct *vma, pte_t *pte,
+ unsigned long addr, swp_entry_t entry, struct page *page)
{
vma->vm_mm->rss++;
get_page(page);
- set_pte_at(vma->vm_mm, address, dir,
+ set_pte_at(vma->vm_mm, addr, pte,
pte_mkold(mk_pte(page, vma->vm_page_prot)));
- page_add_anon_rmap(page, vma, address);
+ page_add_anon_rmap(page, vma, addr);
swap_free(entry);
+ /*
+ * Move the page to the active list so it is not
+ * immediately swapped out again after swapon.
+ */
+ activate_page(page);
}
-/* vma->vm_mm->page_table_lock is held */
-static unsigned long unuse_pmd(struct vm_area_struct *vma, pmd_t *dir,
- unsigned long address, unsigned long end,
- swp_entry_t entry, struct page *page)
+static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
+ unsigned long addr, unsigned long end,
+ swp_entry_t entry, struct page *page)
{
pte_t *pte;
pte_t swp_pte = swp_entry_to_pte(entry);
- if (pmd_none_or_clear_bad(dir))
+ if (pmd_none_or_clear_bad(pmd))
return 0;
- pte = pte_offset_map(dir, address);
+ pte = pte_offset_map(pmd, addr);
do {
/*
* swapoff spends a _lot_ of time in this loop!
* Test inline before going to call unuse_pte.
*/
if (unlikely(pte_same(*pte, swp_pte))) {
- unuse_pte(vma, address, pte, entry, page);
+ unuse_pte(vma, pte, addr, entry, page);
pte_unmap(pte);
-
- /*
- * Move the page to the active list so it is not
- * immediately swapped out again after swapon.
- */
- activate_page(page);
-
- /* add 1 since address may be 0 */
- return 1 + address;
+ return 1;
}
- address += PAGE_SIZE;
- pte++;
- } while (address < end);
+ } while (pte++, addr += PAGE_SIZE, addr != end);
pte_unmap(pte - 1);
return 0;
}
-/* vma->vm_mm->page_table_lock is held */
-static unsigned long unuse_pud(struct vm_area_struct *vma, pud_t *pud,
- unsigned long address, unsigned long end,
- swp_entry_t entry, struct page *page)
+static int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud,
+ unsigned long addr, unsigned long end,
+ swp_entry_t entry, struct page *page)
{
pmd_t *pmd;
unsigned long next;
- unsigned long foundaddr;
if (pud_none_or_clear_bad(pud))
return 0;
- pmd = pmd_offset(pud, address);
+ pmd = pmd_offset(pud, addr);
do {
- next = (address + PMD_SIZE) & PMD_MASK;
- if (next > end || !next)
- next = end;
- foundaddr = unuse_pmd(vma, pmd, address, next, entry, page);
- if (foundaddr)
- return foundaddr;
- address = next;
- pmd++;
- } while (address < end);
+ next = pmd_addr_end(addr, end);
+ if (unuse_pte_range(vma, pmd, addr, next, entry, page))
+ return 1;
+ } while (pmd++, addr = next, addr != end);
return 0;
}
-/* vma->vm_mm->page_table_lock is held */
-static unsigned long unuse_pgd(struct vm_area_struct *vma, pgd_t *pgd,
- unsigned long address, unsigned long end,
- swp_entry_t entry, struct page *page)
+static int unuse_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
+ unsigned long addr, unsigned long end,
+ swp_entry_t entry, struct page *page)
{
pud_t *pud;
unsigned long next;
- unsigned long foundaddr;
if (pgd_none_or_clear_bad(pgd))
return 0;
- pud = pud_offset(pgd, address);
+ pud = pud_offset(pgd, addr);
do {
- next = (address + PUD_SIZE) & PUD_MASK;
- if (next > end || !next)
- next = end;
- foundaddr = unuse_pud(vma, pud, address, next, entry, page);
- if (foundaddr)
- return foundaddr;
- address = next;
- pud++;
- } while (address < end);
+ next = pud_addr_end(addr, end);
+ if (unuse_pmd_range(vma, pud, addr, next, entry, page))
+ return 1;
+ } while (pud++, addr = next, addr != end);
return 0;
}
-/* vma->vm_mm->page_table_lock is held */
-static unsigned long unuse_vma(struct vm_area_struct *vma,
- swp_entry_t entry, struct page *page)
+static int unuse_vma(struct vm_area_struct *vma,
+ swp_entry_t entry, struct page *page)
{
pgd_t *pgd;
- unsigned long address, next, end;
- unsigned long foundaddr;
+ unsigned long addr, end, next;
if (page->mapping) {
- address = page_address_in_vma(page, vma);
- if (address == -EFAULT)
+ addr = page_address_in_vma(page, vma);
+ if (addr == -EFAULT)
return 0;
else
- end = address + PAGE_SIZE;
+ end = addr + PAGE_SIZE;
} else {
- address = vma->vm_start;
+ addr = vma->vm_start;
end = vma->vm_end;
}
- pgd = pgd_offset(vma->vm_mm, address);
+
+ pgd = pgd_offset(vma->vm_mm, addr);
do {
- next = (address + PGDIR_SIZE) & PGDIR_MASK;
- if (next > end || !next)
- next = end;
- foundaddr = unuse_pgd(vma, pgd, address, next, entry, page);
- if (foundaddr)
- return foundaddr;
- address = next;
- pgd++;
- } while (address < end);
+ next = pgd_addr_end(addr, end);
+ if (unuse_pud_range(vma, pgd, addr, next, entry, page))
+ return 1;
+ } while (pgd++, addr = next, addr != end);
return 0;
}
-static int unuse_process(struct mm_struct * mm,
- swp_entry_t entry, struct page* page)
+static int unuse_mm(struct mm_struct *mm,
+ swp_entry_t entry, struct page *page)
{
- struct vm_area_struct* vma;
- unsigned long foundaddr = 0;
+ struct vm_area_struct *vma;
- /*
- * Go through process' page directory.
- */
if (!down_read_trylock(&mm->mmap_sem)) {
/*
* Our reference to the page stops try_to_unmap_one from
@@ -553,16 +520,13 @@ static int unuse_process(struct mm_struc
}
spin_lock(&mm->page_table_lock);
for (vma = mm->mmap; vma; vma = vma->vm_next) {
- if (vma->anon_vma) {
- foundaddr = unuse_vma(vma, entry, page);
- if (foundaddr)
- break;
- }
+ if (vma->anon_vma && unuse_vma(vma, entry, page))
+ break;
}
spin_unlock(&mm->page_table_lock);
up_read(&mm->mmap_sem);
/*
- * Currently unuse_process cannot fail, but leave error handling
+ * Currently unuse_mm cannot fail, but leave error handling
* at call sites for now, since we change it from time to time.
*/
return 0;
@@ -706,7 +670,7 @@ static int try_to_unuse(unsigned int typ
if (start_mm == &init_mm)
shmem = shmem_unuse(entry, page);
else
- retval = unuse_process(start_mm, entry, page);
+ retval = unuse_mm(start_mm, entry, page);
}
if (*swap_map > 1) {
int set_start_mm = (*swap_map >= swcount);
@@ -738,7 +702,7 @@ static int try_to_unuse(unsigned int typ
set_start_mm = 1;
shmem = shmem_unuse(entry, page);
} else
- retval = unuse_process(mm, entry, page);
+ retval = unuse_mm(mm, entry, page);
if (set_start_mm && *swap_map < swcount) {
mmput(new_start_mm);
atomic_inc(&mm->mm_users);
_