From: Hugh Dickins <hugh@veritas.com>
First of a batch of five patches to eliminate rmap's page_map_lock, replace
its trylocking by spinlocking, and use anon_vma to speed up swapoff.
Patches updated from the originals against 2.6.7-mm7: nothing new so I won't
spam the list, but including Manfred's SLAB_DESTROY_BY_RCU fixes, and omitting
the unuse_process mmap_sem fix already in 2.6.8-rc3.
This patch:
Replace the PG_anon page->flags bit by setting the lower bit of the pointer in
page->mapping when it's anon_vma: PAGE_MAPPING_ANON bit.
We're about to eliminate the locking which kept the flags and mapping in
synch: it's much easier to work on a local copy of page->mapping, than worry
about whether flags and mapping are in synch (though I imagine it could be
done, at greater cost, with some barriers).
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
---
25-akpm/include/linux/mm.h | 23 ++++++++++++++++-------
25-akpm/include/linux/page-flags.h | 6 ------
25-akpm/mm/page_alloc.c | 3 ---
25-akpm/mm/rmap.c | 20 +++-----------------
4 files changed, 19 insertions(+), 33 deletions(-)
diff -puN include/linux/mm.h~rmaplock-1-5-pageanon-in-mapping include/linux/mm.h
--- 25/include/linux/mm.h~rmaplock-1-5-pageanon-in-mapping 2004-08-15 22:49:45.213729480 -0700
+++ 25-akpm/include/linux/mm.h 2004-08-15 22:49:45.221728264 -0700
@@ -211,11 +211,12 @@ struct page {
* if PagePrivate set; used for
* swp_entry_t if PageSwapCache
*/
- struct address_space *mapping; /* If PG_anon clear, points to
+ struct address_space *mapping; /* If low bit clear, points to
* inode address_space, or NULL.
* If page mapped as anonymous
- * memory, PG_anon is set, and
- * it points to anon_vma object.
+ * memory, low bit is set, and
+ * it points to anon_vma object:
+ * see PAGE_MAPPING_ANON below.
*/
pgoff_t index; /* Our offset within mapping. */
struct list_head lru; /* Pageout list, eg. active_list
@@ -439,24 +440,32 @@ void page_address_init(void);
/*
* On an anonymous page mapped into a user virtual memory area,
- * page->mapping points to its anon_vma, not to a struct address_space.
+ * page->mapping points to its anon_vma, not to a struct address_space;
+ * with the PAGE_MAPPING_ANON bit set to distinguish it.
*
* Please note that, confusingly, "page_mapping" refers to the inode
* address_space which maps the page from disk; whereas "page_mapped"
* refers to user virtual address space into which the page is mapped.
*/
+#define PAGE_MAPPING_ANON 1
+
extern struct address_space swapper_space;
static inline struct address_space *page_mapping(struct page *page)
{
- struct address_space *mapping = NULL;
+ struct address_space *mapping = page->mapping;
if (unlikely(PageSwapCache(page)))
mapping = &swapper_space;
- else if (likely(!PageAnon(page)))
- mapping = page->mapping;
+ else if (unlikely((unsigned long)mapping & PAGE_MAPPING_ANON))
+ mapping = NULL;
return mapping;
}
+static inline int PageAnon(struct page *page)
+{
+ return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
+}
+
/*
* Return the pagecache index of the passed page. Regular pagecache pages
* use ->index whereas swapcache pages use ->private
diff -puN include/linux/page-flags.h~rmaplock-1-5-pageanon-in-mapping include/linux/page-flags.h
--- 25/include/linux/page-flags.h~rmaplock-1-5-pageanon-in-mapping 2004-08-15 22:49:45.214729328 -0700
+++ 25-akpm/include/linux/page-flags.h 2004-08-15 22:49:45.222728112 -0700
@@ -76,8 +76,6 @@
#define PG_reclaim 18 /* To be reclaimed asap */
#define PG_compound 19 /* Part of a compound page */
-#define PG_anon 20 /* Anonymous: anon_vma in mapping */
-
/*
* Global page accounting. One instance per CPU. Only unsigned longs are
@@ -292,10 +290,6 @@ extern unsigned long __read_page_state(u
#define SetPageCompound(page) set_bit(PG_compound, &(page)->flags)
#define ClearPageCompound(page) clear_bit(PG_compound, &(page)->flags)
-#define PageAnon(page) test_bit(PG_anon, &(page)->flags)
-#define SetPageAnon(page) set_bit(PG_anon, &(page)->flags)
-#define ClearPageAnon(page) clear_bit(PG_anon, &(page)->flags)
-
#ifdef CONFIG_SWAP
#define PageSwapCache(page) test_bit(PG_swapcache, &(page)->flags)
#define SetPageSwapCache(page) set_bit(PG_swapcache, &(page)->flags)
diff -puN mm/page_alloc.c~rmaplock-1-5-pageanon-in-mapping mm/page_alloc.c
--- 25/mm/page_alloc.c~rmaplock-1-5-pageanon-in-mapping 2004-08-15 22:49:45.216729024 -0700
+++ 25-akpm/mm/page_alloc.c 2004-08-15 22:49:45.223727960 -0700
@@ -90,7 +90,6 @@ static void bad_page(const char *functio
1 << PG_active |
1 << PG_dirty |
1 << PG_maplock |
- 1 << PG_anon |
1 << PG_swapcache |
1 << PG_writeback);
set_page_count(page, 0);
@@ -232,7 +231,6 @@ static inline void free_pages_check(cons
1 << PG_reclaim |
1 << PG_slab |
1 << PG_maplock |
- 1 << PG_anon |
1 << PG_swapcache |
1 << PG_writeback )))
bad_page(function, page);
@@ -355,7 +353,6 @@ static void prep_new_page(struct page *p
1 << PG_dirty |
1 << PG_reclaim |
1 << PG_maplock |
- 1 << PG_anon |
1 << PG_swapcache |
1 << PG_writeback )))
bad_page(__FUNCTION__, page);
diff -puN mm/rmap.c~rmaplock-1-5-pageanon-in-mapping mm/rmap.c
--- 25/mm/rmap.c~rmaplock-1-5-pageanon-in-mapping 2004-08-15 22:49:45.217728872 -0700
+++ 25-akpm/mm/rmap.c 2004-08-15 22:49:45.224727808 -0700
@@ -168,7 +168,6 @@ static inline void clear_page_anon(struc
{
BUG_ON(!page->mapping);
page->mapping = NULL;
- ClearPageAnon(page);
}
/*
@@ -249,7 +248,7 @@ out:
static inline int page_referenced_anon(struct page *page)
{
unsigned int mapcount = page->mapcount;
- struct anon_vma *anon_vma = (struct anon_vma *) page->mapping;
+ struct anon_vma *anon_vma = (void *) page->mapping - PAGE_MAPPING_ANON;
struct vm_area_struct *vma;
int referenced = 0;
@@ -349,31 +348,18 @@ void page_add_anon_rmap(struct page *pag
BUG_ON(PageReserved(page));
BUG_ON(!anon_vma);
+ anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
index = (address - vma->vm_start) >> PAGE_SHIFT;
index += vma->vm_pgoff;
index >>= PAGE_CACHE_SHIFT - PAGE_SHIFT;
- /*
- * Setting and clearing PG_anon must always happen inside
- * page_map_lock to avoid races between mapping and
- * unmapping on different processes of the same
- * shared cow swapcache page. And while we take the
- * page_map_lock PG_anon cannot change from under us.
- * Actually PG_anon cannot change under fork either
- * since fork holds a reference on the page so it cannot
- * be unmapped under fork and in turn copy_page_range is
- * allowed to read PG_anon outside the page_map_lock.
- */
page_map_lock(page);
if (!page->mapcount) {
- BUG_ON(PageAnon(page));
BUG_ON(page->mapping);
- SetPageAnon(page);
page->index = index;
page->mapping = (struct address_space *) anon_vma;
inc_page_state(nr_mapped);
} else {
- BUG_ON(!PageAnon(page));
BUG_ON(page->index != index);
BUG_ON(page->mapping != (struct address_space *) anon_vma);
}
@@ -632,7 +618,7 @@ out_unlock:
static inline int try_to_unmap_anon(struct page *page)
{
- struct anon_vma *anon_vma = (struct anon_vma *) page->mapping;
+ struct anon_vma *anon_vma = (void *) page->mapping - PAGE_MAPPING_ANON;
struct vm_area_struct *vma;
int ret = SWAP_AGAIN;
_