From: Hugh Dickins <hugh@veritas.com>
If file-based vmas are to be kept in a tree, according to the file offsets
they map, then adjusting the vma's start pgoff or its end involves
repositioning in the tree, while holding i_shared_lock (and page_table_lock).
We used to avoid that if possible, e.g. when just moving end; but if we're
heading that way, let's now tidy up vma_merge and split_vma, and do all the
locking and adjustment in a new helper vma_adjust. And please, let's call the
next vma in vma_merge "next" rather than "prev".
Since these patches are diffed over 2.6.6-rc2-mm2, they include the NUMA
mpolicy mods which you'll have to remove to go earlier in the series, sorry
for that nuisance. I have intentionally changed the one vma_mpol_equal to
mpol_equal, to make the merge cases more alike.
---
25-akpm/include/linux/mm.h | 2
25-akpm/mm/mmap.c | 161 ++++++++++++++++++++++-----------------------
25-akpm/mm/mremap.c | 7 +
3 files changed, 87 insertions(+), 83 deletions(-)
diff -pN -U1 include/linux/mm.h~rmap-15-vma_adjust include/linux/mm.h
--- 25/include/linux/mm.h~rmap-15-vma_adjust 2004-04-27 20:17:36.559809664 -0700
+++ 25-akpm/include/linux/mm.h 2004-04-27 20:17:36.566808600 -0700
@@ -554,2 +554,4 @@ extern void si_meminfo_node(struct sysin
/* mmap.c */
+extern void vma_adjust(struct vm_area_struct *vma, unsigned long start,
+ unsigned long end, pgoff_t pgoff, struct vm_area_struct *next);
extern void insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
diff -pN -U1 mm/mmap.c~rmap-15-vma_adjust mm/mmap.c
--- 25/mm/mmap.c~rmap-15-vma_adjust 2004-04-27 20:17:36.561809360 -0700
+++ 25-akpm/mm/mmap.c 2004-04-27 20:17:36.568808296 -0700
@@ -68,9 +68,7 @@ EXPORT_SYMBOL(vm_committed_space);
static inline void
-__remove_shared_vm_struct(struct vm_area_struct *vma, struct inode *inode)
+__remove_shared_vm_struct(struct vm_area_struct *vma, struct file *file)
{
- if (inode) {
- if (vma->vm_flags & VM_DENYWRITE)
- atomic_inc(&inode->i_writecount);
- list_del_init(&vma->shared);
- }
+ if (vma->vm_flags & VM_DENYWRITE)
+ atomic_inc(&file->f_dentry->d_inode->i_writecount);
+ list_del_init(&vma->shared);
}
@@ -87,3 +85,3 @@ static void remove_shared_vm_struct(stru
spin_lock(&mapping->i_shared_lock);
- __remove_shared_vm_struct(vma, file->f_dentry->d_inode);
+ __remove_shared_vm_struct(vma, file);
spin_unlock(&mapping->i_shared_lock);
@@ -321,2 +319,50 @@ __insert_vm_struct(struct mm_struct * mm
/*
+ * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that is
+ * already present in an i_mmap{_shared} tree without adjusting the tree.
+ * The following helper function should be used when such adjustments
+ * are necessary. The "next" vma (if any) is to be removed or inserted
+ * before we drop the necessary locks.
+ */
+void vma_adjust(struct vm_area_struct *vma, unsigned long start,
+ unsigned long end, pgoff_t pgoff, struct vm_area_struct *next)
+{
+ struct mm_struct *mm = vma->vm_mm;
+ struct address_space *mapping = NULL;
+ struct file *file = vma->vm_file;
+
+ if (file) {
+ mapping = file->f_mapping;
+ spin_lock(&mapping->i_shared_lock);
+ }
+ spin_lock(&mm->page_table_lock);
+
+ vma->vm_start = start;
+ vma->vm_end = end;
+ vma->vm_pgoff = pgoff;
+
+ if (next) {
+ if (next == vma->vm_next) {
+ /*
+ * vma_merge has merged next into vma, and needs
+ * us to remove next before dropping the locks.
+ */
+ __vma_unlink(mm, next, vma);
+ if (file)
+ __remove_shared_vm_struct(next, file);
+ } else {
+ /*
+ * split_vma has split next from vma, and needs
+ * us to insert next before dropping the locks
+ * (next may either follow vma or precede it).
+ */
+ __insert_vm_struct(mm, next);
+ }
+ }
+
+ spin_unlock(&mm->page_table_lock);
+ if (mapping)
+ spin_unlock(&mapping->i_shared_lock);
+}
+
+/*
* If the vma has a ->close operation then the driver probably needs to release
@@ -392,9 +438,7 @@ static struct vm_area_struct *vma_merge(
{
- spinlock_t *lock = &mm->page_table_lock;
- struct inode *inode = file ? file->f_dentry->d_inode : NULL;
- spinlock_t *i_shared_lock;
+ struct vm_area_struct *next;
/*
- * We later require that vma->vm_flags == vm_flags, so this tests
- * vma->vm_flags & VM_SPECIAL, too.
+ * We later require that vma->vm_flags == vm_flags,
+ * so this tests vma->vm_flags & VM_SPECIAL, too.
*/
@@ -403,8 +447,7 @@ static struct vm_area_struct *vma_merge(
- i_shared_lock = file ? &file->f_mapping->i_shared_lock : NULL;
-
if (!prev) {
- prev = rb_entry(rb_parent, struct vm_area_struct, vm_rb);
+ next = rb_entry(rb_parent, struct vm_area_struct, vm_rb);
goto merge_next;
}
+ next = prev->vm_next;
@@ -414,32 +457,15 @@ static struct vm_area_struct *vma_merge(
if (prev->vm_end == addr &&
- mpol_equal(vma_policy(prev), policy) &&
+ mpol_equal(vma_policy(prev), policy) &&
can_vma_merge_after(prev, vm_flags, file, pgoff)) {
- struct vm_area_struct *next;
- int need_up = 0;
-
- if (unlikely(file && prev->vm_next &&
- prev->vm_next->vm_file == file)) {
- spin_lock(i_shared_lock);
- need_up = 1;
- }
- spin_lock(lock);
- prev->vm_end = end;
-
/*
- * OK, it did. Can we now merge in the successor as well?
+ * OK, it can. Can we now merge in the successor as well?
*/
- next = prev->vm_next;
- if (next && prev->vm_end == next->vm_start &&
- vma_mpol_equal(prev, next) &&
+ if (next && end == next->vm_start &&
+ mpol_equal(policy, vma_policy(next)) &&
can_vma_merge_before(next, vm_flags, file,
pgoff, (end - addr) >> PAGE_SHIFT)) {
- prev->vm_end = next->vm_end;
- __vma_unlink(mm, next, prev);
- __remove_shared_vm_struct(next, inode);
- spin_unlock(lock);
- if (need_up)
- spin_unlock(i_shared_lock);
+ vma_adjust(prev, prev->vm_start,
+ next->vm_end, prev->vm_pgoff, next);
if (file)
fput(file);
-
mm->map_count--;
@@ -447,7 +473,5 @@ static struct vm_area_struct *vma_merge(
kmem_cache_free(vm_area_cachep, next);
- return prev;
- }
- spin_unlock(lock);
- if (need_up)
- spin_unlock(i_shared_lock);
+ } else
+ vma_adjust(prev, prev->vm_start,
+ end, prev->vm_pgoff, NULL);
return prev;
@@ -456,22 +480,13 @@ static struct vm_area_struct *vma_merge(
/*
- * Can this new request be merged in front of prev->vm_next?
+ * Can this new request be merged in front of next?
*/
- prev = prev->vm_next;
- if (prev) {
+ if (next) {
merge_next:
- if (!mpol_equal(policy, vma_policy(prev)))
- return 0;
- if (!can_vma_merge_before(prev, vm_flags, file,
- pgoff, (end - addr) >> PAGE_SHIFT))
- return NULL;
- if (end == prev->vm_start) {
- if (file)
- spin_lock(i_shared_lock);
- spin_lock(lock);
- prev->vm_start = addr;
- prev->vm_pgoff -= (end - addr) >> PAGE_SHIFT;
- spin_unlock(lock);
- if (file)
- spin_unlock(i_shared_lock);
- return prev;
+ if (end == next->vm_start &&
+ mpol_equal(policy, vma_policy(next)) &&
+ can_vma_merge_before(next, vm_flags, file,
+ pgoff, (end - addr) >> PAGE_SHIFT)) {
+ vma_adjust(next, addr,
+ next->vm_end, pgoff, NULL);
+ return next;
}
@@ -1215,3 +1230,2 @@ int split_vma(struct mm_struct * mm, str
struct vm_area_struct *new;
- struct address_space *mapping = NULL;
@@ -1249,20 +1263,7 @@ int split_vma(struct mm_struct * mm, str
- if (vma->vm_file)
- mapping = vma->vm_file->f_mapping;
-
- if (mapping)
- spin_lock(&mapping->i_shared_lock);
- spin_lock(&mm->page_table_lock);
-
- if (new_below) {
- vma->vm_start = addr;
- vma->vm_pgoff += ((addr - new->vm_start) >> PAGE_SHIFT);
- } else
- vma->vm_end = addr;
-
- __insert_vm_struct(mm, new);
-
- spin_unlock(&mm->page_table_lock);
- if (mapping)
- spin_unlock(&mapping->i_shared_lock);
+ if (new_below)
+ vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff +
+ ((addr - new->vm_start) >> PAGE_SHIFT), new);
+ else
+ vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
diff -pN -U1 mm/mremap.c~rmap-15-vma_adjust mm/mremap.c
--- 25/mm/mremap.c~rmap-15-vma_adjust 2004-04-27 20:17:36.562809208 -0700
+++ 25-akpm/mm/mremap.c 2004-04-27 20:17:36.569808144 -0700
@@ -394,5 +394,6 @@ unsigned long do_mremap(unsigned long ad
int pages = (new_len - old_len) >> PAGE_SHIFT;
- spin_lock(&vma->vm_mm->page_table_lock);
- vma->vm_end = addr + new_len;
- spin_unlock(&vma->vm_mm->page_table_lock);
+
+ vma_adjust(vma, vma->vm_start,
+ addr + new_len, vma->vm_pgoff, NULL);
+
current->mm->total_vm += pages;
_