From: Andrew Morton <akpm@osdl.org>
Hash the page_uptodate_locks. <insert performance numbers here>
Signed-off-by: Andrew Morton <akpm@osdl.org>
---
fs/buffer.c | 34 ++++++++++++++++++++++++++--------
1 files changed, 26 insertions(+), 8 deletions(-)
diff -puN fs/buffer.c~page_uptodate_lock-hashing fs/buffer.c
--- 25/fs/buffer.c~page_uptodate_lock-hashing 2005-05-03 15:17:09.000000000 -0700
+++ 25-akpm/fs/buffer.c 2005-05-03 15:31:41.000000000 -0700
@@ -532,13 +532,26 @@ static void free_more_memory(void)
}
}
+struct hashed_spinlock {
+ spinlock_t lock;
+} ____cacheline_aligned_in_smp;
+
+#define HSL_SIZE 32
+
+struct hashed_spinlocks {
+ struct hashed_spinlock locks[HSL_SIZE];
+};
+
/*
* I/O completion handler for block_read_full_page() - pages
* which come unlocked at the end of I/O.
*/
static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
{
- static DEFINE_SPINLOCK(page_uptodate_lock);
+ static struct hashed_spinlocks locks = {
+ .locks = { [ 0 ... HSL_SIZE - 1 ] =
+ { .lock = SPIN_LOCK_UNLOCKED } } };
+ spinlock_t *lock;
unsigned long flags;
struct buffer_head *tmp;
struct page *page;
@@ -561,7 +574,8 @@ static void end_buffer_async_read(struct
* two buffer heads end IO at almost the same time and both
* decide that the page is now completely done.
*/
- spin_lock_irqsave(&page_uptodate_lock, flags);
+ lock = &locks.locks[page_to_pfn(page) & (HSL_SIZE - 1)].lock;
+ spin_lock_irqsave(lock, flags);
clear_buffer_async_read(bh);
unlock_buffer(bh);
tmp = bh;
@@ -574,7 +588,7 @@ static void end_buffer_async_read(struct
}
tmp = tmp->b_this_page;
} while (tmp != bh);
- spin_unlock_irqrestore(&page_uptodate_lock, flags);
+ spin_unlock_irqrestore(lock, flags);
/*
* If none of the buffers had errors and they are all
@@ -586,7 +600,7 @@ static void end_buffer_async_read(struct
return;
still_busy:
- spin_unlock_irqrestore(&page_uptodate_lock, flags);
+ spin_unlock_irqrestore(lock, flags);
return;
}
@@ -597,7 +611,10 @@ still_busy:
void end_buffer_async_write(struct buffer_head *bh, int uptodate)
{
char b[BDEVNAME_SIZE];
- static DEFINE_SPINLOCK(page_uptodate_lock);
+ static struct hashed_spinlocks locks = {
+ .locks = { [ 0 ... HSL_SIZE - 1 ] =
+ { .lock = SPIN_LOCK_UNLOCKED } } };
+ spinlock_t *lock;
unsigned long flags;
struct buffer_head *tmp;
struct page *page;
@@ -619,7 +636,8 @@ void end_buffer_async_write(struct buffe
SetPageError(page);
}
- spin_lock_irqsave(&page_uptodate_lock, flags);
+ lock = &locks.locks[page_to_pfn(page) & (HSL_SIZE - 1)].lock;
+ spin_lock_irqsave(lock, flags);
clear_buffer_async_write(bh);
unlock_buffer(bh);
tmp = bh->b_this_page;
@@ -630,12 +648,12 @@ void end_buffer_async_write(struct buffe
}
tmp = tmp->b_this_page;
}
- spin_unlock_irqrestore(&page_uptodate_lock, flags);
+ spin_unlock_irqrestore(lock, flags);
end_page_writeback(page);
return;
still_busy:
- spin_unlock_irqrestore(&page_uptodate_lock, flags);
+ spin_unlock_irqrestore(lock, flags);
return;
}
_