From: Ram Pai <linuxram@us.ibm.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
---
25-akpm/include/linux/fs.h | 7
25-akpm/include/linux/mm.h | 7
25-akpm/mm/filemap.c | 16 +
25-akpm/mm/readahead.c | 431 +++++++++++++++++++--------------------------
4 files changed, 210 insertions(+), 251 deletions(-)
diff -puN include/linux/fs.h~simplified-readahead include/linux/fs.h
--- 25/include/linux/fs.h~simplified-readahead 2004-12-01 21:45:24.212955448 -0800
+++ 25-akpm/include/linux/fs.h 2004-12-01 21:45:24.222953928 -0800
@@ -565,16 +565,17 @@ struct fown_struct {
struct file_ra_state {
unsigned long start; /* Current window */
unsigned long size;
- unsigned long next_size; /* Next window size */
+ unsigned long flags; /* ra flags RA_FLAG_xxx*/
+ unsigned long cache_hit; /* cache hit count*/
unsigned long prev_page; /* Cache last read() position */
unsigned long ahead_start; /* Ahead window */
unsigned long ahead_size;
- unsigned long currnt_wnd_hit; /* locality in the current window */
- unsigned long average; /* size of next current window */
unsigned long ra_pages; /* Maximum readahead window */
unsigned long mmap_hit; /* Cache hit stat for mmap accesses */
unsigned long mmap_miss; /* Cache miss stat for mmap accesses */
};
+#define RA_FLAG_MISS 0x01 /* a cache miss occured against this file */
+#define RA_FLAG_INCACHE 0x02 /* file is already in cache */
struct file {
struct list_head f_list;
diff -puN include/linux/mm.h~simplified-readahead include/linux/mm.h
--- 25/include/linux/mm.h~simplified-readahead 2004-12-01 21:45:24.214955144 -0800
+++ 25-akpm/include/linux/mm.h 2004-12-01 21:45:24.223953776 -0800
@@ -745,15 +745,18 @@ int write_one_page(struct page *page, in
/* readahead.c */
#define VM_MAX_READAHEAD 128 /* kbytes */
#define VM_MIN_READAHEAD 16 /* kbytes (includes current page) */
+#define VM_MAX_CACHE_HIT 256 /* max pages in a row in cache before
+ * turning readahead off */
int do_page_cache_readahead(struct address_space *mapping, struct file *filp,
unsigned long offset, unsigned long nr_to_read);
int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
unsigned long offset, unsigned long nr_to_read);
-void page_cache_readahead(struct address_space *mapping,
+unsigned long page_cache_readahead(struct address_space *mapping,
struct file_ra_state *ra,
struct file *filp,
- unsigned long offset);
+ unsigned long offset,
+ unsigned long size);
void handle_ra_miss(struct address_space *mapping,
struct file_ra_state *ra, pgoff_t offset);
unsigned long max_sane_readahead(unsigned long nr);
diff -puN mm/filemap.c~simplified-readahead mm/filemap.c
--- 25/mm/filemap.c~simplified-readahead 2004-12-01 21:45:24.216954840 -0800
+++ 25-akpm/mm/filemap.c 2004-12-01 21:45:24.225953472 -0800
@@ -689,14 +689,15 @@ void do_generic_mapping_read(struct addr
read_actor_t actor)
{
struct inode *inode = mapping->host;
- unsigned long index, end_index, offset;
+ unsigned long index, end_index, offset, req_size, next_index;
loff_t isize;
struct page *cached_page;
int error;
struct file_ra_state ra = *_ra;
cached_page = NULL;
- index = *ppos >> PAGE_CACHE_SHIFT;
+ next_index = index = *ppos >> PAGE_CACHE_SHIFT;
+ req_size = (desc->count + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
offset = *ppos & ~PAGE_CACHE_MASK;
isize = i_size_read(inode);
@@ -706,7 +707,7 @@ void do_generic_mapping_read(struct addr
end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
for (;;) {
struct page *page;
- unsigned long nr, ret;
+ unsigned long ret_size, nr, ret;
/* nr is the maximum number of bytes to copy from this page */
nr = PAGE_CACHE_SIZE;
@@ -721,7 +722,12 @@ void do_generic_mapping_read(struct addr
nr = nr - offset;
cond_resched();
- page_cache_readahead(mapping, &ra, filp, index);
+ if (index == next_index && req_size) {
+ ret_size = page_cache_readahead(mapping, &ra,
+ filp, index, req_size);
+ next_index += ret_size;
+ req_size -= ret_size;
+ }
find_page:
page = find_get_page(mapping, index);
@@ -1167,7 +1173,7 @@ retry_all:
* For sequential accesses, we use the generic readahead logic.
*/
if (VM_SequentialReadHint(area))
- page_cache_readahead(mapping, ra, file, pgoff);
+ page_cache_readahead(mapping, ra, file, pgoff, 1);
/*
* Do we have something in the page cache already?
diff -puN mm/readahead.c~simplified-readahead mm/readahead.c
--- 25/mm/readahead.c~simplified-readahead 2004-12-01 21:45:24.218954536 -0800
+++ 25-akpm/mm/readahead.c 2004-12-01 21:45:24.228953016 -0800
@@ -35,7 +35,6 @@ void
file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping)
{
ra->ra_pages = mapping->backing_dev_info->ra_pages;
- ra->average = ra->ra_pages / 2;
}
/*
@@ -51,6 +50,63 @@ static inline unsigned long get_min_read
return (VM_MIN_READAHEAD * 1024) / PAGE_CACHE_SIZE;
}
+static inline void ra_off(struct file_ra_state *ra)
+{
+ ra->start = 0;
+ ra->flags = 0;
+ ra->size = -1UL;
+ ra->ahead_start = 0;
+ ra->ahead_size = 0;
+ return;
+}
+
+/*
+ * Set the initial window size, round to next power of 2 and square
+ * for small size, x 4 for medium, and x 2 for large
+ * for 128k (32 page) max ra
+ * 1-8 page = 32k initial, > 8 page = 128k initial
+ */
+unsigned long get_init_ra_size(unsigned long size, unsigned long max)
+{
+ unsigned long s_size=1, newsize;
+
+ do {
+ s_size = s_size << 1;
+ } while ((size = size >> 1));
+ if (s_size <= max / 64) {
+ newsize = s_size * s_size;
+ } else if (s_size <= max/4) {
+ newsize = max / 4;
+ } else {
+ newsize = max;
+ }
+ return newsize;
+}
+
+/*
+ * Set the new window size, this is called only when
+ * I/O is to be submitted, not for each call to readahead
+ * If cache miss occered, reduce next I/O size, else
+ * increase depending on how close to max we are.
+ */
+unsigned long get_next_ra_size(unsigned long cur, unsigned long max,
+ unsigned long min, unsigned long * flags)
+{
+ unsigned long newsize;
+
+ if (*flags & RA_FLAG_MISS) {
+ newsize = max((cur - 2),min);
+ *flags &= ~RA_FLAG_MISS;
+ } else if ( cur < max/16 ) {
+ newsize = 4 * cur;
+ } else {
+ newsize = 2 * cur;
+ }
+ newsize = min(newsize, max);
+ return newsize;
+}
+
+
#define list_to_page(head) (list_entry((head)->prev, struct page, lru))
/**
@@ -65,7 +121,7 @@ static inline unsigned long get_min_read
* Hides the details of the LRU cache etc from the filesystems.
*/
int read_cache_pages(struct address_space *mapping, struct list_head *pages,
- int (*filler)(void *, struct page *), void *data)
+ int (*filler)(void *, struct page *), void *data)
{
struct page *page;
struct pagevec lru_pvec;
@@ -151,19 +207,17 @@ out:
* ahead_size: Together, these form the "ahead window".
* ra_pages: The externally controlled max readahead for this fd.
*
- * When readahead is in the "maximally shrunk" state (next_size == -1UL),
- * readahead is disabled. In this state, prev_page and size are used, inside
- * handle_ra_miss(), to detect the resumption of sequential I/O. Once there
- * has been a decent run of sequential I/O (defined by get_min_readahead),
- * readahead is reenabled.
+ * When readahead is in the off state (size == -1UL),
+ * readahead is disabled. In this state, prev_page is used
+ * to detect the resumption of sequential I/O.
*
* The readahead code manages two windows - the "current" and the "ahead"
* windows. The intent is that while the application is walking the pages
* in the current window, I/O is underway on the ahead window. When the
* current window is fully traversed, it is replaced by the ahead window
* and the ahead window is invalidated. When this copying happens, the
- * new current window's pages are probably still locked. When I/O has
- * completed, we submit a new batch of I/O, creating a new ahead window.
+ * new current window's pages are probably still locked. So
+ * we submit a new batch of I/O immediately, creating a new ahead window.
*
* So:
*
@@ -175,34 +229,25 @@ out:
* ahead window.
*
* A `readahead hit' occurs when a read request is made against a page which is
- * inside the current window. Hits are good, and the window size (next_size)
- * is grown aggressively when hits occur. Two pages are added to the next
- * window size on each hit, which will end up doubling the next window size by
- * the time I/O is submitted for it.
- *
- * If readahead hits are more sparse (say, the application is only reading
- * every second page) then the window will build more slowly.
- *
- * On a readahead miss (the application seeked away) the readahead window is
- * shrunk by 25%. We don't want to drop it too aggressively, because it is a
- * good assumption that an application which has built a good readahead window
- * will continue to perform linear reads. Either at the new file position, or
- * at the old one after another seek.
+ * the next sequential page. Ahead windowe calculations are done only when it
+ * is time to submit a new IO. The code ramps up the size agressively at first,
+ * but slow down as it approaches max_readhead.
*
- * After enough misses, readahead is fully disabled. (next_size = -1UL).
+ * Any seek/ramdom IO will result in readahead being turned off. It will resume
+ * at the first sequential access.
*
* There is a special-case: if the first page which the application tries to
* read happens to be the first page of the file, it is assumed that a linear
- * read is about to happen and the window is immediately set to half of the
- * device maximum.
+ * read is about to happen and the window is immediately set to the initial size
+ * based on I/O request size and the max_readahead.
*
* A page request at (start + size) is not a miss at all - it's just a part of
* sequential file reading.
*
- * This function is to be called for every page which is read, rather than when
- * it is time to perform readahead. This is so the readahead algorithm can
- * centrally work out the access patterns. This could be costly with many tiny
- * read()s, so we specifically optimise for that case with prev_page.
+ * This function is to be called for every read request, rather than when
+ * it is time to perform readahead. It is called only oce for the entire I/O
+ * regardless of size unless readahead is unable to start enough I/O to satisfy
+ * the request (I/O request > max_readahead).
*/
/*
@@ -211,7 +256,7 @@ out:
* behaviour which would occur if page allocations are causing VM writeback.
* We really don't want to intermingle reads and writes like that.
*
- * Returns the number of pages which actually had IO started against them.
+ * Returns the number of pages requested, or the maximum amount of I/O allowed.
*/
static inline int
__do_page_cache_readahead(struct address_space *mapping, struct file *filp,
@@ -300,265 +345,169 @@ int force_page_cache_readahead(struct ad
}
/*
- * This version skips the IO if the queue is read-congested, and will tell the
- * block layer to abandon the readahead if request allocation would block.
- *
- * force_page_cache_readahead() will ignore queue congestion and will block on
- * request queues.
- */
-int do_page_cache_readahead(struct address_space *mapping, struct file *filp,
- unsigned long offset, unsigned long nr_to_read)
-{
- if (!bdi_read_congested(mapping->backing_dev_info))
- return __do_page_cache_readahead(mapping, filp,
- offset, nr_to_read);
- return 0;
-}
-
-/*
* Check how effective readahead is being. If the amount of started IO is
* less than expected then the file is partly or fully in pagecache and
- * readahead isn't helping. Shrink the window.
+ * readahead isn't helping.
*
- * But don't shrink it too much - the application may read the same page
- * occasionally.
*/
-static inline void
-check_ra_success(struct file_ra_state *ra, pgoff_t attempt,
- pgoff_t actual, pgoff_t orig_next_size)
+int check_ra_success(struct file_ra_state *ra, unsigned long nr_to_read,
+ unsigned long actual)
{
if (actual == 0) {
- if (orig_next_size > 1) {
- ra->next_size = orig_next_size - 1;
- if (ra->ahead_size)
- ra->ahead_size = ra->next_size;
- } else {
- ra->next_size = -1UL;
- ra->size = 0;
+ ra->cache_hit += nr_to_read;
+ if (ra->cache_hit >= VM_MAX_CACHE_HIT) {
+ ra->flags |= RA_FLAG_INCACHE;
+ ra_off(ra);
+ return 0;
}
+ } else {
+ ra->cache_hit=0;
}
+ return 1;
+}
+
+/*
+ * Issue the I/O. If pages already in cache, increment the hit count until
+ * we exceed max, then turn RA off until we start missing again.
+ */
+int do_page_cache_readahead(struct address_space *mapping, struct file *filp,
+ unsigned long offset, unsigned long nr_to_read)
+{
+ return __do_page_cache_readahead(mapping, filp, offset, nr_to_read);
}
+
+
+
/*
* page_cache_readahead is the main function. If performs the adaptive
* readahead window size management and submits the readahead I/O.
*/
-void
+unsigned long
page_cache_readahead(struct address_space *mapping, struct file_ra_state *ra,
- struct file *filp, unsigned long offset)
+ struct file *filp, unsigned long offset,
+ unsigned long req_size)
{
- unsigned max;
- unsigned orig_next_size;
- unsigned actual;
- int first_access=0;
- unsigned long average;
+ unsigned long max, min, newsize=req_size;
+ unsigned long actual=0;
/*
* Here we detect the case where the application is performing
* sub-page sized reads. We avoid doing extra work and bogusly
* perturbing the readahead window expansion logic.
- * If next_size is zero, this is the very first read for this
- * file handle, or the window is maximally shrunk.
+ * If size is zero, there is no read ahead window so we need one
*/
- if (offset == ra->prev_page) {
- if (ra->next_size != 0)
- goto out;
+ if (offset == ra->prev_page && req_size == 1 && ra->size != 0) {
+ goto out;
}
- if (ra->next_size == -1UL)
- goto out; /* Maximally shrunk */
-
max = get_max_readahead(ra);
- if (max == 0)
- goto out; /* No readahead */
-
- orig_next_size = ra->next_size;
-
- if (ra->next_size == 0) {
- /*
- * Special case - first read.
- * We'll assume it's a whole-file read, and
- * grow the window fast.
- */
- first_access=1;
- ra->next_size = max / 2;
- ra->prev_page = offset;
- ra->currnt_wnd_hit++;
- goto do_io;
+ min = get_min_readahead(ra);
+// maxsane = max_sane_readahead(max);
+ newsize = min(req_size, max);
+
+ if (newsize == 0 || (ra->flags & RA_FLAG_INCACHE)){
+ newsize = 1;
+ goto out; /* No readahead or file already in cache*/
}
-
- ra->prev_page = offset;
-
- if (offset >= ra->start && offset <= (ra->start + ra->size)) {
- /*
- * A readahead hit. Either inside the window, or one
- * page beyond the end. Expand the next readahead size.
- */
- ra->next_size += 2;
-
- if (ra->currnt_wnd_hit <= (max * 2))
- ra->currnt_wnd_hit++;
- } else {
- /*
- * A miss - lseek, pagefault, pread, etc. Shrink the readahead
- * window.
+ /*
+ * Special case - first read.
+ * We'll assume it's a whole-file read if at start of file, and
+ * grow the window fast.
+ * or detect first sequential access
+ */
+ if ((ra->size == 0 && offset == 0) // first io and start of file
+ || (ra->size == -1UL && ra->prev_page == offset-1)) { //1st seq
+ ra->prev_page = offset + newsize-1;
+ ra->size = get_init_ra_size(newsize, max);
+ ra->start = offset;
+ actual = do_page_cache_readahead(mapping, filp, offset,
+ ra->size);
+ if (!check_ra_success(ra, ra->size, actual)) {
+ goto out;
+ }
+ /* if the request size is larger than our max readahead, we
+ * at least want to be sure that we get 2 IOs if flight and
+ * we know that we will definitly need the new I/O.
+ * once we do this, subsequent calls should be able to overlap
+ * IOs,* thus preventing stalls. so issue the ahead window
+ * immediately.
*/
- ra->next_size -= 2;
-
- average = ra->average;
- if (average < ra->currnt_wnd_hit) {
- average++;
+ if (req_size >= max) {
+ ra->ahead_size = get_next_ra_size(ra->size, max, min,
+ &ra->flags);
+ ra->ahead_start = ra->start + ra->size;
+ actual = do_page_cache_readahead(mapping, filp,
+ ra->ahead_start, ra->ahead_size);
+ check_ra_success(ra, ra->ahead_size, actual);
}
- ra->average = (average + ra->currnt_wnd_hit) / 2;
- ra->currnt_wnd_hit = 1;
+ goto out;
}
- if ((long)ra->next_size > (long)max)
- ra->next_size = max;
- if ((long)ra->next_size <= 0L) {
- ra->next_size = -1UL;
- ra->size = 0;
- goto out; /* Readahead is off */
+ /* now handle the random case:
+ * partial page reads and first access were handled above,
+ * so this must be the next page otherwise it is random
+ */
+ if ((offset != (ra->prev_page+1) || (ra->size == 0))) {
+ ra_off(ra);
+ ra->prev_page = offset + newsize-1;
+ actual = do_page_cache_readahead(mapping, filp, offset,
+ newsize);
+ check_ra_success(ra, newsize, actual);
+ goto out;
}
- /*
- * Is this request outside the current window?
+ /* If we get here we are doing sequential IO and this was
+ * not the first occurence (ie we have an existing window)
*/
- if (offset < ra->start || offset >= (ra->start + ra->size)) {
- /*
- * A miss against the current window. Have we merely
- * advanced into the ahead window?
- */
- if (offset == ra->ahead_start) {
- /*
- * Yes, we have. The ahead window now becomes
- * the current window.
- */
- ra->start = ra->ahead_start;
- ra->size = ra->ahead_size;
- ra->prev_page = ra->start;
- ra->ahead_start = 0;
- ra->ahead_size = 0;
-
- /*
- * Control now returns, probably to sleep until I/O
- * completes against the first ahead page.
- * When the second page in the old ahead window is
- * requested, control will return here and more I/O
- * will be submitted to build the new ahead window.
- */
- goto out;
- }
-do_io:
- /*
- * This is the "unusual" path. We come here during
- * startup or after an lseek. We invalidate the
- * ahead window and get some I/O underway for the new
- * current window.
- */
- if (!first_access) {
- /* Heuristic: there is a high probability
- * that around ra->average number of
- * pages shall be accessed in the next
- * current window.
- */
- average = ra->average;
- if (ra->currnt_wnd_hit > average)
- average = (ra->currnt_wnd_hit + ra->average + 1) / 2;
- ra->next_size = min(average , (unsigned long)max);
- }
- ra->start = offset;
- ra->size = ra->next_size;
- ra->ahead_start = 0; /* Invalidate these */
- ra->ahead_size = 0;
- actual = do_page_cache_readahead(mapping, filp, offset,
- ra->size);
- if(!first_access) {
- /*
- * do not adjust the readahead window size the first
- * time, the ahead window might get closed if all
- * the pages are already in the cache.
- */
- check_ra_success(ra, ra->size, actual, orig_next_size);
- }
- } else {
- /*
- * This read request is within the current window. It may be
- * time to submit I/O for the ahead window while the
- * application is about to step into the ahead window.
- */
- if (ra->ahead_start == 0) {
- /*
- * If the average io-size is more than maximum
- * readahead size of the file the io pattern is
- * sequential. Hence bring in the readahead window
- * immediately.
- * If the average io-size is less than maximum
- * readahead size of the file the io pattern is
- * random. Hence don't bother to readahead.
- */
- average = ra->average;
- if (ra->currnt_wnd_hit > average)
- average = (ra->currnt_wnd_hit + ra->average + 1) / 2;
-
- if (average > max) {
- ra->ahead_start = ra->start + ra->size;
- ra->ahead_size = ra->next_size;
- actual = do_page_cache_readahead(mapping, filp,
- ra->ahead_start, ra->ahead_size);
- check_ra_success(ra, ra->ahead_size,
- actual, orig_next_size);
- }
+ if (ra->ahead_start == 0) { /* no ahead window yet */
+ ra->ahead_size = get_next_ra_size(max(newsize,ra->size),
+ max, min, &ra->flags);
+ ra->ahead_start = ra->start + ra->size;
+ newsize = min (newsize, ra->ahead_start - offset);
+ actual = do_page_cache_readahead(mapping, filp,
+ ra->ahead_start, ra->ahead_size);
+ if (!check_ra_success(ra, ra->ahead_size, actual)) {
+ goto out;
}
}
+ /* already have an ahead window, check if we crossed into it
+ if so, shift windows and issue a new ahead window.
+ Only return the #pages that are in the current window, so that
+ we get called back on the first page of the ahead window which
+ will allow us to submit more IO.
+ */
+ if ((offset + newsize -1) >= ra->ahead_start) {
+ ra->start = ra->ahead_start;
+ ra->size = ra->ahead_size;
+ ra->ahead_start = ra->ahead_start + ra->ahead_size;
+ ra->ahead_size = get_next_ra_size(ra->ahead_size,
+ max, min, &ra->flags);
+ newsize = min (newsize, ra->ahead_start - offset);
+ actual = do_page_cache_readahead(mapping, filp,
+ ra->ahead_start, ra->ahead_size);
+ check_ra_success(ra, ra->ahead_size, actual);
+ }
out:
- return;
+ ra->prev_page = offset + newsize - 1;
+ return(newsize);
}
-
/*
* handle_ra_miss() is called when it is known that a page which should have
* been present in the pagecache (we just did some readahead there) was in fact
* not found. This will happen if it was evicted by the VM (readahead
- * thrashing) or if the readahead window is maximally shrunk.
+ * thrashing)
*
- * If the window has been maximally shrunk (next_size == -1UL) then look to see
- * if we are getting misses against sequential file offsets. If so, and this
- * persists then resume readahead.
- *
- * Otherwise we're thrashing, so shrink the readahead window by three pages.
- * This is because it is grown by two pages on a readahead hit. Theory being
- * that the readahead window size will stabilise around the maximum level at
- * which there is no thrashing.
+ * turn on the cache miss flag in the RA struct, this will cause the RA code
+ * to reduce the RA size on the next read.
*/
void handle_ra_miss(struct address_space *mapping,
struct file_ra_state *ra, pgoff_t offset)
{
- if (ra->next_size == -1UL) {
- const unsigned long max = get_max_readahead(ra);
-
- if (offset != ra->prev_page + 1) {
- ra->size = ra->size?ra->size-1:0; /* Not sequential */
- } else {
- ra->size++; /* A sequential read */
- if (ra->size >= max) { /* Resume readahead */
- ra->start = offset - max;
- ra->next_size = max;
- ra->size = max;
- ra->ahead_start = 0;
- ra->ahead_size = 0;
- ra->average = max / 2;
- }
- }
- ra->prev_page = offset;
- } else {
- const unsigned long min = get_min_readahead(ra);
-
- ra->next_size -= 3;
- if (ra->next_size < min)
- ra->next_size = min;
- }
+ ra->flags |= RA_FLAG_MISS;
+ ra->flags &= ~RA_FLAG_INCACHE;
}
/*
_