radix-tree: use iterators in find_get_pages* functions
Replace radix_tree_gang_lookup_slot() and radix_tree_gang_lookup_tag_slot() in page-cache lookup functions with brand-new radix-tree direct iterating. This avoids the double-scanning and pointer copying. Iterator don't stop after nr_pages page-get fails in a row, it continue lookup till the radix-tree end. Thus we can safely remove these restart conditions. Unfortunately, old implementation didn't forbid nr_pages == 0, this corner case does not fit into new code, so the patch adds an extra check at the beginning. Signed-off-by: Konstantin Khlebnikov <khlebnikov@openvz.org> Tested-by: Hugh Dickins <hughd@google.com> Cc: Christoph Hellwig <hch@lst.de> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
parent
cebbd29e1c
commit
0fc9d10403
1 changed files with 38 additions and 48 deletions
86
mm/filemap.c
86
mm/filemap.c
|
@ -813,20 +813,19 @@ EXPORT_SYMBOL(find_or_create_page);
|
|||
unsigned find_get_pages(struct address_space *mapping, pgoff_t start,
|
||||
unsigned int nr_pages, struct page **pages)
|
||||
{
|
||||
unsigned int i;
|
||||
unsigned int ret;
|
||||
unsigned int nr_found, nr_skip;
|
||||
struct radix_tree_iter iter;
|
||||
void **slot;
|
||||
unsigned ret = 0;
|
||||
|
||||
if (unlikely(!nr_pages))
|
||||
return 0;
|
||||
|
||||
rcu_read_lock();
|
||||
restart:
|
||||
nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree,
|
||||
(void ***)pages, NULL, start, nr_pages);
|
||||
ret = 0;
|
||||
nr_skip = 0;
|
||||
for (i = 0; i < nr_found; i++) {
|
||||
radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
|
||||
struct page *page;
|
||||
repeat:
|
||||
page = radix_tree_deref_slot((void **)pages[i]);
|
||||
page = radix_tree_deref_slot(slot);
|
||||
if (unlikely(!page))
|
||||
continue;
|
||||
|
||||
|
@ -837,7 +836,7 @@ repeat:
|
|||
* when entry at index 0 moves out of or back
|
||||
* to root: none yet gotten, safe to restart.
|
||||
*/
|
||||
WARN_ON(start | i);
|
||||
WARN_ON(iter.index);
|
||||
goto restart;
|
||||
}
|
||||
/*
|
||||
|
@ -845,7 +844,6 @@ repeat:
|
|||
* here as an exceptional entry: so skip over it -
|
||||
* we only reach this from invalidate_mapping_pages().
|
||||
*/
|
||||
nr_skip++;
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -853,21 +851,16 @@ repeat:
|
|||
goto repeat;
|
||||
|
||||
/* Has the page moved? */
|
||||
if (unlikely(page != *((void **)pages[i]))) {
|
||||
if (unlikely(page != *slot)) {
|
||||
page_cache_release(page);
|
||||
goto repeat;
|
||||
}
|
||||
|
||||
pages[ret] = page;
|
||||
ret++;
|
||||
if (++ret == nr_pages)
|
||||
break;
|
||||
}
|
||||
|
||||
/*
|
||||
* If all entries were removed before we could secure them,
|
||||
* try again, because callers stop trying once 0 is returned.
|
||||
*/
|
||||
if (unlikely(!ret && nr_found > nr_skip))
|
||||
goto restart;
|
||||
rcu_read_unlock();
|
||||
return ret;
|
||||
}
|
||||
|
@ -887,21 +880,22 @@ repeat:
|
|||
unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
|
||||
unsigned int nr_pages, struct page **pages)
|
||||
{
|
||||
unsigned int i;
|
||||
unsigned int ret;
|
||||
unsigned int nr_found;
|
||||
struct radix_tree_iter iter;
|
||||
void **slot;
|
||||
unsigned int ret = 0;
|
||||
|
||||
if (unlikely(!nr_pages))
|
||||
return 0;
|
||||
|
||||
rcu_read_lock();
|
||||
restart:
|
||||
nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree,
|
||||
(void ***)pages, NULL, index, nr_pages);
|
||||
ret = 0;
|
||||
for (i = 0; i < nr_found; i++) {
|
||||
radix_tree_for_each_contig(slot, &mapping->page_tree, &iter, index) {
|
||||
struct page *page;
|
||||
repeat:
|
||||
page = radix_tree_deref_slot((void **)pages[i]);
|
||||
page = radix_tree_deref_slot(slot);
|
||||
/* The hole, there no reason to continue */
|
||||
if (unlikely(!page))
|
||||
continue;
|
||||
break;
|
||||
|
||||
if (radix_tree_exception(page)) {
|
||||
if (radix_tree_deref_retry(page)) {
|
||||
|
@ -924,7 +918,7 @@ repeat:
|
|||
goto repeat;
|
||||
|
||||
/* Has the page moved? */
|
||||
if (unlikely(page != *((void **)pages[i]))) {
|
||||
if (unlikely(page != *slot)) {
|
||||
page_cache_release(page);
|
||||
goto repeat;
|
||||
}
|
||||
|
@ -934,14 +928,14 @@ repeat:
|
|||
* otherwise we can get both false positives and false
|
||||
* negatives, which is just confusing to the caller.
|
||||
*/
|
||||
if (page->mapping == NULL || page->index != index) {
|
||||
if (page->mapping == NULL || page->index != iter.index) {
|
||||
page_cache_release(page);
|
||||
break;
|
||||
}
|
||||
|
||||
pages[ret] = page;
|
||||
ret++;
|
||||
index++;
|
||||
if (++ret == nr_pages)
|
||||
break;
|
||||
}
|
||||
rcu_read_unlock();
|
||||
return ret;
|
||||
|
@ -962,19 +956,20 @@ EXPORT_SYMBOL(find_get_pages_contig);
|
|||
unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,
|
||||
int tag, unsigned int nr_pages, struct page **pages)
|
||||
{
|
||||
unsigned int i;
|
||||
unsigned int ret;
|
||||
unsigned int nr_found;
|
||||
struct radix_tree_iter iter;
|
||||
void **slot;
|
||||
unsigned ret = 0;
|
||||
|
||||
if (unlikely(!nr_pages))
|
||||
return 0;
|
||||
|
||||
rcu_read_lock();
|
||||
restart:
|
||||
nr_found = radix_tree_gang_lookup_tag_slot(&mapping->page_tree,
|
||||
(void ***)pages, *index, nr_pages, tag);
|
||||
ret = 0;
|
||||
for (i = 0; i < nr_found; i++) {
|
||||
radix_tree_for_each_tagged(slot, &mapping->page_tree,
|
||||
&iter, *index, tag) {
|
||||
struct page *page;
|
||||
repeat:
|
||||
page = radix_tree_deref_slot((void **)pages[i]);
|
||||
page = radix_tree_deref_slot(slot);
|
||||
if (unlikely(!page))
|
||||
continue;
|
||||
|
||||
|
@ -998,21 +993,16 @@ repeat:
|
|||
goto repeat;
|
||||
|
||||
/* Has the page moved? */
|
||||
if (unlikely(page != *((void **)pages[i]))) {
|
||||
if (unlikely(page != *slot)) {
|
||||
page_cache_release(page);
|
||||
goto repeat;
|
||||
}
|
||||
|
||||
pages[ret] = page;
|
||||
ret++;
|
||||
if (++ret == nr_pages)
|
||||
break;
|
||||
}
|
||||
|
||||
/*
|
||||
* If all entries were removed before we could secure them,
|
||||
* try again, because callers stop trying once 0 is returned.
|
||||
*/
|
||||
if (unlikely(!ret && nr_found))
|
||||
goto restart;
|
||||
rcu_read_unlock();
|
||||
|
||||
if (ret)
|
||||
|
|
Loading…
Reference in a new issue