ksm: stable_node point to page and back
Add a pointer to the ksm page into struct stable_node, holding a reference to the page while the node exists. Put a pointer to the stable_node into the ksm page's ->mapping. Then we don't need get_ksm_page() while traversing the stable tree: the page to compare against is sure to be present and correct, even if it's no longer visible through any of its existing rmap_items. And we can handle the forked ksm page case more efficiently: no need to memcmp our way through the tree to find its match. Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk> Cc: Izik Eidus <ieidus@redhat.com> Cc: Andrea Arcangeli <aarcange@redhat.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
parent
7b6ba2c7d3
commit
08beca44df
2 changed files with 51 additions and 72 deletions
|
@ -12,6 +12,8 @@
|
||||||
#include <linux/sched.h>
|
#include <linux/sched.h>
|
||||||
#include <linux/vmstat.h>
|
#include <linux/vmstat.h>
|
||||||
|
|
||||||
|
struct stable_node;
|
||||||
|
|
||||||
#ifdef CONFIG_KSM
|
#ifdef CONFIG_KSM
|
||||||
int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
|
int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
|
||||||
unsigned long end, int advice, unsigned long *vm_flags);
|
unsigned long end, int advice, unsigned long *vm_flags);
|
||||||
|
@ -34,7 +36,8 @@ static inline void ksm_exit(struct mm_struct *mm)
|
||||||
/*
|
/*
|
||||||
* A KSM page is one of those write-protected "shared pages" or "merged pages"
|
* A KSM page is one of those write-protected "shared pages" or "merged pages"
|
||||||
* which KSM maps into multiple mms, wherever identical anonymous page content
|
* which KSM maps into multiple mms, wherever identical anonymous page content
|
||||||
* is found in VM_MERGEABLE vmas. It's a PageAnon page, with NULL anon_vma.
|
* is found in VM_MERGEABLE vmas. It's a PageAnon page, pointing not to any
|
||||||
|
* anon_vma, but to that page's node of the stable tree.
|
||||||
*/
|
*/
|
||||||
static inline int PageKsm(struct page *page)
|
static inline int PageKsm(struct page *page)
|
||||||
{
|
{
|
||||||
|
@ -42,15 +45,22 @@ static inline int PageKsm(struct page *page)
|
||||||
(PAGE_MAPPING_ANON | PAGE_MAPPING_KSM);
|
(PAGE_MAPPING_ANON | PAGE_MAPPING_KSM);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
static inline struct stable_node *page_stable_node(struct page *page)
|
||||||
* But we have to avoid the checking which page_add_anon_rmap() performs.
|
{
|
||||||
*/
|
return PageKsm(page) ? page_rmapping(page) : NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline void set_page_stable_node(struct page *page,
|
||||||
|
struct stable_node *stable_node)
|
||||||
|
{
|
||||||
|
page->mapping = (void *)stable_node +
|
||||||
|
(PAGE_MAPPING_ANON | PAGE_MAPPING_KSM);
|
||||||
|
}
|
||||||
|
|
||||||
static inline void page_add_ksm_rmap(struct page *page)
|
static inline void page_add_ksm_rmap(struct page *page)
|
||||||
{
|
{
|
||||||
if (atomic_inc_and_test(&page->_mapcount)) {
|
if (atomic_inc_and_test(&page->_mapcount))
|
||||||
page->mapping = (void *) (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM);
|
|
||||||
__inc_zone_page_state(page, NR_ANON_PAGES);
|
__inc_zone_page_state(page, NR_ANON_PAGES);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
#else /* !CONFIG_KSM */
|
#else /* !CONFIG_KSM */
|
||||||
|
|
||||||
|
|
99
mm/ksm.c
99
mm/ksm.c
|
@ -107,10 +107,12 @@ struct ksm_scan {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* struct stable_node - node of the stable rbtree
|
* struct stable_node - node of the stable rbtree
|
||||||
|
* @page: pointer to struct page of the ksm page
|
||||||
* @node: rb node of this ksm page in the stable tree
|
* @node: rb node of this ksm page in the stable tree
|
||||||
* @hlist: hlist head of rmap_items using this ksm page
|
* @hlist: hlist head of rmap_items using this ksm page
|
||||||
*/
|
*/
|
||||||
struct stable_node {
|
struct stable_node {
|
||||||
|
struct page *page;
|
||||||
struct rb_node node;
|
struct rb_node node;
|
||||||
struct hlist_head hlist;
|
struct hlist_head hlist;
|
||||||
};
|
};
|
||||||
|
@ -434,23 +436,6 @@ out: page = NULL;
|
||||||
return page;
|
return page;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
* get_ksm_page: checks if the page at the virtual address in rmap_item
|
|
||||||
* is still PageKsm, in which case we can trust the content of the page,
|
|
||||||
* and it returns the gotten page; but NULL if the page has been zapped.
|
|
||||||
*/
|
|
||||||
static struct page *get_ksm_page(struct rmap_item *rmap_item)
|
|
||||||
{
|
|
||||||
struct page *page;
|
|
||||||
|
|
||||||
page = get_mergeable_page(rmap_item);
|
|
||||||
if (page && !PageKsm(page)) {
|
|
||||||
put_page(page);
|
|
||||||
page = NULL;
|
|
||||||
}
|
|
||||||
return page;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Removing rmap_item from stable or unstable tree.
|
* Removing rmap_item from stable or unstable tree.
|
||||||
* This function will clean the information from the stable/unstable tree.
|
* This function will clean the information from the stable/unstable tree.
|
||||||
|
@ -465,6 +450,9 @@ static void remove_rmap_item_from_tree(struct rmap_item *rmap_item)
|
||||||
if (stable_node->hlist.first)
|
if (stable_node->hlist.first)
|
||||||
ksm_pages_sharing--;
|
ksm_pages_sharing--;
|
||||||
else {
|
else {
|
||||||
|
set_page_stable_node(stable_node->page, NULL);
|
||||||
|
put_page(stable_node->page);
|
||||||
|
|
||||||
rb_erase(&stable_node->node, &root_stable_tree);
|
rb_erase(&stable_node->node, &root_stable_tree);
|
||||||
free_stable_node(stable_node);
|
free_stable_node(stable_node);
|
||||||
ksm_pages_shared--;
|
ksm_pages_shared--;
|
||||||
|
@ -740,8 +728,7 @@ out:
|
||||||
* try_to_merge_one_page - take two pages and merge them into one
|
* try_to_merge_one_page - take two pages and merge them into one
|
||||||
* @vma: the vma that holds the pte pointing to page
|
* @vma: the vma that holds the pte pointing to page
|
||||||
* @page: the PageAnon page that we want to replace with kpage
|
* @page: the PageAnon page that we want to replace with kpage
|
||||||
* @kpage: the PageKsm page (or newly allocated page which page_add_ksm_rmap
|
* @kpage: the PageKsm page that we want to map instead of page
|
||||||
* will make PageKsm) that we want to map instead of page
|
|
||||||
*
|
*
|
||||||
* This function returns 0 if the pages were merged, -EFAULT otherwise.
|
* This function returns 0 if the pages were merged, -EFAULT otherwise.
|
||||||
*/
|
*/
|
||||||
|
@ -793,6 +780,9 @@ static int try_to_merge_with_ksm_page(struct rmap_item *rmap_item,
|
||||||
struct vm_area_struct *vma;
|
struct vm_area_struct *vma;
|
||||||
int err = -EFAULT;
|
int err = -EFAULT;
|
||||||
|
|
||||||
|
if (page == kpage) /* ksm page forked */
|
||||||
|
return 0;
|
||||||
|
|
||||||
down_read(&mm->mmap_sem);
|
down_read(&mm->mmap_sem);
|
||||||
if (ksm_test_exit(mm))
|
if (ksm_test_exit(mm))
|
||||||
goto out;
|
goto out;
|
||||||
|
@ -846,6 +836,9 @@ static struct page *try_to_merge_two_pages(struct rmap_item *rmap_item,
|
||||||
goto up;
|
goto up;
|
||||||
|
|
||||||
copy_user_highpage(kpage, page, rmap_item->address, vma);
|
copy_user_highpage(kpage, page, rmap_item->address, vma);
|
||||||
|
|
||||||
|
set_page_stable_node(kpage, NULL); /* mark it PageKsm */
|
||||||
|
|
||||||
err = try_to_merge_one_page(vma, page, kpage);
|
err = try_to_merge_one_page(vma, page, kpage);
|
||||||
up:
|
up:
|
||||||
up_read(&mm->mmap_sem);
|
up_read(&mm->mmap_sem);
|
||||||
|
@ -876,41 +869,31 @@ up:
|
||||||
* This function returns the stable tree node of identical content if found,
|
* This function returns the stable tree node of identical content if found,
|
||||||
* NULL otherwise.
|
* NULL otherwise.
|
||||||
*/
|
*/
|
||||||
static struct stable_node *stable_tree_search(struct page *page,
|
static struct stable_node *stable_tree_search(struct page *page)
|
||||||
struct page **tree_pagep)
|
|
||||||
{
|
{
|
||||||
struct rb_node *node = root_stable_tree.rb_node;
|
struct rb_node *node = root_stable_tree.rb_node;
|
||||||
struct stable_node *stable_node;
|
struct stable_node *stable_node;
|
||||||
|
|
||||||
|
stable_node = page_stable_node(page);
|
||||||
|
if (stable_node) { /* ksm page forked */
|
||||||
|
get_page(page);
|
||||||
|
return stable_node;
|
||||||
|
}
|
||||||
|
|
||||||
while (node) {
|
while (node) {
|
||||||
struct hlist_node *hlist, *hnext;
|
|
||||||
struct rmap_item *tree_rmap_item;
|
|
||||||
struct page *tree_page;
|
|
||||||
int ret;
|
int ret;
|
||||||
|
|
||||||
|
cond_resched();
|
||||||
stable_node = rb_entry(node, struct stable_node, node);
|
stable_node = rb_entry(node, struct stable_node, node);
|
||||||
hlist_for_each_entry_safe(tree_rmap_item, hlist, hnext,
|
|
||||||
&stable_node->hlist, hlist) {
|
|
||||||
BUG_ON(!in_stable_tree(tree_rmap_item));
|
|
||||||
cond_resched();
|
|
||||||
tree_page = get_ksm_page(tree_rmap_item);
|
|
||||||
if (tree_page)
|
|
||||||
break;
|
|
||||||
remove_rmap_item_from_tree(tree_rmap_item);
|
|
||||||
}
|
|
||||||
if (!hlist)
|
|
||||||
return NULL;
|
|
||||||
|
|
||||||
ret = memcmp_pages(page, tree_page);
|
ret = memcmp_pages(page, stable_node->page);
|
||||||
|
|
||||||
if (ret < 0) {
|
if (ret < 0)
|
||||||
put_page(tree_page);
|
|
||||||
node = node->rb_left;
|
node = node->rb_left;
|
||||||
} else if (ret > 0) {
|
else if (ret > 0)
|
||||||
put_page(tree_page);
|
|
||||||
node = node->rb_right;
|
node = node->rb_right;
|
||||||
} else {
|
else {
|
||||||
*tree_pagep = tree_page;
|
get_page(stable_node->page);
|
||||||
return stable_node;
|
return stable_node;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -932,26 +915,12 @@ static struct stable_node *stable_tree_insert(struct page *kpage)
|
||||||
struct stable_node *stable_node;
|
struct stable_node *stable_node;
|
||||||
|
|
||||||
while (*new) {
|
while (*new) {
|
||||||
struct hlist_node *hlist, *hnext;
|
|
||||||
struct rmap_item *tree_rmap_item;
|
|
||||||
struct page *tree_page;
|
|
||||||
int ret;
|
int ret;
|
||||||
|
|
||||||
|
cond_resched();
|
||||||
stable_node = rb_entry(*new, struct stable_node, node);
|
stable_node = rb_entry(*new, struct stable_node, node);
|
||||||
hlist_for_each_entry_safe(tree_rmap_item, hlist, hnext,
|
|
||||||
&stable_node->hlist, hlist) {
|
|
||||||
BUG_ON(!in_stable_tree(tree_rmap_item));
|
|
||||||
cond_resched();
|
|
||||||
tree_page = get_ksm_page(tree_rmap_item);
|
|
||||||
if (tree_page)
|
|
||||||
break;
|
|
||||||
remove_rmap_item_from_tree(tree_rmap_item);
|
|
||||||
}
|
|
||||||
if (!hlist)
|
|
||||||
return NULL;
|
|
||||||
|
|
||||||
ret = memcmp_pages(kpage, tree_page);
|
ret = memcmp_pages(kpage, stable_node->page);
|
||||||
put_page(tree_page);
|
|
||||||
|
|
||||||
parent = *new;
|
parent = *new;
|
||||||
if (ret < 0)
|
if (ret < 0)
|
||||||
|
@ -977,6 +946,10 @@ static struct stable_node *stable_tree_insert(struct page *kpage)
|
||||||
|
|
||||||
INIT_HLIST_HEAD(&stable_node->hlist);
|
INIT_HLIST_HEAD(&stable_node->hlist);
|
||||||
|
|
||||||
|
get_page(kpage);
|
||||||
|
stable_node->page = kpage;
|
||||||
|
set_page_stable_node(kpage, stable_node);
|
||||||
|
|
||||||
return stable_node;
|
return stable_node;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1085,14 +1058,10 @@ static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item)
|
||||||
remove_rmap_item_from_tree(rmap_item);
|
remove_rmap_item_from_tree(rmap_item);
|
||||||
|
|
||||||
/* We first start with searching the page inside the stable tree */
|
/* We first start with searching the page inside the stable tree */
|
||||||
stable_node = stable_tree_search(page, &tree_page);
|
stable_node = stable_tree_search(page);
|
||||||
if (stable_node) {
|
if (stable_node) {
|
||||||
kpage = tree_page;
|
kpage = stable_node->page;
|
||||||
if (page == kpage) /* forked */
|
err = try_to_merge_with_ksm_page(rmap_item, page, kpage);
|
||||||
err = 0;
|
|
||||||
else
|
|
||||||
err = try_to_merge_with_ksm_page(rmap_item,
|
|
||||||
page, kpage);
|
|
||||||
if (!err) {
|
if (!err) {
|
||||||
/*
|
/*
|
||||||
* The page was successfully merged:
|
* The page was successfully merged:
|
||||||
|
|
Loading…
Reference in a new issue