[PATCH 03/18] Compound read / write locking aka get / put.
From: Radosław Smogura <hidden>
Date: 2012-02-16 14:32:50
Also in:
linux-ext4
Subsystem:
memory management, memory management - core, memory management - mglru (multi-gen lru), memory management - page allocator, memory management - swap, the rest · Maintainers:
Andrew Morton, David Hildenbrand, Vlastimil Babka, Chris Li, Kairui Song, Linus Torvalds
Adds compound usage count for pages of higher order. This change is required to add faster locking techniques then compound_lock, and to prevents dead locks during operating on compound page. Signed-off-by: RadosA?aw Smogura <redacted> --- include/linux/mm.h | 127 +++++++++++++++++++++++++++++++++++++++++++--- include/linux/mm_types.h | 15 +++++- mm/page_alloc.c | 4 +- mm/swap.c | 58 +++++++++++++++++++-- 4 files changed, 191 insertions(+), 13 deletions(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index bacb023..72f6a50 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h@@ -284,6 +284,126 @@ static inline void compound_unlock(struct page *page) #endif } +static inline int compound_order(struct page *page) +{ + if (!PageHead(page)) + return 0; + return (unsigned long)page[1]._compound_order; +} + +/** Get's usage count for compound page. + * This involves compound_lock, so do not call it having compound lock + * raised. + * @return 1 - success, 0 - page was splitted. + */ +static inline int compound_get(struct page *head) +{ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + VM_BUG_ON(PageTail(head)); +repeat: + if (!PageHead(head)) + return 0; + + VM_BUG_ON(!atomic_read(&head->_count)); + VM_BUG_ON(compound_order(head) < 2); + + compound_lock(head); + if (unlikely(!PageHead(head))) { + compound_unlock(head); + return 0; + } + + if (atomic_inc_not_zero(&head[2]._compound_usage)) { + compound_unlock(head); + return 1; + } else { + compound_unlock(head); + goto repeat; + } +#else + return 0; +#endif +} + +/** Decrases compound usage count. + * This involves compound_lock, so do not call it having compound lock + * raised. + */ +extern void compound_put(struct page *head); + + + +/** Tries to freeze compound page. If upgrade_lock is true function tries to + * <b>exchange</b> page "gotten" to "forozen" (so after unfreeze page will be + * "not used"), caller must have page excatly once. If upgrade_lock is false + * then page must be "not gotten". + * + * @return 0 - success, -1 splitted, 1 - can't freez, but not splitted + */ +static inline int compound_try_freeze(struct page *head, int upgrade_lock) +{ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + int expected_usage; + + VM_BUG_ON(PageTail(head)); + VM_BUG_ON(compound_order(head) < 2); + VM_BUG_ON(!atomic_read(&head->_count)); + VM_BUG_ON(upgrade_lock && atomic_read(&head[2]._compound_usage) == 1); + + if (!PageHead(head)) + return 0; + + compound_lock(head); + if (!upgrade_lock) { + /* Not needed. Page is gotten so no split, GCC will make this + * faster. + */ + if (unlikely(!PageHead(head))) { + return -1; + } + } + + expected_usage = upgrade_lock ? 2 : 1; + if (atomic_cmpxchg(&head[2]._compound_usage, expected_usage, 0) == 1) { + compound_unlock(head); + return 0; + } else { + compound_unlock(head); + return 1; + } +#else + return 0; +#endif +} + +/** Freeze compound page (like write barrier. + * This involves compound_lock, so do not call it having compound lock + * raised. + * + * @return 1 - success, 0 - page was splitted. + */ +static inline int compound_freeze(struct page *head) +{ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +repeat: + switch (compound_try_freeze(head, false)) { + case 0: + return 1; + case -1: + return 0; + default: + goto repeat; + } +#else + return 1; +#endif +} + +/** Unfreezes compound page. + * Do not call this after you splitted page or you may corrupt memory. + */ +extern void compound_unfreeze(struct page *head); + /** Gets head of compound page. If page is no longer head returns {@code page}. * This function involves makes memory barrier to ensure page was not splitted. */
@@ -485,13 +605,6 @@ static inline compound_page_dtor *get_compound_page_dtor(struct page *page) return page[1]._dtor; } -static inline int compound_order(struct page *page) -{ - if (!PageHead(page)) - return 0; - return (unsigned long)page[1]._compound_order; -} - static inline int compound_trans_order(struct page *page) { int order;
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 05fefae..7649722 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h@@ -117,7 +117,8 @@ struct page { * address 64L. So if we will see here value * less then 64L we are sure it's 2nd page of * compound (so first page is "this - 1"). - * <b>Valid only on 3rd and next elements</b> + * <b>Valid only on 3rd and next elements, + * head[2], head[3]...</b> */ struct page *__first_page; };
@@ -131,6 +132,18 @@ struct page { */ compound_page_dtor *_dtor; + /** Usage count of compound page "as whole". + * This is rather split barrier then something + * usefull. Compound page with order greater + * then 1 should start with this value setted to + * {@code 1} - mean no lock, locking page for + * reading is obtained by bumping lock if not + * zero, locking for splitting by setting it + * to zero when value of counter is {@code 1}. + * <b>Valid only on 3rd element (head[2])</b> + */ + atomic_t _compound_usage; + /** Number of pages in compound page(including * head and tails) that are used (having * {@code _count > 0}). If this number fell to
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index b48e313..bbdd94e 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c@@ -357,8 +357,10 @@ void prep_compound_page(struct page *page, unsigned long order) /* Order, dtor was replaced in for loop, set it correctly. */ set_compound_order(page, order); set_compound_page_dtor(page, free_compound_page); - if (order > 1) + if (order > 1) { atomic_set(&page[3]._tail_count, 0); + atomic_set(&page[2]._compound_usage, 1); + } } /* update __split_huge_page_refcount if you change this function */
diff --git a/mm/swap.c b/mm/swap.c
index 365363c..ded81c9 100644
--- a/mm/swap.c
+++ b/mm/swap.c@@ -104,10 +104,17 @@ int put_compound_head(struct page *head) * __recover_page bumps if head->_count > 0, then at this * point head->_count will be 1 - contradiction. */ - if (PageCompound(head)) - __free_compound_page(head); - else + smp_rmb(); + if (PageCompound(head)) { + if (compound_order(head) > 1) { + if (atomic_read(&head[2]._compound_usage) == 1) + __free_compound_page(head); + } else { + __free_compound_page(head); + } + } else { __put_single_page(head); + } return 1; } return 0;
@@ -173,7 +180,9 @@ int put_compound_tail(struct page *page) VM_BUG_ON(!atomic_read(&head->_count)); /* and this one for get_page_unless_zero(head)*/ - if (atomic_dec_and_test(&head->_count)) { + if (atomic_dec_and_test(&head->_count) && + (atomic_read(&head[2]._compound_usage) + == 1)) { /* Putted last ref - now noone may get * head. Details in put_compound_head */
@@ -201,6 +210,47 @@ int put_compound_tail(struct page *page) } EXPORT_SYMBOL(put_compound_tail); +extern void compound_put(struct page *head) +{ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + VM_BUG_ON(PageTail(head)); + /* Bug if page was splitted. */ + VM_BUG_ON(!PageHead(head)); + VM_BUG_ON(!atomic_read(&head[2]._compound_usage)); + VM_BUG_ON(compound_order(head) < 2); + compound_lock(head); + if (atomic_add_return(-1, &head[2]._compound_usage) == 1) { + if (!atomic_read(&head->_count)) { + compound_unlock(head); + __free_compound_page(head); + } + } + compound_unlock(head); +#endif +} + +extern void compound_unfreeze(struct page *head) +{ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + VM_BUG_ON(PageTail(head)); + VM_BUG_ON(atomic_read(&head[2]._compound_usage)); + VM_BUG_ON(compound_order(head) < 2); + + /* It's quite important to check during "experimental" phase if page is + * unfrozen on splitted page (the counter overlaps lru, so this may + * cause problems. + */ + BUG_ON(!PageCompound(head)); + compound_lock(head); + atomic_set(&head[2]._compound_usage, 1); + if (!atomic_read(&head->_count)) { + compound_unlock(head); + __free_compound_page(head); + } + compound_unlock(head); +#endif +} + void put_page(struct page *page) { if (unlikely(PageCompound(page))) {
--
1.7.3.4
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>