[PATCH v3 50/66] mm/ksm: Use maple tree iterators instead of vma linked list
From: Liam Howlett <hidden>
Date: 2021-10-05 01:33:43
Also in:
lkml
Subsystem:
memory management, memory management - ksm (kernel samepage merging), the rest · Maintainers:
Andrew Morton, David Hildenbrand, Linus Torvalds
From: "Liam R. Howlett" <redacted> Signed-off-by: Liam R. Howlett <redacted> --- mm/ksm.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-)
diff --git a/mm/ksm.c b/mm/ksm.c
index a5716fdec1aa..fe93a55f5158 100644
--- a/mm/ksm.c
+++ b/mm/ksm.c@@ -968,11 +968,14 @@ static int unmerge_and_remove_all_rmap_items(void) struct mm_slot, mm_list); spin_unlock(&ksm_mmlist_lock); - for (mm_slot = ksm_scan.mm_slot; - mm_slot != &ksm_mm_head; mm_slot = ksm_scan.mm_slot) { + for (mm_slot = ksm_scan.mm_slot; mm_slot != &ksm_mm_head; + mm_slot = ksm_scan.mm_slot) { + MA_STATE(mas, &mm_slot->mm->mm_mt, 0, 0); + mm = mm_slot->mm; mmap_read_lock(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) { + rcu_read_lock(); + mas_for_each(&mas, vma, ULONG_MAX) { if (ksm_test_exit(mm)) break; if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)
@@ -984,6 +987,7 @@ static int unmerge_and_remove_all_rmap_items(void) } remove_trailing_rmap_items(&mm_slot->rmap_list); + rcu_read_unlock(); mmap_read_unlock(mm); spin_lock(&ksm_mmlist_lock);
@@ -1007,6 +1011,7 @@ static int unmerge_and_remove_all_rmap_items(void) return 0; error: + rcu_read_unlock(); mmap_read_unlock(mm); spin_lock(&ksm_mmlist_lock); ksm_scan.mm_slot = &ksm_mm_head;
@@ -2221,6 +2226,7 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) struct vm_area_struct *vma; struct rmap_item *rmap_item; int nid; + MA_STATE(mas, NULL, 0, 0); if (list_empty(&ksm_mm_head.mm_list)) return NULL;
@@ -2278,13 +2284,15 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) } mm = slot->mm; + mas.tree = &mm->mm_mt; + mas_set(&mas, ksm_scan.address); + mmap_read_lock(mm); + rcu_read_lock(); if (ksm_test_exit(mm)) - vma = NULL; - else - vma = find_vma(mm, ksm_scan.address); + goto no_vmas; - for (; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { if (!(vma->vm_flags & VM_MERGEABLE)) continue; if (ksm_scan.address < vma->vm_start)
@@ -2312,6 +2320,7 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) ksm_scan.address += PAGE_SIZE; } else put_page(*page); + rcu_read_unlock(); mmap_read_unlock(mm); return rmap_item; }
@@ -2322,6 +2331,7 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) } if (ksm_test_exit(mm)) { +no_vmas: ksm_scan.address = 0; ksm_scan.rmap_list = &slot->rmap_list; }
@@ -2350,9 +2360,11 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) free_mm_slot(slot); clear_bit(MMF_VM_MERGEABLE, &mm->flags); + rcu_read_unlock(); mmap_read_unlock(mm); mmdrop(mm); } else { + rcu_read_unlock(); mmap_read_unlock(mm); /* * mmap_read_unlock(mm) first because after
--
2.30.2