diff --git a/include/linux/kmsan.h b/include/linux/kmsan.h
index 7da9fd506b39..0016c032d1e7 100644
--- a/include/linux/kmsan.h
+++ b/include/linux/kmsan.h
@@ -150,11 +150,14 @@ int __must_check kmsan_vmap_pages_range_noflush(unsigned long start,
* kmsan_vunmap_kernel_range_noflush() - Notify KMSAN about a vunmap.
* @start: start of vunmapped range.
* @end: end of vunmapped range.
+ * @can_resched: whether the unmap walk may call cond_resched(); pass
+ * false from atomic context.
*
* KMSAN unmaps the contiguous metadata ranges created by
* kmsan_map_kernel_range_noflush().
*/
-void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end);
+void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end,
+ bool can_resched);
/**
* kmsan_ioremap_page_range() - Notify KMSAN about a ioremap_page_range() call.
@@ -355,7 +358,8 @@ static inline int __must_check kmsan_vmap_pages_range_noflush(
}
static inline void kmsan_vunmap_range_noflush(unsigned long start,
- unsigned long end)
+ unsigned long end,
+ bool can_resched)
{
}
diff --git a/mm/kmsan/hooks.c b/mm/kmsan/hooks.c
index 107f66841274..0cd1edaa1120 100644
--- a/mm/kmsan/hooks.c
+++ b/mm/kmsan/hooks.c
@@ -136,13 +136,16 @@ static unsigned long vmalloc_origin(unsigned long addr)
KMSAN_META_ORIGIN);
}
-void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end)
+void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end,
+ bool can_resched)
{
if (!kmsan_enabled)
return;
- __vunmap_range_noflush(vmalloc_shadow(start), vmalloc_shadow(end));
- __vunmap_range_noflush(vmalloc_origin(start), vmalloc_origin(end));
+ __vunmap_range_noflush(vmalloc_shadow(start), vmalloc_shadow(end),
+ can_resched);
+ __vunmap_range_noflush(vmalloc_origin(start), vmalloc_origin(end),
+ can_resched);
flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
}@@ -171,8 +174,9 @@ static void kmsan_iounmap_pages(unsigned long start, unsigned long end)
if (origin)
__free_pages(origin, KMSAN_IOREMAP_META_ORDER);
}
- __vunmap_range_noflush(shadow_start, shadow_end);
- __vunmap_range_noflush(origin_start, origin_end);
+ /* ioremap metadata mappings are always GFP_KERNEL, so resched is safe. */
+ __vunmap_range_noflush(shadow_start, shadow_end, true);
+ __vunmap_range_noflush(origin_start, origin_end, true);
flush_tlb_kernel_range(shadow_start, shadow_end);
flush_tlb_kernel_range(origin_start, origin_end);
}@@ -207,7 +211,7 @@ int kmsan_ioremap_page_range(unsigned long start, unsigned long end,
mapped = __vmap_pages_range_noflush(
vmalloc_shadow(start + off),
vmalloc_shadow(start + off + PAGE_SIZE), prot, &shadow,
- PAGE_SHIFT);
+ PAGE_SHIFT, gfp_mask);
if (mapped) {
err = mapped;
goto ret;@@ -217,7 +221,7 @@ int kmsan_ioremap_page_range(unsigned long start, unsigned long end,
mapped = __vmap_pages_range_noflush(
vmalloc_origin(start + off),
vmalloc_origin(start + off + PAGE_SIZE), prot, &origin,
- PAGE_SHIFT);
+ PAGE_SHIFT, gfp_mask);
if (mapped) {
err = mapped;
goto ret;diff --git a/mm/kmsan/shadow.c b/mm/kmsan/shadow.c
index 2166086d3dc3..452760a333e4 100644
--- a/mm/kmsan/shadow.c
+++ b/mm/kmsan/shadow.c
@@ -247,7 +247,7 @@ int kmsan_vmap_pages_range_noflush(unsigned long start, unsigned long end,
origin_end = vmalloc_meta((void *)end, KMSAN_META_ORIGIN);
kmsan_enter_runtime();
mapped = __vmap_pages_range_noflush(shadow_start, shadow_end, prot,
- s_pages, page_shift);
+ s_pages, page_shift, gfp_mask);
kmsan_leave_runtime();
if (mapped) {
err = mapped;@@ -255,12 +255,13 @@ int kmsan_vmap_pages_range_noflush(unsigned long start, unsigned long end,
}
kmsan_enter_runtime();
mapped = __vmap_pages_range_noflush(origin_start, origin_end, prot,
- o_pages, page_shift);
+ o_pages, page_shift, gfp_mask);
kmsan_leave_runtime();
if (mapped) {
/* Undo the shadow mapping set up above. */
kmsan_enter_runtime();
- __vunmap_range_noflush(shadow_start, shadow_end);
+ __vunmap_range_noflush(shadow_start, shadow_end,
+ gfpflags_allow_blocking(gfp_mask));
kmsan_leave_runtime();
err = mapped;
goto ret;diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 3f98056f8c7e..d1986f68eab4 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -372,8 +372,12 @@ int vmap_page_range(unsigned long addr, unsigned long end,
if (!err)
err = kmsan_ioremap_page_range(addr, end, phys_addr, prot,
ioremap_max_page_shift);
- if (!err)
+ if (err) {
+ /* Callers are ioremap and arch setup code, always sleepable. */
+ __vunmap_range_noflush(addr, end, true);
+ } else {
flush_cache_vmap(addr, end);
+ }
return err;
}@@ -429,7 +433,7 @@ static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
}
static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
- pgtbl_mod_mask *mask)
+ pgtbl_mod_mask *mask, bool can_resched)
{
pmd_t *pmd;
unsigned long next;@@ -451,12 +455,14 @@ static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
continue;
vunmap_pte_range(pmd, addr, next, mask);
- cond_resched();
+ /* The undo path may run in atomic context. */
+ if (can_resched)
+ cond_resched();
} while (pmd++, addr = next, addr != end);
}
static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
- pgtbl_mod_mask *mask)
+ pgtbl_mod_mask *mask, bool can_resched)
{
pud_t *pud;
unsigned long next;@@ -476,12 +482,12 @@ static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
}
if (pud_none_or_clear_bad(pud))
continue;
- vunmap_pmd_range(pud, addr, next, mask);
+ vunmap_pmd_range(pud, addr, next, mask, can_resched);
} while (pud++, addr = next, addr != end);
}
static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
- pgtbl_mod_mask *mask)
+ pgtbl_mod_mask *mask, bool can_resched)
{
p4d_t *p4d;
unsigned long next;@@ -496,7 +502,7 @@ static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
if (p4d_none_or_clear_bad(p4d))
continue;
- vunmap_pud_range(p4d, addr, next, mask);
+ vunmap_pud_range(p4d, addr, next, mask, can_resched);
} while (p4d++, addr = next, addr != end);
}
@@ -512,7 +518,8 @@ static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
*
* This is an internal function only. Do not use outside mm/.
*/
-void __vunmap_range_noflush(unsigned long start, unsigned long end)
+void __vunmap_range_noflush(unsigned long start, unsigned long end,
+ bool can_resched)
{
unsigned long next;
pgd_t *pgd;@@ -529,7 +536,7 @@ void __vunmap_range_noflush(unsigned long start, unsigned long end)
mask |= PGTBL_PGD_MODIFIED;
if (pgd_none_or_clear_bad(pgd))
continue;
- vunmap_p4d_range(pgd, addr, next, &mask);
+ vunmap_p4d_range(pgd, addr, next, &mask, can_resched);
} while (pgd++, addr = next, addr != end);
if (mask & ARCH_PAGE_TABLE_SYNC_MASK)@@ -538,8 +545,9 @@ void __vunmap_range_noflush(unsigned long start, unsigned long end)
void vunmap_range_noflush(unsigned long start, unsigned long end)
{
- kmsan_vunmap_range_noflush(start, end);
- __vunmap_range_noflush(start, end);
+ /* The unmap paths are sleepable, keep the resched point. */
+ kmsan_vunmap_range_noflush(start, end, true);
+ __vunmap_range_noflush(start, end, true);
}
/**@@ -727,18 +735,32 @@ static int vmap_pages_range_noflush_walk(unsigned long addr, unsigned long end,
* The caller is responsible for calling flush_cache_vmap() after this
* function returns successfully and before the addresses are accessed.
*
+ * On failure the mapping is torn down before returning, so a retry
+ * sees a clean range. When @gfp_mask cannot block, the teardown
+ * skips cond_resched(), it may run in atomic context.
+ *
* This is an internal function only. Do not use outside mm/.
*/
int __vmap_pages_range_noflush(unsigned long addr, unsigned long end,
- pgprot_t prot, struct page **pages, unsigned int page_shift)
+ pgprot_t prot, struct page **pages, unsigned int page_shift,
+ gfp_t gfp_mask)
{
+ int err;
+
if (WARN_ON_ONCE(page_shift < PAGE_SHIFT))
return -EINVAL;
if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMALLOC))
page_shift = PAGE_SHIFT;
- return vmap_pages_range_noflush_walk(addr, end, prot, pages, page_shift);
+ err = vmap_pages_range_noflush_walk(addr, end, prot, pages, page_shift);
+
+ /* Undo the PTEs installed before the failure. */
+ if (err)
+ __vunmap_range_noflush(addr, end,
+ gfpflags_allow_blocking(gfp_mask));