[PATCH v5 11/39] x86/mm: Update pte_modify for _PAGE_COW
From: Rick Edgecombe <rick.p.edgecombe@intel.com>
Date: 2023-01-19 21:34:02
Also in:
linux-arch, linux-doc, linux-mm, lkml
Subsystem:
the rest, x86 architecture (32-bit and 64-bit) · Maintainers:
Linus Torvalds, Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen
From: Yu-cheng Yu <redacted> The Write=0,Dirty=1 PTE has been used to indicate copy-on-write pages. However, newer x86 processors also regard a Write=0,Dirty=1 PTE as a shadow stack page. In order to separate the two, the software-defined _PAGE_DIRTY is changed to _PAGE_COW for the copy-on-write case, and pte_*() are updated to do this. pte_modify() takes a "raw" pgprot_t which was not necessarily created with any of the existing PTE bit helpers. That means that it can return a pte_t with Write=0,Dirty=1, a shadow stack PTE, when it did not intend to create one. However pte_modify() changes a PTE to 'newprot', but it doesn't use the pte_*(). Modify it to also move _PAGE_DIRTY to _PAGE_COW. Do this by using the pte_mkdirty() helper. Since pte_mkdirty() also sets the soft dirty bit, extract a helper that optionally doesn't set _PAGE_SOFT_DIRTY. This helper will allow future logic for deciding when to move _PAGE_DIRTY to _PAGE_COW can live in one place. Apply the same changes to pmd_modify(). Tested-by: Pengfei Xu <redacted> Tested-by: John Allen <john.allen@amd.com> Signed-off-by: Yu-cheng Yu <redacted> Co-developed-by: Rick Edgecombe <rick.p.edgecombe@intel.com> Signed-off-by: Rick Edgecombe <rick.p.edgecombe@intel.com> --- v5: - Fix pte_modify() again, to not lose _PAGE_DIRTY, but still not set _PAGE_SOFT_DIRTY as was fixed in v4. v4: - Fix an issue in soft-dirty test, where pte_modify() would detect _PAGE_COW in pte_dirty() and set the soft dirty bit in pte_mkdirty(). v2: - Update commit log with text and suggestions from (Dave Hansen) - Drop fixup_dirty_pte() in favor of clearing the HW dirty bit along with the _PAGE_CHG_MASK masking, then calling pte_mkdirty() (Dave Hansen) arch/x86/include/asm/pgtable.h | 64 +++++++++++++++++++++++++++++----- 1 file changed, 56 insertions(+), 8 deletions(-)
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index 6d2f612c04b5..7942eff2af50 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h@@ -392,9 +392,19 @@ static inline pte_t pte_mkexec(pte_t pte) return pte_clear_flags(pte, _PAGE_NX); } +static inline pte_t __pte_mkdirty(pte_t pte, bool soft) +{ + pteval_t dirty = _PAGE_DIRTY; + + if (soft) + dirty |= _PAGE_SOFT_DIRTY; + + return pte_set_flags(pte, dirty); +} + static inline pte_t pte_mkdirty(pte_t pte) { - return pte_set_flags(pte, _PAGE_DIRTY | _PAGE_SOFT_DIRTY); + return __pte_mkdirty(pte, true); } static inline pte_t pte_mkyoung(pte_t pte)
@@ -503,9 +513,19 @@ static inline pmd_t pmd_wrprotect(pmd_t pmd) return pmd_clear_flags(pmd, _PAGE_RW); } +static inline pmd_t __pmd_mkdirty(pmd_t pmd, bool soft) +{ + pmdval_t dirty = _PAGE_DIRTY; + + if (soft) + dirty |= _PAGE_SOFT_DIRTY; + + return pmd_set_flags(pmd, dirty); +} + static inline pmd_t pmd_mkdirty(pmd_t pmd) { - return pmd_set_flags(pmd, _PAGE_DIRTY | _PAGE_SOFT_DIRTY); + return __pmd_mkdirty(pmd, true); } static inline pmd_t pmd_mkdevmap(pmd_t pmd)
@@ -715,26 +735,54 @@ static inline u64 flip_protnone_guard(u64 oldval, u64 val, u64 mask); static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { + pteval_t _page_chg_mask_no_dirty = _PAGE_CHG_MASK & ~_PAGE_DIRTY; pteval_t val = pte_val(pte), oldval = val; + pte_t pte_result; /* * Chop off the NX bit (if present), and add the NX portion of * the newprot (if present): */ - val &= _PAGE_CHG_MASK; - val |= check_pgprot(newprot) & ~_PAGE_CHG_MASK; + val &= _page_chg_mask_no_dirty; + val |= check_pgprot(newprot) & ~_page_chg_mask_no_dirty; val = flip_protnone_guard(oldval, val, PTE_PFN_MASK); - return __pte(val); + + pte_result = __pte(val); + + /* + * Dirty bit is not preserved above so it can be done + * in a special way for the shadow stack case, where it + * may need to set _PAGE_COW. __pte_mkdirty() will do this in + * the case of shadow stack. + */ + if (pte_dirty(pte)) + pte_result = __pte_mkdirty(pte_result, false); + + return pte_result; } static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) { + pteval_t _hpage_chg_mask_no_dirty = _HPAGE_CHG_MASK & ~_PAGE_DIRTY; pmdval_t val = pmd_val(pmd), oldval = val; + pmd_t pmd_result; - val &= _HPAGE_CHG_MASK; - val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK; + val &= _hpage_chg_mask_no_dirty; + val |= check_pgprot(newprot) & ~_hpage_chg_mask_no_dirty; val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK); - return __pmd(val); + + pmd_result = __pmd(val); + + /* + * Dirty bit is not preserved above so it can be done + * in a special way for the shadow stack case, where it + * may need to set _PAGE_COW. __pmd_mkdirty() will do this in + * the case of shadow stack. + */ + if (pmd_dirty(pmd)) + pmd_result = __pmd_mkdirty(pmd_result, false); + + return pmd_result; } /*
--
2.17.1