--- v1
+++ v8
@@ -1,149 +1,71 @@
-From: Yu-cheng Yu <yu-cheng.yu@intel.com>
+The x86 Control-flow Enforcement Technology (CET) feature includes a
+new type of memory called shadow stack. This shadow stack memory has
+some unusual properties, which requires some core mm changes to
+function properly.
-There was no more caller passing vm_flags to do_mmap(), and vm_flags was
-removed from the function's input by:
+In userspace, shadow stack memory is writable only in very specific,
+controlled ways. However, since userspace can, even in the limited
+ways, modify shadow stack contents, the kernel treats it as writable
+memory. As a result, without additional work there would remain many
+ways for userspace to trigger the kernel to write arbitrary data to
+shadow stacks via get_user_pages(, FOLL_WRITE) based operations. To
+help userspace protect their shadow stacks, make this a little less
+exposed by blocking writable get_user_pages() operations for shadow
+stack VMAs.
- commit 45e55300f114 ("mm: remove unnecessary wrapper function do_mmap_pgoff()").
+Still allow FOLL_FORCE to write through shadow stack protections, as it
+does for read-only protections. This is required for debugging use
+cases.
-There is a new user now. Shadow stack allocation passes VM_SHADOW_STACK to
-do_mmap(). Thus, re-introduce vm_flags to do_mmap().
+Signed-off-by: Rick Edgecombe <rick.p.edgecombe@intel.com>
+Reviewed-by: Kees Cook <keescook@chromium.org>
+Acked-by: Mike Rapoport (IBM) <rppt@kernel.org>
+Acked-by: David Hildenbrand <david@redhat.com>
+Tested-by: Pengfei Xu <pengfei.xu@intel.com>
+Tested-by: John Allen <john.allen@amd.com>
+Tested-by: Kees Cook <keescook@chromium.org>
+---
+v8:
+ - Update commit log verbiage (Boris, AndyL)
-Signed-off-by: Yu-cheng Yu <yu-cheng.yu@intel.com>
-Reviewed-by: Peter Collingbourne <pcc@google.com>
-Reviewed-by: Kees Cook <keescook@chromium.org>
-Reviewed-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
-Signed-off-by: Rick Edgecombe <rick.p.edgecombe@intel.com>
-Cc: Andrew Morton <akpm@linux-foundation.org>
-Cc: Oleg Nesterov <oleg@redhat.com>
-Cc: linux-mm@kvack.org
+v3:
+ - Add comment in __pte_access_permitted() (Dave)
+ - Remove unneeded shadow stack specific check in
+ __pte_access_permitted() (Jann)
---
- fs/aio.c | 2 +-
- include/linux/mm.h | 3 ++-
- ipc/shm.c | 2 +-
- mm/mmap.c | 10 +++++-----
- mm/nommu.c | 4 ++--
- mm/util.c | 2 +-
- 6 files changed, 12 insertions(+), 11 deletions(-)
+ arch/x86/include/asm/pgtable.h | 5 +++++
+ mm/gup.c | 2 +-
+ 2 files changed, 6 insertions(+), 1 deletion(-)
-diff --git a/fs/aio.c b/fs/aio.c
-index 4ceba13a7db0..a24618e0e3fc 100644
---- a/fs/aio.c
-+++ b/fs/aio.c
-@@ -554,7 +554,7 @@ static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events)
+diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
+index d81e7ec27507..2e3d8cca1195 100644
+--- a/arch/x86/include/asm/pgtable.h
++++ b/arch/x86/include/asm/pgtable.h
+@@ -1638,6 +1638,11 @@ static inline bool __pte_access_permitted(unsigned long pteval, bool write)
+ {
+ unsigned long need_pte_bits = _PAGE_PRESENT|_PAGE_USER;
- ctx->mmap_base = do_mmap(ctx->aio_ring_file, 0, ctx->mmap_size,
- PROT_READ | PROT_WRITE,
-- MAP_SHARED, 0, &unused, NULL);
-+ MAP_SHARED, 0, 0, &unused, NULL);
- mmap_write_unlock(mm);
- if (IS_ERR((void *)ctx->mmap_base)) {
- ctx->mmap_size = 0;
-diff --git a/include/linux/mm.h b/include/linux/mm.h
-index e125358d7f75..481e1271409f 100644
---- a/include/linux/mm.h
-+++ b/include/linux/mm.h
-@@ -2689,7 +2689,8 @@ extern unsigned long mmap_region(struct file *file, unsigned long addr,
- struct list_head *uf);
- extern unsigned long do_mmap(struct file *file, unsigned long addr,
- unsigned long len, unsigned long prot, unsigned long flags,
-- unsigned long pgoff, unsigned long *populate, struct list_head *uf);
-+ vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate,
-+ struct list_head *uf);
- extern int __do_munmap(struct mm_struct *, unsigned long, size_t,
- struct list_head *uf, bool downgrade);
- extern int do_munmap(struct mm_struct *, unsigned long, size_t,
-diff --git a/ipc/shm.c b/ipc/shm.c
-index b3048ebd5c31..f236b3e14ec4 100644
---- a/ipc/shm.c
-+++ b/ipc/shm.c
-@@ -1646,7 +1646,7 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg,
- goto invalid;
- }
++ /*
++ * Write=0,Dirty=1 PTEs are shadow stack, which the kernel
++ * shouldn't generally allow access to, but since they
++ * are already Write=0, the below logic covers both cases.
++ */
+ if (write)
+ need_pte_bits |= _PAGE_RW;
-- addr = do_mmap(file, addr, size, prot, flags, 0, &populate, NULL);
-+ addr = do_mmap(file, addr, size, prot, flags, 0, 0, &populate, NULL);
- *raddr = addr;
- err = 0;
- if (IS_ERR_VALUE(addr))
-diff --git a/mm/mmap.c b/mm/mmap.c
-index 9bab326332af..9c82a1b02cfc 100644
---- a/mm/mmap.c
-+++ b/mm/mmap.c
-@@ -1410,11 +1410,11 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode,
- */
- unsigned long do_mmap(struct file *file, unsigned long addr,
- unsigned long len, unsigned long prot,
-- unsigned long flags, unsigned long pgoff,
-- unsigned long *populate, struct list_head *uf)
-+ unsigned long flags, vm_flags_t vm_flags,
-+ unsigned long pgoff, unsigned long *populate,
-+ struct list_head *uf)
- {
- struct mm_struct *mm = current->mm;
-- vm_flags_t vm_flags;
- int pkey = 0;
+diff --git a/mm/gup.c b/mm/gup.c
+index eab18ba045db..e7c7bcc0e268 100644
+--- a/mm/gup.c
++++ b/mm/gup.c
+@@ -978,7 +978,7 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags)
+ return -EFAULT;
- *populate = 0;
-@@ -1474,7 +1474,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr,
- * to. we assume access permissions have been handled by the open
- * of the memory object, so we don't do any here.
- */
-- vm_flags = calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(flags) |
-+ vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(flags) |
- mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
-
- if (flags & MAP_LOCKED)
-@@ -3011,7 +3011,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size,
-
- file = get_file(vma->vm_file);
- ret = do_mmap(vma->vm_file, start, size,
-- prot, flags, pgoff, &populate, NULL);
-+ prot, flags, 0, pgoff, &populate, NULL);
- fput(file);
- out:
- mmap_write_unlock(mm);
-diff --git a/mm/nommu.c b/mm/nommu.c
-index 55a9e48a7a02..a6e0243cd69b 100644
---- a/mm/nommu.c
-+++ b/mm/nommu.c
-@@ -1057,6 +1057,7 @@ unsigned long do_mmap(struct file *file,
- unsigned long len,
- unsigned long prot,
- unsigned long flags,
-+ vm_flags_t vm_flags,
- unsigned long pgoff,
- unsigned long *populate,
- struct list_head *uf)
-@@ -1064,7 +1065,6 @@ unsigned long do_mmap(struct file *file,
- struct vm_area_struct *vma;
- struct vm_region *region;
- struct rb_node *rb;
-- vm_flags_t vm_flags;
- unsigned long capabilities, result;
- int ret;
-
-@@ -1083,7 +1083,7 @@ unsigned long do_mmap(struct file *file,
-
- /* we've determined that we can make the mapping, now translate what we
- * now know into VMA flags */
-- vm_flags = determine_vm_flags(file, prot, flags, capabilities);
-+ vm_flags |= determine_vm_flags(file, prot, flags, capabilities);
-
- /* we're going to need to record the mapping */
- region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL);
-diff --git a/mm/util.c b/mm/util.c
-index 7e43369064c8..d419821364cc 100644
---- a/mm/util.c
-+++ b/mm/util.c
-@@ -516,7 +516,7 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr,
- if (!ret) {
- if (mmap_write_lock_killable(mm))
- return -EINTR;
-- ret = do_mmap(file, addr, len, prot, flag, pgoff, &populate,
-+ ret = do_mmap(file, addr, len, prot, flag, 0, pgoff, &populate,
- &uf);
- mmap_write_unlock(mm);
- userfaultfd_unmap_complete(mm, &uf);
+ if (write) {
+- if (!(vm_flags & VM_WRITE)) {
++ if (!(vm_flags & VM_WRITE) || (vm_flags & VM_SHADOW_STACK)) {
+ if (!(gup_flags & FOLL_FORCE))
+ return -EFAULT;
+ /* hugetlb does not support FOLL_FORCE|FOLL_WRITE. */
--
2.17.1