XSAVES saves both system and user states. The Linux kernel
currently does not save/restore any system states. This patch
creates the framework for supporting system states.
Signed-off-by: Yu-cheng Yu <redacted>
---
arch/x86/include/asm/fpu/internal.h | 3 +-
arch/x86/include/asm/fpu/xstate.h | 9 ++-
arch/x86/kernel/fpu/core.c | 7 +-
arch/x86/kernel/fpu/init.c | 10 ---
arch/x86/kernel/fpu/xstate.c | 112 +++++++++++++++++-----------
5 files changed, 80 insertions(+), 61 deletions(-)
@@ -19,10 +19,10 @@#define XSAVE_YMM_SIZE 256#define XSAVE_YMM_OFFSET (XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET)-/* System features */-#define XFEATURE_MASK_SYSTEM (XFEATURE_MASK_PT)--/* All currently supported features */+/*+*SUPPORTED_XFEATURES_MASKindicatesallfeatures+*implementedinandsupportedbythekernel.+*/#define SUPPORTED_XFEATURES_MASK (XFEATURE_MASK_FP | \XFEATURE_MASK_SSE|\XFEATURE_MASK_YMM|\
@@ -219,30 +222,31 @@ void fpstate_sanitize_xstate(struct fpu *fpu)*/voidfpu__init_cpu_xstate(void){-if(!boot_cpu_has(X86_FEATURE_XSAVE)||!xfeatures_mask_user)+if(!boot_cpu_has(X86_FEATURE_XSAVE)||!xfeatures_mask_all)return;++cr4_set_bits(X86_CR4_OSXSAVE);+/*-*MakeitclearthatXSAVESsystemstatesarenotyet-*implementedshouldanyoneexpectittoworkbychanging-*bitsinXFEATURE_MASK_*macrosandXCR0.+*XCR_XFEATURE_ENABLED_MASKsetsthefeaturesthataremanaged+*byXSAVE{C,OPT}andXRSTOR.OnlyXSAVEuserstatescanbe+*sethere.*/-WARN_ONCE((xfeatures_mask_user&XFEATURE_MASK_SYSTEM),-"x86/fpu: XSAVES system states are not yet implemented.\n");+xsetbv(XCR_XFEATURE_ENABLED_MASK,+xfeatures_mask_user);-xfeatures_mask_user&=~XFEATURE_MASK_SYSTEM;--cr4_set_bits(X86_CR4_OSXSAVE);-xsetbv(XCR_XFEATURE_ENABLED_MASK,xfeatures_mask_user);+/*+*MSR_IA32_XSSsetswhichXSAVESsystemstatestobemanagedby+*XSAVES.OnlyXSAVESsystemstatescanbesethere.+*/+if(boot_cpu_has(X86_FEATURE_XSAVES))+wrmsrl(MSR_IA32_XSS,+xfeatures_mask_all&~xfeatures_mask_user);}-/*-*Notethatinthefuturewewilllikelyneedapairof-*functionshere:oneforuserxstatesandtheotherfor-*systemxstates.Fornow,theyarethesame.-*/staticintxfeature_enabled(enumxfeaturexfeature){-return!!(xfeatures_mask_user&BIT_ULL(xfeature));+return!!(xfeatures_mask_all&BIT_ULL(xfeature));}/*
@@ -348,7 +352,7 @@ static int xfeature_is_aligned(int xfeature_nr)*/staticvoid__initsetup_xstate_comp(void){-unsignedintxstate_comp_sizes[sizeof(xfeatures_mask_user)*8];+unsignedintxstate_comp_sizes[sizeof(xfeatures_mask_all)*8];inti;/*
@@ -441,11 +445,10 @@ static int xfeature_uncompacted_offset(int xfeature_nr)u32eax,ebx,ecx,edx;/*-*OnlyXSAVESsupportssystemstatesanditusescompacted-*format.Checkingasystemstate'suncompactedoffsetis-*anerror.+*Checkingasystemorunsupportedstate'suncompactedoffset+*isanerror.*/-if(XFEATURE_MASK_SYSTEM&(1<<xfeature_nr)){+if(~xfeatures_mask_user&BIT_ULL(xfeature_nr)){WARN_ONCE(1,"No fixed offset for xstate %d\n",xfeature_nr);return-1;}
@@ -482,7 +485,7 @@ int using_compacted_format(void)intvalidate_xstate_header(conststructxstate_header*hdr){/* No unknown or system features may be set */-if(hdr->xfeatures&(~xfeatures_mask_user|XFEATURE_MASK_SYSTEM))+if(hdr->xfeatures&~xfeatures_mask_user)return-EINVAL;/* Userspace must use the uncompacted format */
@@ -760,10 +777,11 @@ void __init fpu__init_system_xstate(void)*/for(i=0;i<ARRAY_SIZE(xsave_cpuid_features);i++){if(!boot_cpu_has(xsave_cpuid_features[i]))-xfeatures_mask_user&=~BIT_ULL(i);+xfeatures_mask_all&=~BIT_ULL(i);}-xfeatures_mask_user&=fpu__get_supported_xfeatures_mask();+xfeatures_mask_all&=SUPPORTED_XFEATURES_MASK;+xfeatures_mask_user=xfeatures_mask_all&cpu_user_xfeatures_mask;/* Enable xstate instructions to be able to continue with initialization: */fpu__init_cpu_xstate();
A RO and dirty PTE exists in the following cases:
(a) A page is modified and then shared with a fork()'ed child;
(b) A R/O page that has been COW'ed;
(c) A SHSTK page.
The processor does not read the dirty bit for (a) and (b), but
checks the dirty bit for (c). To prevent the use of non-SHSTK
memory as SHSTK, we introduce a spare bit of the 64-bit PTE as
_PAGE_BIT_DIRTY_SW and use that for (a) and (b). This results
to the following possible PTE settings:
Modified PTE: (R/W + DIRTY_HW)
Modified and shared PTE: (R/O + DIRTY_SW)
R/O PTE COW'ed: (R/O + DIRTY_SW)
SHSTK PTE: (R/O + DIRTY_HW)
SHSTK PTE COW'ed: (R/O + DIRTY_HW)
SHSTK PTE shared: (R/O + DIRTY_SW)
Note that _PAGE_BIT_DRITY_SW is only used in R/O PTEs but
not R/W PTEs.
When this patch is applied, there are six free bits left in
the 64-bit PTE. There is no more free bit in the 32-bit
PTE (except for PAE) and shadow stack is not implemented
for the 32-bit kernel.
Signed-off-by: Yu-cheng Yu <redacted>
---
arch/x86/include/asm/pgtable.h | 129 ++++++++++++++++++++++-----
arch/x86/include/asm/pgtable_types.h | 14 ++-
include/asm-generic/pgtable.h | 21 +++++
3 files changed, 142 insertions(+), 22 deletions(-)
@@ -23,6 +23,7 @@#define _PAGE_BIT_SOFTW2 10 /* " */#define _PAGE_BIT_SOFTW3 11 /* " */#define _PAGE_BIT_PAT_LARGE 12 /* On 2MB or 1GB pages */+#define _PAGE_BIT_SOFTW5 57 /* available for programmer */#define _PAGE_BIT_SOFTW4 58 /* available for programmer */#define _PAGE_BIT_PKEY_BIT0 59 /* Protection Keys, bit 1/4 */#define _PAGE_BIT_PKEY_BIT1 60 /* Protection Keys, bit 2/4 */
@@ -34,6 +35,7 @@#define _PAGE_BIT_CPA_TEST _PAGE_BIT_SOFTW1#define _PAGE_BIT_SOFT_DIRTY _PAGE_BIT_SOFTW3 /* software dirty tracking */#define _PAGE_BIT_DEVMAP _PAGE_BIT_SOFTW4+#define _PAGE_BIT_DIRTY_SW _PAGE_BIT_SOFTW5 /* was written to *//* If _PAGE_BIT_PRESENT is clear, we use these: *//* - if the user mapped it with PROT_NONE; pte_present gives true */
We are going to create _PAGE_DIRTY_SW for non-hardware, memory
management purposes. Rename _PAGE_DIRTY to _PAGE_DIRTY_HW and
_PAGE_BIT_DIRTY to _PAGE_BIT_DIRTY_HW to make these PTE dirty
bits more clear. There are no functional changes in this
patch.
Signed-off-by: Yu-cheng Yu <redacted>
---
arch/x86/include/asm/pgtable.h | 6 +++---
arch/x86/include/asm/pgtable_types.h | 17 +++++++++--------
arch/x86/kernel/relocate_kernel_64.S | 2 +-
arch/x86/kvm/vmx.c | 2 +-
4 files changed, 14 insertions(+), 13 deletions(-)
@@ -5848,7 +5848,7 @@ static int init_rmode_identity_map(struct kvm *kvm)/* Set up identity-mapping pagetable for EPT in real mode */for(i=0;i<PT32_ENT_PER_PAGE;i++){tmp=(i<<22)+(_PAGE_PRESENT|_PAGE_RW|_PAGE_USER|-_PAGE_ACCESSED|_PAGE_DIRTY|_PAGE_PSE);+_PAGE_ACCESSED|_PAGE_DIRTY_HW|_PAGE_PSE);r=kvm_write_guest_page(kvm,identity_map_pfn,&tmp,i*sizeof(tmp),sizeof(tmp));if(r<0)
To support XSAVES system states, change some names to distinguish
user and system states.
Change:
supervisor to system
copy_init_fpstate_to_fpregs() to copy_init_user_fpstate_to_fpregs()
xfeatures_mask to xfeatures_mask_user
XCNTXT_MASK to SUPPORTED_XFEATURES_MASK (states supported)
Signed-off-by: Yu-cheng Yu <redacted>
---
arch/x86/include/asm/fpu/internal.h | 5 +-
arch/x86/include/asm/fpu/xstate.h | 24 ++++----
arch/x86/kernel/fpu/core.c | 4 +-
arch/x86/kernel/fpu/init.c | 2 +-
arch/x86/kernel/fpu/signal.c | 6 +-
arch/x86/kernel/fpu/xstate.c | 88 +++++++++++++++--------------
6 files changed, 66 insertions(+), 63 deletions(-)
@@ -219,20 +219,20 @@ void fpstate_sanitize_xstate(struct fpu *fpu)*/voidfpu__init_cpu_xstate(void){-if(!boot_cpu_has(X86_FEATURE_XSAVE)||!xfeatures_mask)+if(!boot_cpu_has(X86_FEATURE_XSAVE)||!xfeatures_mask_user)return;/*-*MakeitclearthatXSAVESsupervisorstatesarenotyet+*MakeitclearthatXSAVESsystemstatesarenotyet*implementedshouldanyoneexpectittoworkbychanging*bitsinXFEATURE_MASK_*macrosandXCR0.*/-WARN_ONCE((xfeatures_mask&XFEATURE_MASK_SUPERVISOR),-"x86/fpu: XSAVES supervisor states are not yet implemented.\n");+WARN_ONCE((xfeatures_mask_user&XFEATURE_MASK_SYSTEM),+"x86/fpu: XSAVES system states are not yet implemented.\n");-xfeatures_mask&=~XFEATURE_MASK_SUPERVISOR;+xfeatures_mask_user&=~XFEATURE_MASK_SYSTEM;cr4_set_bits(X86_CR4_OSXSAVE);-xsetbv(XCR_XFEATURE_ENABLED_MASK,xfeatures_mask);+xsetbv(XCR_XFEATURE_ENABLED_MASK,xfeatures_mask_user);}/*
@@ -440,11 +441,11 @@ static int xfeature_uncompacted_offset(int xfeature_nr)u32eax,ebx,ecx,edx;/*-*OnlyXSAVESsupportssupervisorstatesanditusescompacted-*format.Checkingasupervisorstate'suncompactedoffsetis+*OnlyXSAVESsupportssystemstatesanditusescompacted+*format.Checkingasystemstate'suncompactedoffsetis*anerror.*/-if(XFEATURE_MASK_SUPERVISOR&(1<<xfeature_nr)){+if(XFEATURE_MASK_SYSTEM&(1<<xfeature_nr)){WARN_ONCE(1,"No fixed offset for xstate %d\n",xfeature_nr);return-1;}
@@ -465,7 +466,7 @@ static int xfeature_size(int xfeature_nr)/**'XSAVES'impliestwodifferentthings:-*1.savingofsupervisor/systemstate+*1.savingofsystemstate*2.usingthecompactedformat**Usethisfunctionwhendealingwiththecompactedformatso
@@ -480,8 +481,8 @@ int using_compacted_format(void)/* Validate an xstate header supplied by userspace (ptrace or sigreturn) */intvalidate_xstate_header(conststructxstate_header*hdr){-/* No unknown or supervisor features may be set */-if(hdr->xfeatures&(~xfeatures_mask|XFEATURE_MASK_SUPERVISOR))+/* No unknown or system features may be set */+if(hdr->xfeatures&(~xfeatures_mask_user|XFEATURE_MASK_SYSTEM))return-EINVAL;/* Userspace must use the uncompacted format */
@@ -588,11 +589,11 @@ static void do_extra_xstate_size_checks(void)check_xstate_against_struct(i);/*-*Supervisorstatecomponentscanbemanagedonlyby+*Systemstatecomponentscanbemanagedonlyby*XSAVES,whichiscompacted-formatonly.*/if(!using_compacted_format())-XSTATE_WARN_ON(xfeature_is_supervisor(i));+XSTATE_WARN_ON(xfeature_is_system(i));/* Align from the end of the previous feature */if(xfeature_is_aligned(i))
@@ -706,7 +707,7 @@ static int init_xstate_size(void)*/staticvoidfpu__init_disable_system_xstate(void){-xfeatures_mask=0;+xfeatures_mask_user=0;cr4_clear_bits(X86_CR4_OSXSAVE);fpu__xstate_clear_all_cpu_caps();}
@@ -742,15 +743,15 @@ void __init fpu__init_system_xstate(void)}cpuid_count(XSTATE_CPUID,0,&eax,&ebx,&ecx,&edx);-xfeatures_mask=eax+((u64)edx<<32);+xfeatures_mask_user=eax+((u64)edx<<32);-if((xfeatures_mask&XFEATURE_MASK_FPSSE)!=XFEATURE_MASK_FPSSE){+if((xfeatures_mask_user&XFEATURE_MASK_FPSSE)!=XFEATURE_MASK_FPSSE){/**Thisindicatesthatsomethingreallyunexpectedhappened*withtheenumeration.DisableXSAVEandtrytocontinue*bootingwithoutit.ThisistooearlytoBUG().*/-pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n",xfeatures_mask);+pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n",xfeatures_mask_user);gotoout_disable;}
@@ -759,10 +760,10 @@ void __init fpu__init_system_xstate(void)*/for(i=0;i<ARRAY_SIZE(xsave_cpuid_features);i++){if(!boot_cpu_has(xsave_cpuid_features[i]))-xfeatures_mask&=~BIT(i);+xfeatures_mask_user&=~BIT_ULL(i);}-xfeatures_mask&=fpu__get_supported_xfeatures_mask();+xfeatures_mask_user&=fpu__get_supported_xfeatures_mask();/* Enable xstate instructions to be able to continue with initialization: */fpu__init_cpu_xstate();
arch_prctl(ARCH_CET_STATUS, unsigned long *addr)
Return CET feature status.
The parameter 'addr' is a pointer to a user buffer.
On returning to the caller, the kernel fills the following
information:
*addr = SHSTK/IBT status
*(addr + 1) = SHSTK base address
*(addr + 2) = SHSTK size
arch_prctl(ARCH_CET_DISABLE, unsigned long features)
Disable CET features specified in 'features'. Return
-EPERM if CET is locked.
arch_prctl(ARCH_CET_LOCK)
Lock in CET feature.
arch_prctl(ARCH_CET_ALLOC_SHSTK, unsigned long *addr)
Allocate a new SHSTK.
The parameter 'addr' is a pointer to a user buffer and indicates
the desired SHSTK size to allocate. On returning to the caller
the buffer contains the address of the new SHSTK.
Signed-off-by: H.J. Lu <redacted>
Signed-off-by: Yu-cheng Yu <redacted>
---
arch/x86/include/asm/cet.h | 5 ++
arch/x86/include/uapi/asm/prctl.h | 5 ++
arch/x86/kernel/Makefile | 2 +-
arch/x86/kernel/cet.c | 27 +++++++++++
arch/x86/kernel/cet_prctl.c | 79 +++++++++++++++++++++++++++++++
arch/x86/kernel/process.c | 5 ++
6 files changed, 122 insertions(+), 1 deletion(-)
create mode 100644 arch/x86/kernel/cet_prctl.c
@@ -110,6 +110,33 @@ static int create_rstor_token(bool ia32, unsigned long ssp,return0;}+intcet_alloc_shstk(unsignedlong*arg)+{+unsignedlonglen=*arg;+unsignedlongaddr;+unsignedlongtoken;+unsignedlongssp;++addr=do_mmap_locked(0,len,PROT_READ,+MAP_ANONYMOUS|MAP_PRIVATE,VM_SHSTK);+if(addr>=TASK_SIZE_MAX)+return-ENOMEM;++/* Restore token is 8 bytes and aligned to 8 bytes */+ssp=addr+len;+token=ssp;++if(!in_ia32_syscall())+token|=1;+ssp-=8;++if(write_user_shstk_64(ssp,token))+return-EINVAL;++*arg=addr;+return0;+}+intcet_setup_shstk(void){unsignedlongaddr,size;
@@ -792,6 +792,11 @@ long do_arch_prctl_common(struct task_struct *task, int option,returnget_cpuid_mode();caseARCH_SET_CPUID:returnset_cpuid_mode(task,cpuid_enabled);+caseARCH_CET_STATUS:+caseARCH_CET_DISABLE:+caseARCH_CET_LOCK:+caseARCH_CET_ALLOC_SHSTK:+returnprctl_cet(option,cpuid_enabled);}return-EINVAL;
WRUSS is a new kernel-mode instruction but writes directly
to user shadow stack memory. This is used to construct
a return address on the shadow stack for the signal
handler.
This instruction can fault if the user shadow stack is
invalid shadow stack memory. In that case, the kernel does
fixup.
Signed-off-by: Yu-cheng Yu <redacted>
---
arch/x86/include/asm/special_insns.h | 37 ++++++++++++++++++++++++++++
arch/x86/mm/extable.c | 11 +++++++++
arch/x86/mm/fault.c | 9 +++++++
3 files changed, 57 insertions(+)
When a task does fork(), its shadow stack must be duplicated for
the child. However, the child may not actually use all pages of
of the copied shadow stack. This patch implements a flow that
is similar to copy-on-write of an anonymous page, but for shadow
stack memory. A shadow stack PTE needs to be RO and dirty. We
use this dirty bit requirement to effect the copying of shadow
stack pages.
In copy_one_pte(), we clear the dirty bit from the shadow stack
PTE. On the next shadow stack access to the PTE, a page fault
occurs. At that time, we then copy/re-use the page and fix the
PTE.
Signed-off-by: Yu-cheng Yu <redacted>
---
arch/x86/mm/pgtable.c | 10 ++++++++++
include/asm-generic/pgtable.h | 7 +++++++
mm/memory.c | 3 +++
3 files changed, 20 insertions(+)
When setting up a signal, the kernel creates a shadow stack
restore token at the current SHSTK address and then stores the
token's address in the signal frame, right after the FPU state.
Before restoring a signal, the kernel verifies and then uses the
restore token to set the SHSTK pointer.
Signed-off-by: Yu-cheng Yu <redacted>
---
arch/x86/ia32/ia32_signal.c | 13 +++
arch/x86/include/asm/cet.h | 5 ++
arch/x86/include/asm/sighandling.h | 5 ++
arch/x86/include/uapi/asm/sigcontext.h | 17 ++++
arch/x86/kernel/cet.c | 115 +++++++++++++++++++++++++
arch/x86/kernel/signal.c | 96 +++++++++++++++++++++
6 files changed, 251 insertions(+)
@@ -196,6 +196,23 @@ struct _xstate {/* New processor state extensions go here: */};+#ifdef __x86_64__+/*+*Sigcontextextension(structsc_ext)islocatedafter+*sigcontext->fpstate.Becausecurrentlyonlytheshadow+*stackpointerissavedthereandtheshadowstackdepends+*onXSAVES,wecanfindsc_extfromsigcontext->fpstate.+*+*The64-bitfpstatehasasizeoffpu_user_xstate_size,plus+*FP_XSTATE_MAGIC2_SIZEwhenXSAVE*isused.Thestructsc_ext+*islocatedattheendofsigcontext->fpstate,alignedto8.+*/+structsc_ext{+unsignedlongtotal_size;+unsignedlongssp;+};+#endif+/**The32-bitsignalframe:*/
@@ -46,6 +47,69 @@ static unsigned long get_shstk_addr(void)returnptr;}+/*+*Verifytherestoretokenattheaddressof'ssp'is+*validandthensetshadowstackpointeraccordingtothe+*token.+*/+staticintverify_rstor_token(boolia32,unsignedlongssp,+unsignedlong*new_ssp)+{+unsignedlongtoken;++*new_ssp=0;++if(!IS_ALIGNED(ssp,8))+return-EINVAL;++if(get_user(token,(unsignedlong__user*)ssp))+return-EFAULT;++/* Is 64-bit mode flag correct? */+if(ia32&&(token&3)!=0)+return-EINVAL;+elseif((token&3)!=1)+return-EINVAL;++token&=~(1UL);++if((!ia32&&!IS_ALIGNED(token,8))||!IS_ALIGNED(token,4))+return-EINVAL;++if((ALIGN_DOWN(token,8)-8)!=ssp)+return-EINVAL;++*new_ssp=token;+return0;+}++/*+*Createarestoretokenontheshadowstack.+*Atokenisalways8-byteandalignedto8.+*/+staticintcreate_rstor_token(boolia32,unsignedlongssp,+unsignedlong*new_ssp)+{+unsignedlongaddr;++*new_ssp=0;++if((!ia32&&!IS_ALIGNED(ssp,8))||!IS_ALIGNED(ssp,4))+return-EINVAL;++addr=ALIGN_DOWN(ssp,8)-8;++/* Is the token for 64-bit? */+if(!ia32)+ssp|=1;++if(write_user_shstk_64(addr,ssp))+return-EFAULT;++*new_ssp=addr;+return0;+}+intcet_setup_shstk(void){unsignedlongaddr,size;
This patch implements THP shadow stack memory copying in the same
way as the previous patch for regular PTE.
In copy_huge_pmd(), we clear the dirty bit from the PMD. On the
next shadow stack access to the PMD, a page fault occurs. At
that time, the page is copied/re-used and the PMD is fixed.
Signed-off-by: Yu-cheng Yu <redacted>
---
arch/x86/mm/pgtable.c | 8 ++++++++
include/asm-generic/pgtable.h | 6 ++++++
mm/huge_memory.c | 4 ++++
3 files changed, 18 insertions(+)
Introduce Kconfig option X86_INTEL_SHADOW_STACK_USER.
An application has shadow stack protection when all the following are
true:
(1) The kernel has X86_INTEL_SHADOW_STACK_USER enabled,
(2) The running processor supports the shadow stack,
(3) The application is built with shadow stack enabled tools & libs
and, and at runtime, all dependent shared libs can support shadow
stack.
If this kernel config option is enabled, but (2) or (3) above is not
true, the application runs without the shadow stack protection.
Existing legacy applications will continue to work without the shadow
stack protection.
The user-mode shadow stack protection is only implemented for the
64-bit kernel. Thirty-two bit applications are supported under the
compatibility mode.
Signed-off-by: Yu-cheng Yu <redacted>
---
arch/x86/Kconfig | 24 ++++++++++++++++++++++++
arch/x86/Makefile | 7 +++++++
2 files changed, 31 insertions(+)
@@ -152,6 +152,13 @@ ifdef CONFIG_X86_X32endifexportCONFIG_X86_X32_ABI+# Check assembler shadow stack suppot+ifdef CONFIG_X86_INTEL_SHADOW_STACK_USER+ ifeq ($(call as-instr, saveprevssp, y),)+$(errorCONFIG_X86_INTEL_SHADOW_STACK_USERnotsupportedbytheassembler)+ endif+endif+## If the function graph tracer is used with mcount instead of fentry,# '-maccumulate-outgoing-args' is needed to prevent a GCC bug
There are a few places that need do_mmap() with mm->mmap_sem held.
Create an in-line function for that.
Signed-off-by: Yu-cheng Yu <redacted>
---
include/linux/mm.h | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
@@ -2318,6 +2318,24 @@ static inline void mm_populate(unsigned long addr, unsigned long len)staticinlinevoidmm_populate(unsignedlongaddr,unsignedlonglen){}#endif+staticinlineunsignedlongdo_mmap_locked(unsignedlongaddr,+unsignedlonglen,unsignedlongprot,unsignedlongflags,+vm_flags_tvm_flags)+{+structmm_struct*mm=current->mm;+unsignedlongpopulate;++down_write(&mm->mmap_sem);+addr=do_mmap(NULL,addr,len,prot,flags,vm_flags,0,+&populate,NULL);+up_write(&mm->mmap_sem);++if(populate)+mm_populate(addr,populate);++returnaddr;+}+/* These take the mm semaphore themselves */externint__must_checkvm_brk(unsignedlong,unsignedlong);externint__must_checkvm_brk_flags(unsignedlong,unsignedlong,unsignedlong);
If a page fault is triggered by a shadow stack access (e.g.
call/ret) or shadow stack management instructions (e.g.
wrussq), then bit[6] of the page fault error code is set.
In access_error(), we check if a shadow stack page fault
is within a shadow stack memory area.
Signed-off-by: Yu-cheng Yu <redacted>
---
arch/x86/include/asm/traps.h | 2 ++
arch/x86/mm/fault.c | 18 ++++++++++++++++++
2 files changed, 20 insertions(+)
When Shadow Stack is enabled, the read-only and PAGE_DIRTY_HW PTE
setting is reserved only for the Shadow Stack. To track dirty of
non-Shadow Stack read-only PTEs, we use PAGE_DIRTY_SW.
Update ptep_set_wrprotect() and pmdp_set_wrprotect().
Signed-off-by: Yu-cheng Yu <redacted>
---
arch/x86/include/asm/pgtable.h | 42 ++++++++++++++++++++++++++++++++++
1 file changed, 42 insertions(+)
Update _PAGE_DIRTY to _PAGE_DIRTY_BITS in split_2MB_gtt_entry().
In order to support Control Flow Enforcement (CET), _PAGE_DIRTY
is now _PAGE_DIRTY_HW or _PAGE_DIRTY_SW.
Signed-off-by: Yu-cheng Yu <redacted>
---
drivers/gpu/drm/i915/gvt/gtt.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
A control protection exception is triggered when a control flow transfer
attempt violated shadow stack or indirect branch tracking constraints.
For example, the return address for a RET instruction differs from the
safe copy on the shadow stack; or a JMP instruction arrives at a non-
ENDBR instruction.
The control protection exception handler works in a similar way as the
general protection fault handler.
Signed-off-by: Yu-cheng Yu <redacted>
---
arch/x86/entry/entry_64.S | 2 +-
arch/x86/include/asm/traps.h | 3 ++
arch/x86/kernel/idt.c | 4 +++
arch/x86/kernel/traps.c | 58 ++++++++++++++++++++++++++++++++++++
4 files changed, 66 insertions(+), 1 deletion(-)
@@ -578,6 +578,64 @@ do_general_protection(struct pt_regs *regs, long error_code)}NOKPROBE_SYMBOL(do_general_protection);+staticconstchar*control_protection_err[]=+{+"unknown",+"near-ret",+"far-ret/iret",+"endbranch",+"rstorssp",+"setssbsy",+};++/*+*Whenacontrolprotectionexceptionoccurs,sendasignal+*totheresponsibleapplication.Currently,control+*protectionisonlyenabledfortheusermode.This+*exceptionshouldnotcomefromthekernelmode.+*/+dotraplinkagevoid+do_control_protection(structpt_regs*regs,longerror_code)+{+structtask_struct*tsk;++RCU_LOCKDEP_WARN(!rcu_is_watching(),"entry code didn't wake RCU");+if(notify_die(DIE_TRAP,"control protection fault",regs,+error_code,X86_TRAP_CP,SIGSEGV)==NOTIFY_STOP)+return;+cond_local_irq_enable(regs);++if(!user_mode(regs))+die("kernel control protection fault",regs,error_code);++if(!static_cpu_has(X86_FEATURE_SHSTK)&&+!static_cpu_has(X86_FEATURE_IBT))+WARN_ONCE(1,"CET is disabled but got control "+"protection fault\n");++tsk=current;+tsk->thread.error_code=error_code;+tsk->thread.trap_nr=X86_TRAP_CP;++if(show_unhandled_signals&&unhandled_signal(tsk,SIGSEGV)&&+printk_ratelimit()){+unsignedintmax_err;++max_err=ARRAY_SIZE(control_protection_err)-1;+if((error_code<0)||(error_code>max_err))+error_code=0;+pr_info("%s[%d] control protection ip:%lx sp:%lx error:%lx(%s)",+tsk->comm,task_pid_nr(tsk),+regs->ip,regs->sp,error_code,+control_protection_err[error_code]);+print_vma_addr(" in ",regs->ip);+pr_cont("\n");+}++force_sig_info(SIGSEGV,SEND_SIG_PRIV,tsk);+}+NOKPROBE_SYMBOL(do_control_protection);+dotraplinkagevoidnotracedo_int3(structpt_regs*regs,longerror_code){#ifdef CONFIG_DYNAMIC_FTRACE
@@ -0,0 +1,252 @@+=========================================+Control Flow Enforcement Technology (CET)+=========================================++[1] Overview+============++Control Flow Enforcement Technology (CET) provides protection against+return/jump-oriented programing (ROP) attacks. It can be implemented+to protect both the kernel and applications. In the first phase,+only the user-mode protection is implemented for the 64-bit kernel.+Thirty-two bit applications are supported under the compatibility+mode.++CET includes shadow stack (SHSTK) and indirect branch tracking (IBT)+and they are enabled from two kernel configuration options:++ INTEL_X86_SHADOW_STACK_USER, and+ INTEL_X86_BRANCH_TRACKING_USER.++To build a CET-enabled kernel, Binutils v2.31 and GCC v8.1 or later+are required. To build a CET-enabled application, GLIBC v2.28 or+later is also required.++There are two command-line options for disabling CET features:++ no_cet_shstk - disables SHSTK, and+ no_cet_ibt - disables IBT.++At run time, /proc/cpuinfo shows the availability of SHSTK and IBT.++[2] CET assembly instructions+=============================++RDSSP %r+ Read the SHSTK pointer into %r.++INCSSP %r+ Unwind (increment) the SHSTK pointer (0 ~ 255) steps as indicated+ in the operand register. The GLIBC longjmp uses INCSSP to unwind+ the SHSTK until that matches the program stack. When it is+ necessary to unwind beyond 255 steps, longjmp divides and repeats+ the process.++RSTORSSP (%r)+ Switch to the SHSTK indicated in the 'restore token' pointed by+ the operand register and replace the 'restore token' with a new+ token to be saved (with SAVEPREVSSP) for the outgoing SHSTK.++::++ Before RSTORSSP++ Incoming SHSTK Current/Outgoing SHSTK++ |----------------------| |----------------------|+ addr=x | | ssp-> | |+ |----------------------| |----------------------|+ (%r)-> | rstor_token=(x|Lg) | addr=y-8 | |+ |----------------------| |----------------------|++ After RSTORSSP++ |----------------------| |----------------------|+ ssp-> | | | |+ |----------------------| |----------------------|+| rstor_token=(y|Bz|Lg)| addr=y-8 | |+ |----------------------| |----------------------|++ note:+1. Only valid addresses and restore tokens can be on the+ user-mode SHSTK.+2. A token is always of type u64 and must align to u64.+3. The incoming SHSTK pointer in a rstor_token must point to+ immediately above the token.+4. 'Lg' is bit[0] of a rstor_token indicating a 64-bit SHSTK.+5. 'Bz' is bit[1] of a rstor_token indicating the token is to+ be used only for the next SAVEPREVSSP and invalid for the+ RSTORSSP.++SAVEPREVSSP+ Store the SHSTK 'restore token' pointed by+ (current_SHSTK_pointer + 8).++::++ After SAVEPREVSSP++ |----------------------| |----------------------|+ ssp-> | | | |+ |----------------------| |----------------------|+| rstor_token=(y|Bz|Lg)| addr=y-8 | rstor_token(y|Lg) |+ |----------------------| |----------------------|++WRUSS %r0, (%r1)+ Write the value in %r0 to the SHSTK address pointed by (%r1).+ This is a kernel-mode only instruction.++ENDBR+ The compiler inserts an ENDBR at all valid branch targets. Any+ CALL/JMP to a target without an ENDBR triggers a control+ protection fault.++[3] Application Enabling+========================++An application's CET capability is marked in its ELF header and can+be verified from the following command output, in the+NT_GNU_PROPERTY_TYPE_0 field:++ readelf -n <application>++If an application supports CET and is statically linked, it will run+with CET protection. If the application needs any shared libraries,+the loader checks all dependencies and enables CET only when all+requirements are met.++[4] Legacy Libraries+====================++GLIBC provides a few tunables for backward compatibility.++GLIBC_TUNABLES=glibc.tune.hwcaps=-SHSTK,-IBT+ Turn off SHSTK/IBT for the current shell.++GLIBC_TUNABLES=glibc.tune.x86_shstk=<on, permissive>+ This controls how dlopen() handles SHSTK legacy libraries:+ on: continue with SHSTK enabled;+ permissive: continue with SHSTK off.++[5] CET system calls+====================++The following arch_prctl() system calls are added for CET:++arch_prctl(ARCH_CET_STATUS, unsigned long *addr)+ Return CET feature status.++ The parameter 'addr' is a pointer to a user buffer.+ On returning to the caller, the kernel fills the following+ information:++ *addr = SHSTK/IBT status+ *(addr + 1) = SHSTK base address+ *(addr + 2) = SHSTK size++arch_prctl(ARCH_CET_DISABLE, unsigned long features)+ Disable SHSTK and/or IBT specified in 'features'. Return -EPERM+ if CET is locked.++arch_prctl(ARCH_CET_LOCK)+ Lock in CET feature.++arch_prctl(ARCH_CET_ALLOC_SHSTK, unsigned long *addr)+ Allocate a new SHSTK.++ The parameter 'addr' is a pointer to a user buffer and indicates+ the desired SHSTK size to allocate. On returning to the caller+ the buffer contains the address of the new SHSTK.++arch_prctl(ARCH_CET_LEGACY_BITMAP, unsigned long *addr)+ Allocate an IBT legacy code bitmap if the current task does not+ have one.++ The parameter 'addr' is a pointer to a user buffer.+ On returning to the caller, the kernel fills the following+ information:++ *addr = IBT bitmap base address+ *(addr + 1) = IBT bitmap size++[6] The implementation of the SHSTK+===================================++SHSTK size+----------++A task's SHSTK is allocated from memory to a fixed size of+RLIMIT_STACK.++Signal+------++The main program and its signal handlers use the same SHSTK. Because+the SHSTK stores only return addresses, we can estimate a large+enough SHSTK to cover the condition that both the program stack and+the sigaltstack run out.++The kernel creates a restore token at the SHSTK restoring address and+verifies that token when restoring from the signal handler.++Fork+----++The SHSTK's vma has VM_SHSTK flag set; its PTEs are required to be+read-only and dirty. When a SHSTK PTE is not present, RO, and dirty,+a SHSTK access triggers a page fault with an additional SHSTK bit set+in the page fault error code.++When a task forks a child, its SHSTK PTEs are copied and both the+parent's and the child's SHSTK PTEs are cleared of the dirty bit.+Upon the next SHSTK access, the resulting SHSTK page fault is handled+by page copy/re-use.++When a pthread child is created, the kernel allocates a new SHSTK for+the new thread.++Setjmp/Longjmp+--------------++Longjmp unwinds SHSTK until it matches the program stack.++Ucontext+--------++In GLIBC, getcontext/setcontext is implemented in similar way as+setjmp/longjmp.++When makecontext creates a new ucontext, a new SHSTK is allocated for+that context with ARCH_CET_ALLOC_SHSTK the syscall. The kernel+creates a restore token at the top of the new SHSTK and the user-mode+code switches to the new SHSTK with the RSTORSSP instruction.++[7] The management of read-only & dirty PTEs for SHSTK+======================================================++A RO and dirty PTE exists in the following cases:++(a) A page is modified and then shared with a fork()'ed child;+(b) A R/O page that has been COW'ed;+(c) A SHSTK page.++The processor only checks the dirty bit for (c). To prevent the use+of non-SHSTK memory as SHSTK, we use a spare bit of the 64-bit PTE as+DIRTY_SW for (a) and (b) above. This results to the following PTE+settings:++Modified PTE: (R/W + DIRTY_HW)+Modified and shared PTE: (R/O + DIRTY_SW)+R/O PTE, COW'ed: (R/O + DIRTY_SW)+SHSTK PTE: (R/O + DIRTY_HW)+SHSTK PTE, COW'ed: (R/O + DIRTY_HW)+SHSTK PTE, shared: (R/O + DIRTY_SW)++Note that DIRTY_SW is only used in R/O PTEs but not R/W PTEs.++[8] The implementation of IBT+=============================++The kernel provides IBT support in mmap() of the legacy code bit map.+However, the management of the bitmap is done in the GLIBC or the+application.
Add the following shadow stack management instructions.
INCSSP:
Increment shadow stack pointer by the steps specified.
RDSSP:
Read SSP register into a GPR.
SAVEPREVSSP:
Use "prev ssp" token at top of current shadow stack to
create a "restore token" on previous shadow stack.
RSTORSSP:
Restore from a "restore token" pointed by a GPR to SSP.
WRSS:
Write to kernel-mode shadow stack (kernel-mode instruction).
WRUSS:
Write to user-mode shadow stack (kernel-mode instruction).
SETSSBSY:
Verify the "supervisor token" pointed by IA32_PL0_SSP MSR,
if valid, set the token to busy, and set SSP to the value
of IA32_PL0_SSP MSR.
CLRSSBSY:
Verify the "supervisor token" pointed by a GPR, if valid,
clear the busy bit from the token.
Signed-off-by: Yu-cheng Yu <redacted>
---
arch/x86/lib/x86-opcode-map.txt | 26 +++++++++++++------
tools/objtool/arch/x86/lib/x86-opcode-map.txt | 26 +++++++++++++------
2 files changed, 36 insertions(+), 16 deletions(-)
can_follow_write_pte/pmd look for the (RO & DIRTY) PTE/PMD to
verify an exclusive RO page still exists after a broken COW.
A shadow stack PTE is RO & PAGE_DIRTY_SW when it is shared,
otherwise RO & PAGE_DIRTY_HW.
Introduce pte_exclusive() and pmd_exclusive() to also verify a
shadow stack PTE is exclusive.
Also rename can_follow_write_pte/pmd() to can_follow_write() to
make their meaning clear; i.e. "Can we write to the page?", not
"Is the PTE writable?"
Signed-off-by: Yu-cheng Yu <redacted>
---
arch/x86/mm/pgtable.c | 18 ++++++++++++++++++
include/asm-generic/pgtable.h | 18 ++++++++++++++++++
mm/gup.c | 8 +++++---
mm/huge_memory.c | 8 +++++---
4 files changed, 46 insertions(+), 6 deletions(-)
The shadow stack for clone/fork is handled as the following:
(1) If ((clone_flags & (CLONE_VFORK | CLONE_VM)) == CLONE_VM),
the kernel allocates (and frees on thread exit) a new SHSTK
for the child.
It is possible for the kernel to complete the clone syscall
and set the child's SHSTK pointer to NULL and let the child
thread allocate a SHSTK for itself. There are two issues
in this approach: It is not compatible with existing code
that does inline syscall and it cannot handle signals before
the child can successfully allocate a SHSTK.
(2) For (clone_flags & CLONE_VFORK), the child uses the existing
SHSTK.
(3) For all other cases, the SHSTK is copied/reused whenever the
parent or the child does a call/ret.
This patch handles cases (1) & (2). Case (3) is handled in
the SHSTK page fault patches.
Signed-off-by: Yu-cheng Yu <redacted>
---
arch/x86/include/asm/cet.h | 2 ++
arch/x86/include/asm/mmu_context.h | 3 +++
arch/x86/kernel/cet.c | 34 ++++++++++++++++++++++++++++++
arch/x86/kernel/process.c | 1 +
arch/x86/kernel/process_64.c | 7 ++++++
5 files changed, 47 insertions(+)
@@ -317,6 +317,13 @@ int copy_thread_tls(unsigned long clone_flags, unsigned long sp,if(sp)childregs->sp=sp;+/* Allocate a new shadow stack for pthread */+if((clone_flags&(CLONE_VFORK|CLONE_VM))==CLONE_VM){+err=cet_setup_thread_shstk(p);+if(err)+gotoout;+}+err=-ENOMEM;if(unlikely(test_tsk_thread_flag(me,TIF_IO_BITMAP))){p->thread.io_bitmap_ptr=kmemdup(me->thread.io_bitmap_ptr,
On Thu, Aug 30, 2018 at 4:44 PM Yu-cheng Yu [off-list ref] wrote:
WRUSS is a new kernel-mode instruction but writes directly
to user shadow stack memory. This is used to construct
a return address on the shadow stack for the signal
handler.
This instruction can fault if the user shadow stack is
invalid shadow stack memory. In that case, the kernel does
fixup.
Signed-off-by: Yu-cheng Yu <redacted>
And here you just write into regs->ax, but your "asm volatile" doesn't
reserve that register. This looks wrong to me.
I think you probably want to add something like an explicit
`"+&a"(err)` output to the asm statements.
quoted hunk
@@ -1305,6 +1305,15 @@ __do_page_fault(struct pt_regs *regs, unsigned long error_code, error_code |= X86_PF_USER; flags |= FAULT_FLAG_USER; } else {+ /*+ * WRUSS is a kernel instrcution and but writes
Nits: typo ("instrcution"), weird grammar ("and but writes")
On Thu, Aug 30, 2018 at 4:43 PM Yu-cheng Yu [off-list ref] wrote:
quoted hunk
When Shadow Stack is enabled, the read-only and PAGE_DIRTY_HW PTE
setting is reserved only for the Shadow Stack. To track dirty of
non-Shadow Stack read-only PTEs, we use PAGE_DIRTY_SW.
Update ptep_set_wrprotect() and pmdp_set_wrprotect().
Signed-off-by: Yu-cheng Yu <redacted>
---
arch/x86/include/asm/pgtable.h | 42 ++++++++++++++++++++++++++++++++++
1 file changed, 42 insertions(+)
I don't understand why it's okay that you first atomically clear the
RW bit, then atomically switch from DIRTY_HW to DIRTY_SW. Doesn't that
mean that between the two atomic writes, another core can incorrectly
see a shadow stack?
From: Andy Lutomirski <luto@amacapital.net> Date: 2018-08-30 15:56:08
On Thu, Aug 30, 2018 at 8:39 AM, Jann Horn [off-list ref] wrote:
On Thu, Aug 30, 2018 at 4:44 PM Yu-cheng Yu [off-list ref] wrote:
quoted
WRUSS is a new kernel-mode instruction but writes directly
to user shadow stack memory. This is used to construct
a return address on the shadow stack for the signal
handler.
This instruction can fault if the user shadow stack is
invalid shadow stack memory. In that case, the kernel does
fixup.
Signed-off-by: Yu-cheng Yu <redacted>
And here you just write into regs->ax, but your "asm volatile" doesn't
reserve that register. This looks wrong to me.
I think you probably want to add something like an explicit
`"+&a"(err)` output to the asm statements.
We require asm goto support these days. How about using that? You
won't even need a special exception handler.
Also, please change the BUG to WARN in the you-did-it-wrong 32-bit
case. And return -EFAULT.
--Andy
On Thu, 2018-08-30 at 17:49 +0200, Jann Horn wrote:
On Thu, Aug 30, 2018 at 4:43 PM Yu-cheng Yu [off-list ref]
wrote:
quoted
When Shadow Stack is enabled, the read-only and PAGE_DIRTY_HW PTE
setting is reserved only for the Shadow Stack. To track dirty of
non-Shadow Stack read-only PTEs, we use PAGE_DIRTY_SW.
Update ptep_set_wrprotect() and pmdp_set_wrprotect().
Signed-off-by: Yu-cheng Yu <redacted>
---
arch/x86/include/asm/pgtable.h | 42
++++++++++++++++++++++++++++++++++
1 file changed, 42 insertions(+)
diff --git a/arch/x86/include/asm/pgtable.h
b/arch/x86/include/asm/pgtable.h
index 4d50de77ea96..556ef258eeff 100644
ptep_get_and_clear_full(struct mm_struct *mm,
static inline void ptep_set_wrprotect(struct mm_struct *mm,
unsigned long addr, pte_t
*ptep)
{
+ pte_t pte;
+
clear_bit(_PAGE_BIT_RW, (unsigned long *)&ptep->pte);
+ pte = *ptep;
+
+ /*
+ * Some processors can start a write, but ending up seeing
+ * a read-only PTE by the time they get to the Dirty bit.
+ * In this case, they will set the Dirty bit, leaving a
+ * read-only, Dirty PTE which looks like a Shadow Stack
PTE.
+ *
+ * However, this behavior has been improved and will not
occur
+ * on processors supporting Shadow Stacks. Without this
+ * guarantee, a transition to a non-present PTE and flush
the
+ * TLB would be needed.
+ *
+ * When change a writable PTE to read-only and if the PTE
has
+ * _PAGE_DIRTY_HW set, we move that bit to _PAGE_DIRTY_SW
so
+ * that the PTE is not a valid Shadow Stack PTE.
+ */
+ pte = pte_move_flags(pte, _PAGE_DIRTY_HW, _PAGE_DIRTY_SW);
+ set_pte_at(mm, addr, ptep, pte);
}
I don't understand why it's okay that you first atomically clear the
RW bit, then atomically switch from DIRTY_HW to DIRTY_SW. Doesn't
that
mean that between the two atomic writes, another core can
incorrectly
see a shadow stack?
Yes, we had that concern earlier and checked.
On processors supporting Shadow Stacks, that will not happen.
Yu-cheng
From: Dave Hansen <dave.hansen@linux.intel.com> Date: 2018-08-30 16:09:38
On 08/30/2018 08:49 AM, Jann Horn wrote:
quoted
@@ -1203,7 +1203,28 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) {+ pte_t pte;+ clear_bit(_PAGE_BIT_RW, (unsigned long *)&ptep->pte);+ pte = *ptep;++ /*+ * Some processors can start a write, but ending up seeing+ * a read-only PTE by the time they get to the Dirty bit.+ * In this case, they will set the Dirty bit, leaving a+ * read-only, Dirty PTE which looks like a Shadow Stack PTE.+ *+ * However, this behavior has been improved and will not occur+ * on processors supporting Shadow Stacks. Without this+ * guarantee, a transition to a non-present PTE and flush the+ * TLB would be needed.+ *+ * When change a writable PTE to read-only and if the PTE has+ * _PAGE_DIRTY_HW set, we move that bit to _PAGE_DIRTY_SW so+ * that the PTE is not a valid Shadow Stack PTE.+ */+ pte = pte_move_flags(pte, _PAGE_DIRTY_HW, _PAGE_DIRTY_SW);+ set_pte_at(mm, addr, ptep, pte); }
I don't understand why it's okay that you first atomically clear the
RW bit, then atomically switch from DIRTY_HW to DIRTY_SW. Doesn't that
mean that between the two atomic writes, another core can incorrectly
see a shadow stack?
Good point.
This could result in a spurious shadow-stack fault, or allow a
shadow-stack write to the page in the transient state.
But, the shadow-stack permissions are more restrictive than what could
be in the TLB at this point, so I don't think there's a real security
implication here.
The only trouble is handling the spurious shadow-stack fault. The
alternative is to go !Present for a bit, which we would probably just
handle fine in the existing page fault code.
On Thu, Aug 30, 2018 at 4:44 PM Yu-cheng Yu [off-list ref] wrote:
This patch adds basic shadow stack enabling/disabling routines.
A task's shadow stack is allocated from memory with VM_SHSTK
flag set and read-only protection. The shadow stack is
allocated to a fixed size of RLIMIT_STACK.
Signed-off-by: Yu-cheng Yu <redacted>
[...]
+static int set_shstk_ptr(unsigned long addr)
+{
+ u64 r;
+
+ if (!cpu_feature_enabled(X86_FEATURE_SHSTK))
+ return -1;
+
+ if ((addr >= TASK_SIZE_MAX) || (!IS_ALIGNED(addr, 4)))
+ return -1;
+
+ rdmsrl(MSR_IA32_U_CET, r);
+ wrmsrl(MSR_IA32_PL3_SSP, addr);
+ wrmsrl(MSR_IA32_U_CET, r | MSR_IA32_CET_SHSTK_EN);
+ return 0;
+}
Here's a really stupid question: Where is the logic for switching
those MSRs on task switch? MSR_IA32_PL3_SSP contains a userspace
pointer, so it has to be switched on task switch, right? I'm sure I'm
missing something obvious, but grepping for places that set
MSR_IA32_PL3_SSP to nonzero values through the entire patchset, I only
see set_shstk_ptr(), which is called from:
- cet_setup_shstk() (called from arch_setup_features(), which is
called from load_elf_binary())
- cet_restore_signal() (called on signal handler return)
- cet_setup_signal() (called from signal handling code)
On Thu, Aug 30, 2018 at 6:09 PM Dave Hansen [off-list ref] wrote:
On 08/30/2018 08:49 AM, Jann Horn wrote:
quoted
quoted
@@ -1203,7 +1203,28 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) {+ pte_t pte;+ clear_bit(_PAGE_BIT_RW, (unsigned long *)&ptep->pte);+ pte = *ptep;++ /*+ * Some processors can start a write, but ending up seeing+ * a read-only PTE by the time they get to the Dirty bit.+ * In this case, they will set the Dirty bit, leaving a+ * read-only, Dirty PTE which looks like a Shadow Stack PTE.+ *+ * However, this behavior has been improved and will not occur+ * on processors supporting Shadow Stacks. Without this+ * guarantee, a transition to a non-present PTE and flush the+ * TLB would be needed.+ *+ * When change a writable PTE to read-only and if the PTE has+ * _PAGE_DIRTY_HW set, we move that bit to _PAGE_DIRTY_SW so+ * that the PTE is not a valid Shadow Stack PTE.+ */+ pte = pte_move_flags(pte, _PAGE_DIRTY_HW, _PAGE_DIRTY_SW);+ set_pte_at(mm, addr, ptep, pte); }
I don't understand why it's okay that you first atomically clear the
RW bit, then atomically switch from DIRTY_HW to DIRTY_SW. Doesn't that
mean that between the two atomic writes, another core can incorrectly
see a shadow stack?
Good point.
This could result in a spurious shadow-stack fault, or allow a
shadow-stack write to the page in the transient state.
But, the shadow-stack permissions are more restrictive than what could
be in the TLB at this point, so I don't think there's a real security
implication here.
How about this:
Three threads (A, B, C) run with the same CR3.
1. a dirty+writable PTE is placed directly in front of B's shadow stack.
(this can happen, right? or is there a guard page?)
2. C's TLB caches the dirty+writable PTE.
3. A performs some syscall that triggers ptep_set_wrprotect().
4. A's syscall calls clear_bit().
5. B's TLB caches the transient shadow stack.
[now C has write access to B's transiently-extended shadow stack]
6. B recurses into the transiently-extended shadow stack
7. C overwrites the transiently-extended shadow stack area.
8. B returns through the transiently-extended shadow stack, giving
the attacker instruction pointer control in B.
9. A's syscall broadcasts a TLB flush.
Sure, it's not exactly an easy race and probably requires at least
some black timing magic to exploit, if it's exploitable at all - but
still. This seems suboptimal.
The only trouble is handling the spurious shadow-stack fault. The
alternative is to go !Present for a bit, which we would probably just
handle fine in the existing page fault code.
On Thu, 2018-08-30 at 18:10 +0200, Jann Horn wrote:
On Thu, Aug 30, 2018 at 4:44 PM Yu-cheng Yu [off-list ref]
wrote:
quoted
This patch adds basic shadow stack enabling/disabling routines.
A task's shadow stack is allocated from memory with VM_SHSTK
flag set and read-only protection. The shadow stack is
allocated to a fixed size of RLIMIT_STACK.
Signed-off-by: Yu-cheng Yu <redacted>
[...]
quoted
+static int set_shstk_ptr(unsigned long addr)
+{
+ u64 r;
+
+ if (!cpu_feature_enabled(X86_FEATURE_SHSTK))
+ return -1;
+
+ if ((addr >= TASK_SIZE_MAX) || (!IS_ALIGNED(addr, 4)))
+ return -1;
+
+ rdmsrl(MSR_IA32_U_CET, r);
+ wrmsrl(MSR_IA32_PL3_SSP, addr);
+ wrmsrl(MSR_IA32_U_CET, r | MSR_IA32_CET_SHSTK_EN);
+ return 0;
+}
Here's a really stupid question: Where is the logic for switching
those MSRs on task switch? MSR_IA32_PL3_SSP contains a userspace
pointer, so it has to be switched on task switch, right? I'm sure
I'm
missing something obvious, but grepping for places that set
MSR_IA32_PL3_SSP to nonzero values through the entire patchset, I
only
see set_shstk_ptr(), which is called from:
- cet_setup_shstk() (called from arch_setup_features(), which is
called from load_elf_binary())
- cet_restore_signal() (called on signal handler return)
- cet_setup_signal() (called from signal handling code)
The MSR is in the XSAVES buffer and switched by XSAVES/XRSTORS.
Yu-cheng
On Thu, 2018-08-30 at 08:55 -0700, Andy Lutomirski wrote:
On Thu, Aug 30, 2018 at 8:39 AM, Jann Horn [off-list ref] wrote:
quoted
On Thu, Aug 30, 2018 at 4:44 PM Yu-cheng Yu <yu-cheng.yu@intel.com
quoted
wrote:
WRUSS is a new kernel-mode instruction but writes directly
to user shadow stack memory. This is used to construct
a return address on the shadow stack for the signal
handler.
This instruction can fault if the user shadow stack is
invalid shadow stack memory. In that case, the kernel does
fixup.
Signed-off-by: Yu-cheng Yu <redacted>
And here you just write into regs->ax, but your "asm volatile"
doesn't
reserve that register. This looks wrong to me.
I think you probably want to add something like an explicit
`"+&a"(err)` output to the asm statements.
We require asm goto support these days. How about using that? You
won't even need a special exception handler.
Also, please change the BUG to WARN in the you-did-it-wrong 32-bit
case. And return -EFAULT.
--Andy
From: Dave Hansen <dave.hansen@linux.intel.com> Date: 2018-08-30 17:24:22
On 08/30/2018 09:23 AM, Jann Horn wrote:
Three threads (A, B, C) run with the same CR3.
1. a dirty+writable PTE is placed directly in front of B's shadow stack.
(this can happen, right? or is there a guard page?)
2. C's TLB caches the dirty+writable PTE.
3. A performs some syscall that triggers ptep_set_wrprotect().
4. A's syscall calls clear_bit().
5. B's TLB caches the transient shadow stack.
[now C has write access to B's transiently-extended shadow stack]
6. B recurses into the transiently-extended shadow stack
7. C overwrites the transiently-extended shadow stack area.
8. B returns through the transiently-extended shadow stack, giving
the attacker instruction pointer control in B.
9. A's syscall broadcasts a TLB flush.
Heh, that's a good point. The shadow stack permissions are *not*
strictly reduced because a page getting marked as shadow-stack has
*increased* permissions when being used as a shadow stack. Fun.
For general hardening, it seems like we want to ensure that there's a
guard page at the bottom of the shadow stack. Yu-cheng, do we have a
guard page?
But, to keep B's TLB from picking up the entry, I think we can just make
it !Present for a moment. No TLB can cache it, and I believe the same
"don't set Dirty on a !Writable entry" logic also holds for !Present
(modulo a weird erratum or two).
If we do that, we just need to make sure that the fault handler knows it
can get spurious faults from it, and might even run into the !Present
PTE for a moment. It might be a bit confusing because it won't be a
PROT_NONE, migration, or swap PTE, but will be !Present. We'll also
have to make sure that we're doing this in a way which is friendly to
the L1TF PTE handling.
On Thu, 2018-08-30 at 10:19 -0700, Dave Hansen wrote:
On 08/30/2018 09:23 AM, Jann Horn wrote:
quoted
Three threads (A, B, C) run with the same CR3.
1. a dirty+writable PTE is placed directly in front of B's shadow
stack.
(this can happen, right? or is there a guard page?)
2. C's TLB caches the dirty+writable PTE.
3. A performs some syscall that triggers ptep_set_wrprotect().
4. A's syscall calls clear_bit().
5. B's TLB caches the transient shadow stack.
[now C has write access to B's transiently-extended shadow stack]
6. B recurses into the transiently-extended shadow stack
7. C overwrites the transiently-extended shadow stack area.
8. B returns through the transiently-extended shadow stack, giving
the attacker instruction pointer control in B.
9. A's syscall broadcasts a TLB flush.
Heh, that's a good point. The shadow stack permissions are *not*
strictly reduced because a page getting marked as shadow-stack has
*increased* permissions when being used as a shadow stack. Fun.
For general hardening, it seems like we want to ensure that there's
a
guard page at the bottom of the shadow stack. Yu-cheng, do we have
a
guard page?
We don't have the guard page now, but there is a shadow stack token
there, which cannot be used as a return address.
Yu-cheng
From: Andy Lutomirski <luto@amacapital.net> Date: 2018-08-30 17:34:43
On Aug 30, 2018, at 10:19 AM, Dave Hansen [off-list ref] wrote:
quoted
On 08/30/2018 09:23 AM, Jann Horn wrote:
Three threads (A, B, C) run with the same CR3.
1. a dirty+writable PTE is placed directly in front of B's shadow stack.
(this can happen, right? or is there a guard page?)
2. C's TLB caches the dirty+writable PTE.
3. A performs some syscall that triggers ptep_set_wrprotect().
4. A's syscall calls clear_bit().
5. B's TLB caches the transient shadow stack.
[now C has write access to B's transiently-extended shadow stack]
6. B recurses into the transiently-extended shadow stack
7. C overwrites the transiently-extended shadow stack area.
8. B returns through the transiently-extended shadow stack, giving
the attacker instruction pointer control in B.
9. A's syscall broadcasts a TLB flush.
Heh, that's a good point. The shadow stack permissions are *not*
strictly reduced because a page getting marked as shadow-stack has
*increased* permissions when being used as a shadow stack. Fun.
For general hardening, it seems like we want to ensure that there's a
guard page at the bottom of the shadow stack. Yu-cheng, do we have a
guard page?
But, to keep B's TLB from picking up the entry, I think we can just make
it !Present for a moment. No TLB can cache it, and I believe the same
"don't set Dirty on a !Writable entry" logic also holds for !Present
(modulo a weird erratum or two).
From: Dave Hansen <dave.hansen@linux.intel.com> Date: 2018-08-30 17:36:18
On 08/30/2018 10:26 AM, Yu-cheng Yu wrote:
We don't have the guard page now, but there is a shadow stack token
there, which cannot be used as a return address.
The overall concern is that we could overflow into a page that we did
not intend. Either another actual shadow stack or something that a page
that the attacker constructed, like the transient scenario Jann described.
On Thu, 2018-08-30 at 10:33 -0700, Dave Hansen wrote:
On 08/30/2018 10:26 AM, Yu-cheng Yu wrote:
quoted
We don't have the guard page now, but there is a shadow stack
token
there, which cannot be used as a return address.
The overall concern is that we could overflow into a page that we
did
not intend. Either another actual shadow stack or something that a
page
that the attacker constructed, like the transient scenario Jann
described.
A task could go beyond the bottom of its shadow stack by doing either
'ret' or 'incssp'. If it is the 'ret' case, the token prevents it.
If it is the 'incssp' case, a guard page cannot prevent it entirely,
right?
Yu-cheng
On Thu, Aug 30, 2018 at 7:58 PM Yu-cheng Yu [off-list ref] wrote:
On Thu, 2018-08-30 at 10:33 -0700, Dave Hansen wrote:
quoted
On 08/30/2018 10:26 AM, Yu-cheng Yu wrote:
quoted
We don't have the guard page now, but there is a shadow stack
token
there, which cannot be used as a return address.
The overall concern is that we could overflow into a page that we
did
not intend. Either another actual shadow stack or something that a
page
that the attacker constructed, like the transient scenario Jann
described.
A task could go beyond the bottom of its shadow stack by doing either
'ret' or 'incssp'. If it is the 'ret' case, the token prevents it.
If it is the 'incssp' case, a guard page cannot prevent it entirely,
right?
From: Dave Hansen <dave.hansen@linux.intel.com> Date: 2018-08-30 18:55:45
On 08/30/2018 10:34 AM, Andy Lutomirski wrote:
quoted
But, to keep B's TLB from picking up the entry, I think we can just make
it !Present for a moment. No TLB can cache it, and I believe the same
"don't set Dirty on a !Writable entry" logic also holds for !Present
(modulo a weird erratum or two).
Can we get documentation? Pretty please?
The accessed bit description in the SDM looks pretty good to me today:
Whenever the processor uses a paging-structure entry as part of
linear-address translation, it sets the accessed flag in that entry
(if it is not already set).
If it's !Present, it can't used as part of a translation so can't be
set. I think that covers the thing I was unsure about.
But, Dirty is a bit, er, muddier, but mostly because it only gets set on
leaf entries:
Whenever there is a write to a linear address, the processor sets the
dirty flag (if it is not already set) in the paging- structure entry
that identifies the final physical address for the linear address
(either a PTE or a paging-structure entry in which the PS flag is
1).
That little hunk will definitely need to get updated with something like:
On processors enumerating support for CET, the processor will on
set the dirty flag on paging structure entries in which the W
flag is 1.
From: Randy Dunlap <rdunlap@infradead.org> Date: 2018-08-30 19:59:40
On 08/30/2018 07:38 AM, Yu-cheng Yu wrote:
quoted hunk
When Shadow Stack is enabled, the read-only and PAGE_DIRTY_HW PTE
setting is reserved only for the Shadow Stack. To track dirty of
non-Shadow Stack read-only PTEs, we use PAGE_DIRTY_SW.
Update ptep_set_wrprotect() and pmdp_set_wrprotect().
Signed-off-by: Yu-cheng Yu <redacted>
---
arch/x86/include/asm/pgtable.h | 42 ++++++++++++++++++++++++++++++++++
1 file changed, 42 insertions(+)
+ * a read-only PTE by the time they get to the Dirty bit.
+ * In this case, they will set the Dirty bit, leaving a
+ * read-only, Dirty PTE which looks like a Shadow Stack PTE.
+ *
+ * However, this behavior has been improved and will not occur
+ * on processors supporting Shadow Stacks. Without this
+ * guarantee, a transition to a non-present PTE and flush the
+ * TLB would be needed.
+ *
+ * When change a writable PTE to read-only and if the PTE has
changing
quoted hunk
+ * _PAGE_DIRTY_HW set, we move that bit to _PAGE_DIRTY_SW so
+ * that the PTE is not a valid Shadow Stack PTE.
+ */
+ pte = pte_move_flags(pte, _PAGE_DIRTY_HW, _PAGE_DIRTY_SW);
+ set_pte_at(mm, addr, ptep, pte);
}
#define flush_tlb_fix_spurious_fault(vma, address) do { } while (0)
@@ -1266,7 +1287,28 @@ static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm, static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) {+ pmd_t pmd;+ clear_bit(_PAGE_BIT_RW, (unsigned long *)pmdp);+ pmd = *pmdp;++ /*+ * Some processors can start a write, but ending up seeing
but end up seeing
+ * a read-only PTE by the time they get to the Dirty bit.
+ * In this case, they will set the Dirty bit, leaving a
+ * read-only, Dirty PTE which looks like a Shadow Stack PTE.
+ *
+ * However, this behavior has been improved and will not occur
+ * on processors supporting Shadow Stacks. Without this
+ * guarantee, a transition to a non-present PTE and flush the
+ * TLB would be needed.
+ *
+ * When change a writable PTE to read-only and if the PTE has
changing
+ * _PAGE_DIRTY_HW set, we move that bit to _PAGE_DIRTY_SW so
+ * that the PTE is not a valid Shadow Stack PTE.
+ */
+ pmd = pmd_move_flags(pmd, _PAGE_DIRTY_HW, _PAGE_DIRTY_SW);
+ set_pmd_at(mm, addr, pmdp, pmd);
}
#define pud_write pud_write
On Thu, 2018-08-30 at 19:59 +0200, Jann Horn wrote:
On Thu, Aug 30, 2018 at 7:58 PM Yu-cheng Yu [off-list ref]
wrote:
quoted
On Thu, 2018-08-30 at 10:33 -0700, Dave Hansen wrote:
quoted
On 08/30/2018 10:26 AM, Yu-cheng Yu wrote:
quoted
We don't have the guard page now, but there is a shadow stack
token
there, which cannot be used as a return address.
The overall concern is that we could overflow into a page that
we
did
not intend. Either another actual shadow stack or something
that a
page
that the attacker constructed, like the transient scenario Jann
described.
A task could go beyond the bottom of its shadow stack by doing
either
'ret' or 'incssp'. If it is the 'ret' case, the token prevents
it.
If it is the 'incssp' case, a guard page cannot prevent it
entirely,
right?
I mean the other direction, on "call".
In the flow you described, if C writes to the overflow page before B
gets in with a 'call', the return address is still correct for B. To
make an attack, C needs to write again before the TLB flush. I agree
that is possible.
Assume we have a guard page, can someone in the short window do
recursive calls in B, move ssp to the end of the guard page, and
trigger the same again? He can simply take the incssp route.
On Thu, 2018-08-30 at 12:59 -0700, Randy Dunlap wrote:
On 08/30/2018 07:38 AM, Yu-cheng Yu wrote:
quoted
When Shadow Stack is enabled, the read-only and PAGE_DIRTY_HW PTE
setting is reserved only for the Shadow Stack. To track dirty of
non-Shadow Stack read-only PTEs, we use PAGE_DIRTY_SW.
Update ptep_set_wrprotect() and pmdp_set_wrprotect().
Signed-off-by: Yu-cheng Yu <redacted>
---
arch/x86/include/asm/pgtable.h | 42
++++++++++++++++++++++++++++++++++
1 file changed, 42 insertions(+)
diff --git a/arch/x86/include/asm/pgtable.h
b/arch/x86/include/asm/pgtable.h
index 4d50de77ea96..556ef258eeff 100644
ptep_get_and_clear_full(struct mm_struct *mm,
static inline void ptep_set_wrprotect(struct mm_struct *mm,
unsigned long addr, pte_t
*ptep)
{
+ pte_t pte;
+
clear_bit(_PAGE_BIT_RW, (unsigned long *)&ptep->pte);
+ pte = *ptep;
+
+ /*
+ * Some processors can start a write, but ending up
seeing
but end up seeing
quoted
+ * a read-only PTE by the time they get to the Dirty bit.
+ * In this case, they will set the Dirty bit, leaving a
+ * read-only, Dirty PTE which looks like a Shadow Stack
PTE.
+ *
+ * However, this behavior has been improved and will not
occur
+ * on processors supporting Shadow Stacks. Without this
+ * guarantee, a transition to a non-present PTE and flush
the
+ * TLB would be needed.
+ *
+ * When change a writable PTE to read-only and if the PTE
has
changing
quoted
+ * _PAGE_DIRTY_HW set, we move that bit to _PAGE_DIRTY_SW
so
+ * that the PTE is not a valid Shadow Stack PTE.
+ */
+ pte = pte_move_flags(pte, _PAGE_DIRTY_HW,
_PAGE_DIRTY_SW);
+ set_pte_at(mm, addr, ptep, pte);
}
#define flush_tlb_fix_spurious_fault(vma, address) do { } while
(0)
@@ -1266,7 +1287,28 @@ static inline pud_t
pudp_huge_get_and_clear(struct mm_struct *mm,
static inline void pmdp_set_wrprotect(struct mm_struct *mm,
unsigned long addr, pmd_t
*pmdp)
{
+ pmd_t pmd;
+
clear_bit(_PAGE_BIT_RW, (unsigned long *)pmdp);
+ pmd = *pmdp;
+
+ /*
+ * Some processors can start a write, but ending up
seeing
but end up seeing
quoted
+ * a read-only PTE by the time they get to the Dirty bit.
+ * In this case, they will set the Dirty bit, leaving a
+ * read-only, Dirty PTE which looks like a Shadow Stack
PTE.
+ *
+ * However, this behavior has been improved and will not
occur
+ * on processors supporting Shadow Stacks. Without this
+ * guarantee, a transition to a non-present PTE and flush
the
+ * TLB would be needed.
+ *
+ * When change a writable PTE to read-only and if the PTE
has
changing
quoted
+ * _PAGE_DIRTY_HW set, we move that bit to _PAGE_DIRTY_SW
so
+ * that the PTE is not a valid Shadow Stack PTE.
+ */
+ pmd = pmd_move_flags(pmd, _PAGE_DIRTY_HW,
_PAGE_DIRTY_SW);
+ set_pmd_at(mm, addr, pmdp, pmd);
}
#define pud_write pud_write
Can you add something like "It attempts to protect process from
running arbitrary code even after attacker has control of its stack"
-- for people that don't know what ROP is, and perhaps link to
wikipedia explaining ROP or something...
It can be implemented
+to protect both the kernel and applications. In the first phase,
+only the user-mode protection is implemented for the 64-bit kernel.
+Thirty-two bit applications are supported under the compatibility
32-bit (for consistency).
Ok, so CET stops execution of malicious code before architectural
effects are visible, correct? Does it prevent micro-architectural
effects of the malicious code? (cache content would be one example;
see Spectre).
+Signal
+------
+
+The main program and its signal handlers use the same SHSTK. Because
+the SHSTK stores only return addresses, we can estimate a large
+enough SHSTK to cover the condition that both the program stack and
+the sigaltstack run out.
On Thu, Aug 30, 2018 at 10:25 PM Yu-cheng Yu [off-list ref] wrote:
On Thu, 2018-08-30 at 19:59 +0200, Jann Horn wrote:
quoted
On Thu, Aug 30, 2018 at 7:58 PM Yu-cheng Yu [off-list ref]
wrote:
quoted
On Thu, 2018-08-30 at 10:33 -0700, Dave Hansen wrote:
quoted
On 08/30/2018 10:26 AM, Yu-cheng Yu wrote:
quoted
We don't have the guard page now, but there is a shadow stack
token
there, which cannot be used as a return address.
The overall concern is that we could overflow into a page that
we
did
not intend. Either another actual shadow stack or something
that a
page
that the attacker constructed, like the transient scenario Jann
described.
A task could go beyond the bottom of its shadow stack by doing
either
'ret' or 'incssp'. If it is the 'ret' case, the token prevents
it.
If it is the 'incssp' case, a guard page cannot prevent it
entirely,
right?
I mean the other direction, on "call".
In the flow you described, if C writes to the overflow page before B
gets in with a 'call', the return address is still correct for B. To
make an attack, C needs to write again before the TLB flush. I agree
that is possible.
Assume we have a guard page, can someone in the short window do
recursive calls in B, move ssp to the end of the guard page, and
trigger the same again? He can simply take the incssp route.
I don't understand what you're saying. If the shadow stack is between
guard pages, you should never be able to move SSP past that area's
guard pages without an appropriate shadow stack token (not even with
INCSSP, since that has a maximum range of PAGE_SIZE/2), and therefore,
it shouldn't matter whether memory outside that range is incorrectly
marked as shadow stack. Am I missing something?
On Thu, 2018-08-30 at 22:44 +0200, Jann Horn wrote:
On Thu, Aug 30, 2018 at 10:25 PM Yu-cheng Yu [off-list ref]
wrote:
...
quoted
In the flow you described, if C writes to the overflow page before
B
gets in with a 'call', the return address is still correct for
B. To
make an attack, C needs to write again before the TLB flush. I
agree
that is possible.
Assume we have a guard page, can someone in the short window do
recursive calls in B, move ssp to the end of the guard page, and
trigger the same again? He can simply take the incssp route.
I don't understand what you're saying. If the shadow stack is
between
guard pages, you should never be able to move SSP past that area's
guard pages without an appropriate shadow stack token (not even with
INCSSP, since that has a maximum range of PAGE_SIZE/2), and
therefore,
it shouldn't matter whether memory outside that range is incorrectly
marked as shadow stack. Am I missing something?
INCSSP has a range of 256, but we can do multiple of that.
But I realize the key is not to have the transient SHSTK page at all.
The guard page is !pte_write() and even we have flaws in
ptep_set_wrprotect(), there will not be any transient SHSTK pages. I
will add guard pages to both ends.
Still thinking how to fix ptep_set_wrprotect().
Yu-cheng
On Thu, Aug 30, 2018 at 10:57 PM Yu-cheng Yu [off-list ref] wrote:
On Thu, 2018-08-30 at 22:44 +0200, Jann Horn wrote:
quoted
On Thu, Aug 30, 2018 at 10:25 PM Yu-cheng Yu [off-list ref]
wrote:
...
quoted
quoted
In the flow you described, if C writes to the overflow page before
B
gets in with a 'call', the return address is still correct for
B. To
make an attack, C needs to write again before the TLB flush. I
agree
that is possible.
Assume we have a guard page, can someone in the short window do
recursive calls in B, move ssp to the end of the guard page, and
trigger the same again? He can simply take the incssp route.
I don't understand what you're saying. If the shadow stack is
between
guard pages, you should never be able to move SSP past that area's
guard pages without an appropriate shadow stack token (not even with
INCSSP, since that has a maximum range of PAGE_SIZE/2), and
therefore,
it shouldn't matter whether memory outside that range is incorrectly
marked as shadow stack. Am I missing something?
INCSSP has a range of 256, but we can do multiple of that.
But I realize the key is not to have the transient SHSTK page at all.
The guard page is !pte_write() and even we have flaws in
ptep_set_wrprotect(), there will not be any transient SHSTK pages. I
will add guard pages to both ends.
Still thinking how to fix ptep_set_wrprotect().
On Thu, Aug 30, 2018 at 11:01 PM Jann Horn [off-list ref] wrote:
On Thu, Aug 30, 2018 at 10:57 PM Yu-cheng Yu [off-list ref] wrote:
quoted
On Thu, 2018-08-30 at 22:44 +0200, Jann Horn wrote:
quoted
On Thu, Aug 30, 2018 at 10:25 PM Yu-cheng Yu [off-list ref]
wrote:
...
quoted
quoted
In the flow you described, if C writes to the overflow page before
B
gets in with a 'call', the return address is still correct for
B. To
make an attack, C needs to write again before the TLB flush. I
agree
that is possible.
Assume we have a guard page, can someone in the short window do
recursive calls in B, move ssp to the end of the guard page, and
trigger the same again? He can simply take the incssp route.
I don't understand what you're saying. If the shadow stack is
between
guard pages, you should never be able to move SSP past that area's
guard pages without an appropriate shadow stack token (not even with
INCSSP, since that has a maximum range of PAGE_SIZE/2), and
therefore,
it shouldn't matter whether memory outside that range is incorrectly
marked as shadow stack. Am I missing something?
INCSSP has a range of 256, but we can do multiple of that.
But I realize the key is not to have the transient SHSTK page at all.
The guard page is !pte_write() and even we have flaws in
ptep_set_wrprotect(), there will not be any transient SHSTK pages. I
will add guard pages to both ends.
Still thinking how to fix ptep_set_wrprotect().
cmpxchg loop? Or is that slow?
Something like this:
static inline void ptep_set_wrprotect(struct mm_struct *mm,
unsigned long addr, pte_t *ptep)
{
pte_t pte = READ_ONCE(*ptep), new_pte;
/* ... your comment about not needing a TLB shootdown here ... */
do {
pte = pte_wrprotect(pte);
/* note: relies on _PAGE_DIRTY_HW < _PAGE_DIRTY_SW */
/* dirty direct bit-twiddling; you can probably write
this in a nicer way */
pte.pte |= (pte.pte & _PAGE_DIRTY_HW) >>
_PAGE_BIT_DIRTY_HW << _PAGE_BIT_DIRTY_SW;
pte.pte &= ~_PAGE_DIRTY_HW;
pte = cmpxchg(ptep, pte, new_pte);
} while (pte != new_pte);
}
I think this has the advantage of not generating weird spurious pagefaults.
It's not compatible with Xen PV, but I'm guessing that this whole
feature isn't going to support Xen PV anyway? So you could switch
between two implementations of ptep_set_wrprotect using the pvop
mechanism or so - one for environments that support shadow stacks, one
for all other environments.
Or is there some arcane reason why cmpxchg doesn't work here the way I
think it should?
against
+return/jump-oriented programing (ROP) attacks.
Can you add something like "It attempts to protect process from
running arbitrary code even after attacker has control of its stack"
-- for people that don't know what ROP is, and perhaps link to
wikipedia explaining ROP or something...
quoted
It can be implemented
+to protect both the kernel and applications. In the first phase,
+only the user-mode protection is implemented for the 64-bit
kernel.
+Thirty-two bit applications are supported under the compatibility
32-bit (for consistency).
Ok, so CET stops execution of malicious code before architectural
effects are visible, correct? Does it prevent micro-architectural
effects of the malicious code? (cache content would be one example;
see Spectre).
+Signal
+------
+
+The main program and its signal handlers use the same
SHSTK. Because
+the SHSTK stores only return addresses, we can estimate a large
+enough SHSTK to cover the condition that both the program stack
and
+the sigaltstack run out.
English? Is it estimate or is it large enough? "a large" -- "a"
should
be deleted AFAICT.
From: Andy Lutomirski <luto@amacapital.net> Date: 2018-08-31 01:23:21
On Aug 30, 2018, at 10:59 AM, Jann Horn [off-list ref] wrote:
quoted
On Thu, Aug 30, 2018 at 7:58 PM Yu-cheng Yu [off-list ref] wrote:
quoted
On Thu, 2018-08-30 at 10:33 -0700, Dave Hansen wrote:
quoted
On 08/30/2018 10:26 AM, Yu-cheng Yu wrote:
We don't have the guard page now, but there is a shadow stack
token
there, which cannot be used as a return address.
The overall concern is that we could overflow into a page that we
did
not intend. Either another actual shadow stack or something that a
page
that the attacker constructed, like the transient scenario Jann
described.
A task could go beyond the bottom of its shadow stack by doing either
'ret' or 'incssp'. If it is the 'ret' case, the token prevents it.
If it is the 'incssp' case, a guard page cannot prevent it entirely,
right?
I mean the other direction, on "call".
I still think that shadow stacks should work just like mmap and that mmap should learn to add guard pages for all non-MAP_FIXED allocations.
From: Peter Zijlstra <peterz@infradead.org> Date: 2018-08-31 09:53:51
On Thu, Aug 30, 2018 at 11:47:16PM +0200, Jann Horn wrote:
do {
pte = pte_wrprotect(pte);
/* note: relies on _PAGE_DIRTY_HW < _PAGE_DIRTY_SW */
/* dirty direct bit-twiddling; you can probably write
this in a nicer way */
pte.pte |= (pte.pte & _PAGE_DIRTY_HW) >>
_PAGE_BIT_DIRTY_HW << _PAGE_BIT_DIRTY_SW;
pte.pte &= ~_PAGE_DIRTY_HW;
pte = cmpxchg(ptep, pte, new_pte);
} while (pte != new_pte);
Please use the form:
pte_t new_pte, pte = READ_ONCE(*ptep);
do {
new_pte = /* ... */;
} while (!try_cmpxchg(ptep, &pte, new_pte);
Also, this will fail to build on i386-PAE, but I suspect this code will
be under some CONFIG option specific to x86_64 anyway.
On Fri, 2018-08-31 at 11:53 +0200, Peter Zijlstra wrote:
On Thu, Aug 30, 2018 at 11:47:16PM +0200, Jann Horn wrote:
quoted
do {
pte = pte_wrprotect(pte);
/* note: relies on _PAGE_DIRTY_HW < _PAGE_DIRTY_SW
*/
/* dirty direct bit-twiddling; you can probably
write
this in a nicer way */
pte.pte |= (pte.pte & _PAGE_DIRTY_HW) >>
_PAGE_BIT_DIRTY_HW << _PAGE_BIT_DIRTY_SW;
pte.pte &= ~_PAGE_DIRTY_HW;
pte = cmpxchg(ptep, pte, new_pte);
} while (pte != new_pte);
Please use the form:
pte_t new_pte, pte = READ_ONCE(*ptep);
do {
new_pte = /* ... */;
} while (!try_cmpxchg(ptep, &pte, new_pte);
Also, this will fail to build on i386-PAE, but I suspect this code
will
be under some CONFIG option specific to x86_64 anyway.
From: Dave Hansen <dave.hansen@linux.intel.com> Date: 2018-08-31 14:49:55
On 08/31/2018 07:33 AM, Yu-cheng Yu wrote:
Please use the form:
pte_t new_pte, pte = READ_ONCE(*ptep);
do {
new_pte = /* ... */;
} while (!try_cmpxchg(ptep, &pte, new_pte);
It's probably also worth doing some testing to see if you can detect the
cost of the cmpxchg. It's definitely more than the old code.
A loop that does mprotect(PROT_READ) followed by
mprotect(PROT_READ|PROT_WRITE) should do it.
On Thu, Aug 30, 2018 at 4:43 PM Yu-cheng Yu [off-list ref] wrote:
A control protection exception is triggered when a control flow transfer
attempt violated shadow stack or indirect branch tracking constraints.
For example, the return address for a RET instruction differs from the
safe copy on the shadow stack; or a JMP instruction arrives at a non-
ENDBR instruction.
The control protection exception handler works in a similar way as the
general protection fault handler.
Is there a reason why all the code in this patch isn't #ifdef'ed away
on builds that don't support CET? It looks like the CET handler is
hooked up to the IDT conditionally, but the handler code is always
built?
@@ -578,6 +578,64 @@ do_general_protection(struct pt_regs *regs, long error_code)}NOKPROBE_SYMBOL(do_general_protection);+staticconstchar*control_protection_err[]=+{+"unknown",+"near-ret",+"far-ret/iret",+"endbranch",+"rstorssp",+"setssbsy",+};++/*+*Whenacontrolprotectionexceptionoccurs,sendasignal+*totheresponsibleapplication.Currently,control+*protectionisonlyenabledfortheusermode.This+*exceptionshouldnotcomefromthekernelmode.+*/+dotraplinkagevoid+do_control_protection(structpt_regs*regs,longerror_code)+{+structtask_struct*tsk;++RCU_LOCKDEP_WARN(!rcu_is_watching(),"entry code didn't wake RCU");+if(notify_die(DIE_TRAP,"control protection fault",regs,+error_code,X86_TRAP_CP,SIGSEGV)==NOTIFY_STOP)+return;+cond_local_irq_enable(regs);++if(!user_mode(regs))+die("kernel control protection fault",regs,error_code);++if(!static_cpu_has(X86_FEATURE_SHSTK)&&+!static_cpu_has(X86_FEATURE_IBT))+WARN_ONCE(1,"CET is disabled but got control "+"protection fault\n");++tsk=current;+tsk->thread.error_code=error_code;+tsk->thread.trap_nr=X86_TRAP_CP;++if(show_unhandled_signals&&unhandled_signal(tsk,SIGSEGV)&&+printk_ratelimit()){+unsignedintmax_err;++max_err=ARRAY_SIZE(control_protection_err)-1;+if((error_code<0)||(error_code>max_err))+error_code=0;+pr_info("%s[%d] control protection ip:%lx sp:%lx error:%lx(%s)",+tsk->comm,task_pid_nr(tsk),+regs->ip,regs->sp,error_code,+control_protection_err[error_code]);+print_vma_addr(" in ",regs->ip);
Shouldn't this be using KERN_CONT, like other callers of
print_vma_addr(), to get the desired output?
On Fri, 2018-08-31 at 07:47 -0700, Dave Hansen wrote:
On 08/31/2018 07:33 AM, Yu-cheng Yu wrote:
quoted
Please use the form:
pte_t new_pte, pte = READ_ONCE(*ptep);
do {
new_pte = /* ... */;
} while (!try_cmpxchg(ptep, &pte, new_pte);
It's probably also worth doing some testing to see if you can detect
the
cost of the cmpxchg. It's definitely more than the old code.
A loop that does mprotect(PROT_READ) followed by
mprotect(PROT_READ|PROT_WRITE) should do it.
I created the test,
https://github.com/yyu168/cet-smoke-test/blob/quick/quick/mprotect_ben
ch.c
then realized this won't work.
To trigger a race in ptep_set_wrprotect(), we need to fork from one of
three pthread siblings.
Or do we measure only how much this affects fork?
If there is no racing, the effect should be minimal.
Yu-cheng
From: Dave Hansen <dave.hansen@linux.intel.com> Date: 2018-08-31 15:58:43
On 08/31/2018 08:48 AM, Yu-cheng Yu wrote:
To trigger a race in ptep_set_wrprotect(), we need to fork from one of
three pthread siblings.
Or do we measure only how much this affects fork?
If there is no racing, the effect should be minimal.
We don't need a race.
I think the cmpxchg will be slower, even without a race, than the code
that was there before. The cmpxchg is a simple, straightforward
solution, but we're putting it in place of a plain memory write, which
is suboptimal.
But, before I nitpick the performance, I wanted to see if we could even
detect a delta.
On Fri, 2018-08-31 at 17:01 +0200, Jann Horn wrote:
Is there a reason why all the code in this patch isn't #ifdef'ed
away
on builds that don't support CET? It looks like the CET handler is
hooked up to the IDT conditionally, but the handler code is always
built?
Yes, in idt.c, it should have been:
#ifdef CONFIG_X86_64
INTG(X86_TRAP_CP, control_protection),
#endif
I will fix it.
quoted
+dotraplinkage void
+do_control_protection(struct pt_regs *regs, long error_code)
+{
+ struct task_struct *tsk;
+
+ RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't
wake RCU");
+ if (notify_die(DIE_TRAP, "control protection fault", regs,
+ error_code, X86_TRAP_CP, SIGSEGV) ==
NOTIFY_STOP)
+ return;
+ cond_local_irq_enable(regs);
+
+ if (!user_mode(regs))
+ die("kernel control protection fault", regs,
error_code);
+
+ if (!static_cpu_has(X86_FEATURE_SHSTK) &&
+ !static_cpu_has(X86_FEATURE_IBT))
+ WARN_ONCE(1, "CET is disabled but got control "
+ "protection fault\n");
+
+ tsk = current;
+ tsk->thread.error_code = error_code;
+ tsk->thread.trap_nr = X86_TRAP_CP;
+
+ if (show_unhandled_signals && unhandled_signal(tsk,
SIGSEGV) &&
+ printk_ratelimit()) {
+ unsigned int max_err;
+
+ max_err = ARRAY_SIZE(control_protection_err) - 1;
+ if ((error_code < 0) || (error_code > max_err))
+ error_code = 0;
+ pr_info("%s[%d] control protection ip:%lx sp:%lx
error:%lx(%s)",
+ tsk->comm, task_pid_nr(tsk),
+ regs->ip, regs->sp, error_code,
+ control_protection_err[error_code]);
+ print_vma_addr(" in ", regs->ip);
Shouldn't this be using KERN_CONT, like other callers of
print_vma_addr(), to get the desired output?
From: Dave Hansen <dave.hansen@linux.intel.com> Date: 2018-08-31 16:29:55
On 08/30/2018 07:38 AM, Yu-cheng Yu wrote:
+ * Some processors can start a write, but ending up seeing
+ * a read-only PTE by the time they get to the Dirty bit.
+ * In this case, they will set the Dirty bit, leaving a
+ * read-only, Dirty PTE which looks like a Shadow Stack PTE.
+ *
+ * However, this behavior has been improved and will not occur
+ * on processors supporting Shadow Stacks. Without this
+ * guarantee, a transition to a non-present PTE and flush the
+ * TLB would be needed.
Did we publicly document this behavior anywhere? I can't seem to find it.
From: Peter Zijlstra <peterz@infradead.org> Date: 2018-08-31 16:30:07
On Fri, Aug 31, 2018 at 08:58:39AM -0700, Dave Hansen wrote:
On 08/31/2018 08:48 AM, Yu-cheng Yu wrote:
quoted
To trigger a race in ptep_set_wrprotect(), we need to fork from one of
three pthread siblings.
Or do we measure only how much this affects fork?
If there is no racing, the effect should be minimal.
We don't need a race.
I think the cmpxchg will be slower, even without a race, than the code
that was there before. The cmpxchg is a simple, straightforward
solution, but we're putting it in place of a plain memory write, which
is suboptimal.
From: Andy Lutomirski <luto@kernel.org> Date: 2018-08-31 17:47:17
On Thu, Aug 30, 2018 at 11:55 AM, Dave Hansen
[off-list ref] wrote:
On 08/30/2018 10:34 AM, Andy Lutomirski wrote:
quoted
quoted
But, to keep B's TLB from picking up the entry, I think we can just make
it !Present for a moment. No TLB can cache it, and I believe the same
"don't set Dirty on a !Writable entry" logic also holds for !Present
(modulo a weird erratum or two).
Can we get documentation? Pretty please?
The accessed bit description in the SDM looks pretty good to me today:
quoted
Whenever the processor uses a paging-structure entry as part of
linear-address translation, it sets the accessed flag in that entry
(if it is not already set).
If it's !Present, it can't used as part of a translation so can't be
set. I think that covers the thing I was unsure about.
But, Dirty is a bit, er, muddier, but mostly because it only gets set on
leaf entries:
quoted
Whenever there is a write to a linear address, the processor sets the
dirty flag (if it is not already set) in the paging- structure entry
that identifies the final physical address for the linear address
(either a PTE or a paging-structure entry in which the PS flag is
1).
That little hunk will definitely need to get updated with something like:
On processors enumerating support for CET, the processor will on
set the dirty flag on paging structure entries in which the W
flag is 1.
Can we get something much stronger, perhaps? Like this:
On processors enumerating support for CET, the processor will write to
the accessed and/or dirty flags atomically, as if using the LOCK
CMPXCHG instruction. The memory access, any cached entries in any
paging-structure caches, and the values in the paging-structure entry
before and after writing the A and/or D bits will all be consistent.
I'm sure this could be worded better. The point is that the CPU
should, atomically, load the PTE, check if it allows the access, set A
and/or D appropriately, write the new value to the TLB, and use that
value for the access. This is clearly a little bit slower than what
old CPUs could do when writing to an already-in-TLB writable non-dirty
entry, but new CPUs are going to have to atomically check the W bit.
(I assume that even old CPUs will *atomically* set the D bit as if by
LOCK BTS, but this is all very vague in the SDM IIRC.)
From: Dave Hansen <dave.hansen@linux.intel.com> Date: 2018-08-31 17:52:05
On 08/31/2018 10:46 AM, Andy Lutomirski wrote:
On Thu, Aug 30, 2018 at 11:55 AM, Dave Hansen
quoted
That little hunk will definitely need to get updated with something like:
On processors enumerating support for CET, the processor will on
set the dirty flag on paging structure entries in which the W
flag is 1.
Can we get something much stronger, perhaps? Like this:
On processors enumerating support for CET, the processor will write to
the accessed and/or dirty flags atomically, as if using the LOCK
CMPXCHG instruction. The memory access, any cached entries in any
paging-structure caches, and the values in the paging-structure entry
before and after writing the A and/or D bits will all be consistent.
There's some talk of this already in: 8.1.2.1 Automatic Locking:
When updating page-directory and page-table entries — When updating
page-directory and page-table entries, the processor uses locked
cycles to set the accessed and dirty flag in the page-directory and
page-table entries.
As for the A/D consistency, I'll see if I can share that before it hits
the SDM for real and see if it's sufficient for everybody.
On Thu, 2018-08-30 at 09:22 -0700, Yu-cheng Yu wrote:
On Thu, 2018-08-30 at 08:55 -0700, Andy Lutomirski wrote:
quoted
On Thu, Aug 30, 2018 at 8:39 AM, Jann Horn [off-list ref]
wrote:
quoted
On Thu, Aug 30, 2018 at 4:44 PM Yu-cheng Yu <yu-cheng.yu@intel.c
om
quoted
wrote:
WRUSS is a new kernel-mode instruction but writes directly
to user shadow stack memory. This is used to construct
a return address on the shadow stack for the signal
handler.
This instruction can fault if the user shadow stack is
invalid shadow stack memory. In that case, the kernel does
fixup.
Signed-off-by: Yu-cheng Yu <redacted>
And here you just write into regs->ax, but your "asm volatile"
doesn't
reserve that register. This looks wrong to me.
I think you probably want to add something like an explicit
`"+&a"(err)` output to the asm statements.
We require asm goto support these days. How about using
that? You
won't even need a special exception handler.
Maybe something like this? It looks simple now.
static inline int write_user_shstk_64(unsigned long addr, unsigned
long val)
{
asm_volatile_goto("wrussq %1, (%0)\n"
"jmp %l[ok]\n"
".section .fixup,\"ax\"n"
"jmp %l[fail]\n"
".previous\n"
:: "r" (addr), "r" (val)
:: ok, fail);
ok:
return 0;
fail:
return -1;
}
From: Andy Lutomirski <luto@kernel.org> Date: 2018-08-31 22:16:41
On Fri, Aug 31, 2018 at 2:49 PM, Yu-cheng Yu [off-list ref] wrote:
On Thu, 2018-08-30 at 09:22 -0700, Yu-cheng Yu wrote:
quoted
On Thu, 2018-08-30 at 08:55 -0700, Andy Lutomirski wrote:
quoted
On Thu, Aug 30, 2018 at 8:39 AM, Jann Horn [off-list ref]
wrote:
quoted
On Thu, Aug 30, 2018 at 4:44 PM Yu-cheng Yu <yu-cheng.yu@intel.c
om
quoted
wrote:
WRUSS is a new kernel-mode instruction but writes directly
to user shadow stack memory. This is used to construct
a return address on the shadow stack for the signal
handler.
This instruction can fault if the user shadow stack is
invalid shadow stack memory. In that case, the kernel does
fixup.
Signed-off-by: Yu-cheng Yu <redacted>
And here you just write into regs->ax, but your "asm volatile"
doesn't
reserve that register. This looks wrong to me.
I think you probably want to add something like an explicit
`"+&a"(err)` output to the asm statements.
We require asm goto support these days. How about using
that? You
won't even need a special exception handler.
Maybe something like this? It looks simple now.
static inline int write_user_shstk_64(unsigned long addr, unsigned
long val)
{
asm_volatile_goto("wrussq %1, (%0)\n"
"jmp %l[ok]\n"
".section .fixup,\"ax\"n"
"jmp %l[fail]\n"
".previous\n"
:: "r" (addr), "r" (val)
:: ok, fail);
ok:
return 0;
fail:
return -1;
}
I think you can get rid of 'jmp %l[ok]' and the ok label and just fall
through. And you don't need an explicit jmp to fail -- just set the
_EX_HANDLER entry to land on the fail label.
On Thu, Aug 30, 2018 at 07:38:40AM -0700, Yu-cheng Yu wrote:
The previous version of CET patches can be found in the following
link.
https://lkml.org/lkml/2018/7/10/1031
Summary of changes from v2:
Move Shadow Stack page fault handling logic to arch/x86.
Update can_follow_write_pte/pmd; move logic to arch/x86.
Fix problems in WRUSS in-line assembly.
Fix issues in ELF parser.
Split out IBT/PTRACE patches to a second set.
Other small fixes.
Quick question -- is there a simulator or some other way you've
been testing this? Just curious, if it's possible to run these
patches or just a review and internal hardware/simulator where
they are run and posted
Balbir Singh.
@@ -0,0 +1,252 @@+=========================================+Control Flow Enforcement Technology (CET)+=========================================++[1] Overview+============++Control Flow Enforcement Technology (CET) provides protection against+return/jump-oriented programing (ROP) attacks. It can be implemented
programming
+to protect both the kernel and applications. In the first phase,
+only the user-mode protection is implemented for the 64-bit kernel.
+Thirty-two bit applications are supported under the compatibility
+mode.
On Sun, 2018-09-02 at 18:13 +1000, Balbir Singh wrote:
On Thu, Aug 30, 2018 at 07:38:40AM -0700, Yu-cheng Yu wrote:
quoted
The previous version of CET patches can be found in the following
link.
https://lkml.org/lkml/2018/7/10/1031
Summary of changes from v2:
Move Shadow Stack page fault handling logic to arch/x86.
Update can_follow_write_pte/pmd; move logic to arch/x86.
Fix problems in WRUSS in-line assembly.
Fix issues in ELF parser.
Split out IBT/PTRACE patches to a second set.
Other small fixes.
Quick question -- is there a simulator or some other way you've
been testing this? Just curious, if it's possible to run these
patches or just a review and internal hardware/simulator where
they are run and posted
Balbir Singh.
On Fri, 2018-08-31 at 18:29 +0200, Peter Zijlstra wrote:
On Fri, Aug 31, 2018 at 08:58:39AM -0700, Dave Hansen wrote:
quoted
On 08/31/2018 08:48 AM, Yu-cheng Yu wrote:
quoted
To trigger a race in ptep_set_wrprotect(), we need to fork from one of
three pthread siblings.
Or do we measure only how much this affects fork?
If there is no racing, the effect should be minimal.
We don't need a race.
I think the cmpxchg will be slower, even without a race, than the code
that was there before. The cmpxchg is a simple, straightforward
solution, but we're putting it in place of a plain memory write, which
is suboptimal.
Note quite, the clear_bit() is LOCK prefixed.
With the updated ptep_set_wrprotect() below, I did MADV_WILLNEED to a shadow
stack of 8 MB, then 10,000 fork()'s, but could not prove it is more or less
efficient than the other. So can we say this is probably fine in terms of
efficiency?
Yu-cheng
mm_struct *mm,
static inline void ptep_set_wrprotect(struct mm_struct *mm,
unsigned long addr, pte_t *ptep)
{
+#ifdef CONFIG_X86_INTEL_SHADOW_STACK_USER
+ pte_t new_pte, pte = READ_ONCE(*ptep);
+
+ /*
+ * Some processors can start a write, but end up
+ * seeing a read-only PTE by the time they get
+ * to the Dirty bit. In this case, they will
+ * set the Dirty bit, leaving a read-only, Dirty
+ * PTE which looks like a Shadow Stack PTE.
+ *
+ * However, this behavior has been improved and
+ * will not occur on processors supporting
+ * Shadow Stacks. Without this guarantee, a
+ * transition to a non-present PTE and flush the
+ * TLB would be needed.
+ *
+ * When changing a writable PTE to read-only and
+ * if the PTE has _PAGE_DIRTY_HW set, we move
+ * that bit to _PAGE_DIRTY_SW so that the PTE is
+ * not a valid Shadow Stack PTE.
+ */
+ do {
+ new_pte = pte_wrprotect(pte);
+ new_pte.pte |= (new_pte.pte & _PAGE_DIRTY_HW) >>
+ _PAGE_BIT_DIRTY_HW << _PAGE_BIT_DIRTY_SW;
+ new_pte.pte &= ~_PAGE_DIRTY_HW;
+ } while (!try_cmpxchg(ptep, &pte, new_pte));
+#else
clear_bit(_PAGE_BIT_RW, (unsigned long *)&ptep->pte);
+#endif
}
From: Dave Hansen <dave.hansen@linux.intel.com> Date: 2018-09-14 20:46:19
On 09/14/2018 01:39 PM, Yu-cheng Yu wrote:
With the updated ptep_set_wrprotect() below, I did MADV_WILLNEED to a shadow
stack of 8 MB, then 10,000 fork()'s, but could not prove it is more or less
efficient than the other. So can we say this is probably fine in terms of
efficiency?
Well, the first fork() will do all the hard work. I don't think
subsequent fork()s will be affected.
Did you do something to ensure this code was being run?
I would guess that a loop like this:
for (i = 0; i < 10000; i++) {
mprotect(addr, len, PROT_READ);
mprotect(addr, len, PROT_READ|PROT_WRITE);
}
might show it better.
On Fri, 2018-08-31 at 15:16 -0700, Andy Lutomirski wrote:
On Fri, Aug 31, 2018 at 2:49 PM, Yu-cheng Yu [off-list ref] wrote:
quoted
On Thu, 2018-08-30 at 09:22 -0700, Yu-cheng Yu wrote:
quoted
On Thu, 2018-08-30 at 08:55 -0700, Andy Lutomirski wrote:
quoted
On Thu, Aug 30, 2018 at 8:39 AM, Jann Horn [off-list ref]
wrote:
quoted
On Thu, Aug 30, 2018 at 4:44 PM Yu-cheng Yu <yu-cheng.yu@intel.c
om
quoted
wrote:
WRUSS is a new kernel-mode instruction but writes directly
to user shadow stack memory. This is used to construct
a return address on the shadow stack for the signal
handler.
This instruction can fault if the user shadow stack is
invalid shadow stack memory. In that case, the kernel does
fixup.
Signed-off-by: Yu-cheng Yu <redacted>
And here you just write into regs->ax, but your "asm volatile"
doesn't
reserve that register. This looks wrong to me.
I think you probably want to add something like an explicit
`"+&a"(err)` output to the asm statements.
We require asm goto support these days. How about using
that? You
won't even need a special exception handler.
Maybe something like this? It looks simple now.
static inline int write_user_shstk_64(unsigned long addr, unsigned
long val)
{
asm_volatile_goto("wrussq %1, (%0)\n"
"jmp %l[ok]\n"
".section .fixup,\"ax\"n"
"jmp %l[fail]\n"
".previous\n"
:: "r" (addr), "r" (val)
:: ok, fail);
ok:
return 0;
fail:
return -1;
}
I think you can get rid of 'jmp %l[ok]' and the ok label and just fall
through. And you don't need an explicit jmp to fail -- just set the
_EX_HANDLER entry to land on the fail label.
Thanks! This now looks simple and much better.
Yu-cheng
+static inline int write_user_shstk_64(unsigned long addr, unsigned long val)
+{
+ asm_volatile_goto("1: wrussq %1, (%0)\n"
+ _ASM_EXTABLE(1b, %l[fail])
+ :: "r" (addr), "r" (val)
+ :: fail);
+ return 0;
+fail:
+ return -1;
+}
On Fri, 2018-09-14 at 13:46 -0700, Dave Hansen wrote:
On 09/14/2018 01:39 PM, Yu-cheng Yu wrote:
quoted
With the updated ptep_set_wrprotect() below, I did MADV_WILLNEED to a shadow
stack of 8 MB, then 10,000 fork()'s, but could not prove it is more or less
efficient than the other. So can we say this is probably fine in terms of
efficiency?
Well, the first fork() will do all the hard work. I don't think
subsequent fork()s will be affected.
Are you talking about a recent commit:
1b2de5d0 mm/cow: don't bother write protecting already write-protected pages
With that, subsequent fork()s will not do all the hard work.
However, I have not done that for shadow stack PTEs (do we want to do that?).
I think the additional benefit for shadow stack is small?
Did you do something to ensure this code was being run?
I would guess that a loop like this:
for (i = 0; i < 10000; i++) {
mprotect(addr, len, PROT_READ);
mprotect(addr, len, PROT_READ|PROT_WRITE);
}
might show it better.
Would mprotect() do copy_one_pte()? Otherwise it will not go through
ptep_set_wrprotect()?
From: Dave Hansen <dave.hansen@linux.intel.com> Date: 2018-09-14 21:33:43
On 09/14/2018 02:08 PM, Yu-cheng Yu wrote:
On Fri, 2018-09-14 at 13:46 -0700, Dave Hansen wrote:
quoted
On 09/14/2018 01:39 PM, Yu-cheng Yu wrote:
quoted
With the updated ptep_set_wrprotect() below, I did MADV_WILLNEED to a shadow
stack of 8 MB, then 10,000 fork()'s, but could not prove it is more or less
efficient than the other. So can we say this is probably fine in terms of
efficiency?
BTW, I wasn't particularly concerned about shadow stacks. Plain old
memory is affected by this change too. Right?
quoted
Well, the first fork() will do all the hard work. I don't think
subsequent fork()s will be affected.
Are you talking about a recent commit:
1b2de5d0 mm/cow: don't bother write protecting already write-protected pages
With that, subsequent fork()s will not do all the hard work.
However, I have not done that for shadow stack PTEs (do we want to do that?).
I think the additional benefit for shadow stack is small?
You're right. mprotect() doesn't use this path.
But, that reminds me, can you take a quick look at change_pte_range()
and double-check that it's not affected by this issue?