diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index 3fe516b32577..facade819abb 100644
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -494,13 +494,30 @@ static void do_bad_area(unsigned long far, unsigned long esr,
}
}
+/*
+ * Note: not valid for EL1 DC IVAC, but we never use that such that it
+ * should fault. EL0 cannot issue DC IVAC (undef).
+ */
+static bool is_write_abort(unsigned long esr)
+{
+ return (esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM);
+}
+
+static bool is_gcs_fault(unsigned long esr)
+{
+ if (!esr_is_data_abort(esr))
+ return false;
+
+ return ESR_ELx_ISS2(esr) & ESR_ELx_GCS;
+}
+
#define VM_FAULT_BADMAP ((__force vm_fault_t)0x010000)
#define VM_FAULT_BADACCESS ((__force vm_fault_t)0x020000)
static vm_fault_t __do_page_fault(struct mm_struct *mm,
struct vm_area_struct *vma, unsigned long addr,
unsigned int mm_flags, unsigned long vm_flags,
- struct pt_regs *regs)
+ unsigned long esr, struct pt_regs *regs)
{
/*
* Ok, we have a good vm_area for this memory access, so we can handle@@ -510,6 +527,26 @@ static vm_fault_t __do_page_fault(struct mm_struct *mm,
*/
if (!(vma->vm_flags & vm_flags))
return VM_FAULT_BADACCESS;
+
+ if (vma->vm_flags & VM_SHADOW_STACK) {
+ /*
+ * Writes to a GCS must either be generated by a GCS
+ * operation or be from EL1.
+ */
+ if (is_write_abort(esr) &&
+ !(is_gcs_fault(esr) || is_el1_data_abort(esr)))
+ return VM_FAULT_BADACCESS;
+ } else {
+ /*
+ * GCS faults should never happen for pages that are
+ * not part of a GCS and the operation being attempted
+ * can never succeed.
+ */
+ if (is_gcs_fault(esr))
+ return VM_FAULT_BADACCESS;
+ }
+
+
return handle_mm_fault(vma, addr, mm_flags, regs);
}
@@ -518,13 +555,15 @@ static bool is_el0_instruction_abort(unsigned long esr)
return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW;
}
-/*
- * Note: not valid for EL1 DC IVAC, but we never use that such that it
- * should fault. EL0 cannot issue DC IVAC (undef).
- */
-static bool is_write_abort(unsigned long esr)
+static bool is_invalid_el0_gcs_access(struct vm_area_struct *vma, u64 esr)
{
- return (esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM);
+ if (!system_supports_gcs())
+ return false;
+ if (likely(!(vma->vm_flags & VM_SHADOW_STACK)))
+ return false;
+ if (is_gcs_fault(esr))
+ return false;
+ return is_write_abort(esr);
}
static int __kprobes do_page_fault(unsigned long far, unsigned long esr,@@ -573,6 +612,13 @@ static int __kprobes do_page_fault(unsigned long far, unsigned long esr,
/* If EPAN is absent then exec implies read */
if (!cpus_have_const_cap(ARM64_HAS_EPAN))
vm_flags |= VM_EXEC;
+ /*
+ * Upgrade read faults to write faults, GCS reads must
+ * occur on a page marked as GCS so we need to trigger
+ * copy on write always.
+ */
+ if (is_gcs_fault(esr))
+ mm_flags |= FAULT_FLAG_WRITE;
}
if (is_ttbr0_addr(addr) && is_el1_permission_fault(addr, esr, regs)) {@@ -595,6 +641,19 @@ static int __kprobes do_page_fault(unsigned long far, unsigned long esr,
if (!vma)
goto lock_mmap;
+ /*
+ * We get legitimate write faults for GCS pages from GCS
+ * operations and from EL1 writes to EL0 pages but just plain
+ * EL0 writes are invalid. Specifically check for this since
+ * as a result of upgrading read faults to write faults for
+ * CoW the mm core isn't able to distinguish these invalid
+ * writes.
+ */
+ if (is_invalid_el0_gcs_access(vma, esr)) {
+ vma_end_read(vma);
+ goto lock_mmap;
+ }
+
if (!(vma->vm_flags & vm_flags)) {
vma_end_read(vma);
goto lock_mmap;@@ -624,7 +683,7 @@ static int __kprobes do_page_fault(unsigned long far, unsigned long esr,
goto done;
}
- fault = __do_page_fault(mm, vma, addr, mm_flags, vm_flags, regs);
+ fault = __do_page_fault(mm, vma, addr, mm_flags, vm_flags, esr, regs);
/* Quick path to respond to signals */
if (fault_signal_pending(fault, regs)) {
--
2.30.2