[PATCH] arm64: Add copy_mc_to_kernel() that recovers from synchronous external aborts

@@ -11,6 +11,7 @@
 #define EX_TYPE_KACCESS_ERR_ZERO	3
 #define EX_TYPE_UACCESS_CPY		4
 #define EX_TYPE_LOAD_UNALIGNED_ZEROPAD	5
+#define EX_TYPE_KACCESS_MC		6
 
 /* Data fields for EX_TYPE_UACCESS_ERR_ZERO */
 #define EX_DATA_REG_ERR_SHIFT	0

@@ -77,6 +78,14 @@
 	__ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_UACCESS_CPY, \uaccess_is_write)
 	.endm
 
+/*
+ * Create an exception table entry for a kernel load in copy_mc_to_kernel(),
+ * which will branch to `fixup` only on a synchronous external abort.
+ */
+	.macro		_asm_extable_kaccess_mc, insn, fixup
+	__ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_KACCESS_MC, 0)
+	.endm
+
 #else /* __ASSEMBLER__ */
 
 #include <linux/stringify.h>

@@ -105,6 +105,7 @@
 #define ESR_ELx_VNCR		(UL(1) << ESR_ELx_VNCR_SHIFT)
 #define ESR_ELx_SET_SHIFT	(11)
 #define ESR_ELx_SET_MASK	(UL(3) << ESR_ELx_SET_SHIFT)
+#define ESR_ELx_SET_UC		(UL(2) << ESR_ELx_SET_SHIFT)
 #define ESR_ELx_FnV_SHIFT	(10)
 #define ESR_ELx_FnV		(UL(1) << ESR_ELx_FnV_SHIFT)
 #define ESR_ELx_EA_SHIFT	(9)

@@ -48,4 +48,5 @@ bool ex_handler_bpf(const struct exception_table_entry *ex,
 #endif /* !CONFIG_BPF_JIT */
 
 bool fixup_exception(struct pt_regs *regs, unsigned long esr);
+bool fixup_exception_mc(struct pt_regs *regs);
 #endif

@@ -474,6 +474,10 @@ extern long strncpy_from_user(char *dest, const char __user *src, long count);
 
 extern __must_check long strnlen_user(const char __user *str, long n);
 
+unsigned long __must_check
+copy_mc_to_kernel(void *to, const void *from, size_t size);
+#define copy_mc_to_kernel copy_mc_to_kernel
+
 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
 extern unsigned long __must_check __copy_user_flushcache(void *to, const void __user *from, unsigned long n);
 

@@ -11,6 +11,8 @@ lib-y		:= clear_user.o delay.o copy_from_user.o		\
 
 lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o
 
+obj-y		+= copy_mc.o memcpy_mc.o
+
 obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
 
 obj-$(CONFIG_ARM64_MTE) += mte.o

@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include <linux/export.h>
+#include <linux/instrumented.h>
+#include <linux/uaccess.h>
+
+unsigned long __memcpy_mc(void *to, const void *from, unsigned long n);
+
+/**
+ * copy_mc_to_kernel - copy memory, surviving uncorrectable source errors
+ * @to: destination address
+ * @from: source address
+ * @size: number of bytes to copy
+ *
+ * Return: 0 on success, otherwise the number of bytes not copied because a
+ * source load hit memory with an uncorrectable error.
+ */
+unsigned long __must_check copy_mc_to_kernel(void *to, const void *from, size_t size)
+{
+	unsigned long ret;
+
+	instrument_memcpy_before(to, from, size);
+	ret = __memcpy_mc(to, from, size);
+	instrument_memcpy_after(to, from, size, ret);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(copy_mc_to_kernel);

@@ -0,0 +1,45 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+#include <linux/linkage.h>
+#include <asm/asm-extable.h>
+#include <asm/assembler.h>
+
+/*
+ * Copy memory whose source may contain an uncorrectable error. A synchronous
+ * external abort on a source load stops the copy instead of killing the
+ * kernel. Faults of any other kind are handled as they are for memcpy().
+ *
+ * Parameters:
+ *	x0 - dest
+ *	x1 - src
+ *	x2 - n
+ * Returns:
+ *	x0 - bytes not copied
+ */
+SYM_FUNC_START(__memcpy_mc)
+	cmp	x2, #8
+	b.lo	.Lbytes
+.Lwords:
+1:	ldr	x3, [x1], #8
+	str	x3, [x0], #8
+	sub	x2, x2, #8
+	cmp	x2, #8
+	b.hs	.Lwords
+.Lbytes:
+	cbz	x2, .Ldone
+2:	ldrb	w3, [x1], #1
+	strb	w3, [x0], #1
+	subs	x2, x2, #1
+	b.ne	2b
+.Ldone:
+	mov	x0, #0
+	ret
+
+	/* The faulting load did not complete, so x2 still counts its bytes */
+.Lfault:
+	mov	x0, x2
+	ret
+SYM_FUNC_END(__memcpy_mc)
+
+	_asm_extable_kaccess_mc	1b, .Lfault
+	_asm_extable_kaccess_mc	2b, .Lfault

@@ -105,7 +105,22 @@ bool fixup_exception(struct pt_regs *regs, unsigned long esr)
 		return ex_handler_uaccess_cpy(ex, regs, esr);
 	case EX_TYPE_LOAD_UNALIGNED_ZEROPAD:
 		return ex_handler_load_unaligned_zeropad(ex, regs);
+	case EX_TYPE_KACCESS_MC:
+		/* Only a synchronous external abort is fixed up, by fixup_exception_mc() */
+		return false;
 	}
 
 	BUG();
 }
+
+bool fixup_exception_mc(struct pt_regs *regs)
+{
+	const struct exception_table_entry *ex;
+
+	ex = search_exception_tables(instruction_pointer(regs));
+	if (!ex || ex->type != EX_TYPE_KACCESS_MC)
+		return false;
+
+	regs->pc = get_ex_fixup(ex);
+	return true;
+}

@@ -874,6 +874,26 @@ static int do_bad(unsigned long far, unsigned long esr, struct pt_regs *regs)
 	return 1; /* "fault" */
 }
 
+/*
+ * A kernel data abort on memory with an uncorrectable error can be recovered
+ * by copy_mc_to_kernel(). Aborts on instruction fetches or translation table
+ * walks cannot, and neither can errors the CPU reports as uncontainable.
+ */
+static bool is_el1_recoverable_sea(unsigned long esr)
+{
+	if (!is_el1_data_abort(esr))
+		return false;
+
+	if ((esr & ESR_ELx_FSC) != ESR_ELx_FSC_EXTABT)
+		return false;
+
+	if (cpus_have_cap(ARM64_HAS_RAS_EXTN) &&
+	    (esr & ESR_ELx_SET_MASK) == ESR_ELx_SET_UC)
+		return false;
+
+	return true;
+}
+
 static int do_sea(unsigned long far, unsigned long esr, struct pt_regs *regs)
 {
 	const struct fault_info *inf;

@@ -889,6 +909,19 @@ static int do_sea(unsigned long far, unsigned long esr, struct pt_regs *regs)
 		return 0;
 	}
 
+	if (!user_mode(regs) && is_el1_recoverable_sea(esr) &&
+	    fixup_exception_mc(regs)) {
+		/*
+		 * copy_mc_to_kernel() read memory with an uncorrectable error.
+		 * It returns the bytes it could not copy and its caller handles
+		 * the failure, so the kernel does not need to die. Firmware-first
+		 * systems still get to record the error.
+		 */
+		apei_claim_sea(regs);
+		add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK);
+		return 0;
+	}
+
 	if (esr & ESR_ELx_FnV) {
 		siaddr = 0;
 	} else {