[PATCHv2 0/4] ARM64: Improve copy_page and copy_template for ThunderX

@@ -11,6 +11,7 @@
 
 #include <asm/hwcap.h>
 #include <asm/sysreg.h>
+#include <asm/cputype.h>
 
 /*
  * In the arm64 world (as in the ARM world), elf_hwcap is used both internally

@@ -78,14 +79,14 @@ struct arm64_cpu_capabilities {
 	u16 capability;
 	bool (*matches)(const struct arm64_cpu_capabilities *);
 	void (*enable)(void *);		/* Called on all active CPUs */
+	u32 sys_reg;
 	union {
-		struct {	/* To be used for erratum handling only */
+		struct {	/* MIDR matching used if sys_reg is SYS_MIDR_EL1. */
 			u32 midr_model;
 			u32 midr_range_min, midr_range_max;
 		};
 
 		struct {	/* Feature register checking */
-			u32 sys_reg;
 			int field_pos;
 			int min_field_value;
 			int hwcap_type;

@@ -94,6 +95,27 @@ struct arm64_cpu_capabilities {
 	};
 };
 
+static bool __maybe_unused
+is_affected_midr_range(const struct arm64_cpu_capabilities *entry)
+{
+		u32 midr = read_cpuid_id();
+
+		if ((midr & CPU_MODEL_MASK) != entry->midr_model)
+				return false;
+
+		midr &= MIDR_REVISION_MASK | MIDR_VARIANT_MASK;
+
+		return (midr >= entry->midr_range_min && midr <= entry->midr_range_max);
+}
+
+#define MIDR_RANGE(model, min, max) \
+		.matches = is_affected_midr_range, \
+		.sys_reg = SYS_MIDR_EL1, \
+		.midr_model = model, \
+		.midr_range_min = min, \
+		.midr_range_max = max
+
+
 extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
 
 static inline bool cpu_have_feature(unsigned int num)

@@ -75,6 +75,13 @@
 
 #define CAVIUM_CPU_PART_THUNDERX	0x0A1
 
+#define MIDR_CORTEX_A53 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53)
+#define MIDR_CORTEX_A57 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57)
+#define MIDR_THUNDERX   MIDR_CPU_PART(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)
+
+#define CPU_MODEL_MASK (MIDR_IMPLEMENTOR_MASK | MIDR_PARTNUM_MASK | \
+                                                MIDR_ARCHITECTURE_MASK)
+
 #ifndef __ASSEMBLY__
 
 /*

@@ -21,32 +21,6 @@
 #include <asm/cputype.h>
 #include <asm/cpufeature.h>
 
-#define MIDR_CORTEX_A53 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53)
-#define MIDR_CORTEX_A57 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57)
-#define MIDR_THUNDERX	MIDR_CPU_PART(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)
-
-#define CPU_MODEL_MASK (MIDR_IMPLEMENTOR_MASK | MIDR_PARTNUM_MASK | \
-			MIDR_ARCHITECTURE_MASK)
-
-static bool __maybe_unused
-is_affected_midr_range(const struct arm64_cpu_capabilities *entry)
-{
-	u32 midr = read_cpuid_id();
-
-	if ((midr & CPU_MODEL_MASK) != entry->midr_model)
-		return false;
-
-	midr &= MIDR_REVISION_MASK | MIDR_VARIANT_MASK;
-
-	return (midr >= entry->midr_range_min && midr <= entry->midr_range_max);
-}
-
-#define MIDR_RANGE(model, min, max) \
-	.matches = is_affected_midr_range, \
-	.midr_model = model, \
-	.midr_range_min = min, \
-	.midr_range_max = max
-
 const struct arm64_cpu_capabilities arm64_errata[] = {
 #if	defined(CONFIG_ARM64_ERRATUM_826319) || \
 	defined(CONFIG_ARM64_ERRATUM_827319) || \

@@ -869,8 +869,27 @@ void verify_local_cpu_capabilities(void)
 
 	caps = arm64_features;
 	for (i = 0; caps[i].desc; i++) {
-		if (!cpus_have_cap(caps[i].capability) || !caps[i].sys_reg)
+		/*
+		 * If the feature is not enable already, then don't try to
+		 * enable.
+		 */
+		if (!cpus_have_cap(caps[i].capability))
+			continue;
+		if (!caps[i].sys_reg)
+			continue;
+		/* Handle MIDR matching seperately. */
+		if (caps[i].sys_reg == SYS_MIDR_EL1) {
+			/*
+			 * If the new CPU is a different MIDR it means the
+			 * feature is missing, we cannot proceed further,
+			 * park the cpu.
+			 */
+			if (!is_affected_midr_range (&caps[i]))
+				fail_incapable_cpu("arm64_features", &caps[i]);
+			if (caps[i].enable)
+				caps[i].enable(NULL);
 			continue;
+		}
 		/*
 		 * If the new CPU misses an advertised feature, we cannot proceed
 		 * further, park the cpu.

@@ -18,6 +18,8 @@
 #include <linux/const.h>
 #include <asm/assembler.h>
 #include <asm/page.h>
+#include <asm/cpufeature.h>
+#include <asm/alternative.h>
 
 
 /*

@@ -28,6 +30,15 @@
  *	x1 - src
  */
 ENTRY(copy_page)
+alternative_if_not ARM64_NEEDS_PREFETCH_128
+	nop
+	nop
+alternative_else
+	# Prefetch two cache lines ahead.
+	prfm    pldl1strm, [x1, #128]
+	prfm    pldl1strm, [x1, #256]
+alternative_endif
+
 	ldp	x2, x3, [x1]
 	ldp	x4, x5, [x1, #16]
 	ldp	x6, x7, [x1, #32]

@@ -42,6 +53,12 @@ ENTRY(copy_page)
 1:
 	subs	x18, x18, #128
 
+alternative_if_not ARM64_NEEDS_PREFETCH_128
+	nop
+alternative_else
+	prfm    pldl1strm, [x1, #384]
+alternative_endif
+
 	stnp	x2, x3, [x0]
 	ldp	x2, x3, [x1]
 	stnp	x4, x5, [x0, #16]

@@ -163,12 +163,24 @@ D_h	.req	x14
 	*/
 	.p2align	L1_CACHE_SHIFT
 .Lcpy_body_large:
+alternative_if_not ARM64_NEEDS_PREFETCH_128
+	nop
+	nop
+alternative_else
+	prfm	pldl1strm, [src, #128]
+	prfm	pldl1strm, [src, #256]
+alternative_endif
 	/* pre-get 64 bytes data. */
 	ldp1	A_l, A_h, src, #16
 	ldp1	B_l, B_h, src, #16
 	ldp1	C_l, C_h, src, #16
 	ldp1	D_l, D_h, src, #16
 1:
+alternative_if_not ARM64_NEEDS_PREFETCH_128
+	nop
+alternative_else
+	prfm	pldl1strm, [src, #384]
+alternative_endif
 	/*
 	* interlace the load of next 64 bytes data block with store of the last
 	* loaded 64 bytes data.

@@ -25,6 +25,8 @@
 #include <linux/linkage.h>
 #include <asm/assembler.h>
 #include <asm/cache.h>
+#include <asm/alternative.h>
+#include <asm/cpufeature.h>
 
 /*
  * Copy a buffer from src to dest (alignment handled by the hardware)

@@ -32,8 +32,9 @@
 #define ARM64_WORKAROUND_CAVIUM_23154		6
 #define ARM64_WORKAROUND_834220			7
 #define ARM64_WORKAROUND_CAVIUM_27456		8
+#define ARM64_NEEDS_PREFETCH_128		9
 
-#define ARM64_NCAPS				9
+#define ARM64_NCAPS				10
 
 #ifndef __ASSEMBLY__
 

@@ -651,6 +651,12 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.min_field_value = 2,
 	},
 #endif /* CONFIG_AS_LSE && CONFIG_ARM64_LSE_ATOMICS */
+	{
+		.desc = "Needs prefetching (128 byte cache line) for copying: ThunderX T88 pass 1.x and 2.x",
+		.capability = ARM64_NEEDS_PREFETCH_128,
+		MIDR_RANGE(MIDR_THUNDERX, 0,
+			   (1 << MIDR_VARIANT_SHIFT) | MIDR_REVISION_MASK),
+	},
 	{},
 };
 

@@ -19,6 +19,7 @@
 #include <asm/assembler.h>
 #include <asm/page.h>
 
+
 /*
  * Copy a page from src to dest (both are page aligned)
  *

@@ -27,20 +28,50 @@
  *	x1 - src
  */
 ENTRY(copy_page)
-	/* Assume cache line size is 64 bytes. */
-	prfm	pldl1strm, [x1, #64]
-1:	ldp	x2, x3, [x1]
+	ldp	x2, x3, [x1]
+	ldp	x4, x5, [x1, #16]
+	ldp	x6, x7, [x1, #32]
+	ldp	x8, x9, [x1, #48]
+	ldp	x10, x11, [x1, #64]
+	ldp	x12, x13, [x1, #80]
+	ldp	x14, x15, [x1, #96]
+	ldp	x16, x17, [x1, #112]
+
+	mov	x18, #(PAGE_SIZE - 128)
+	add	x1, x1, #128
+1:
+	subs	x18, x18, #128
+
+	stnp	x2, x3, [x0]
+	ldp	x2, x3, [x1]
+	stnp	x4, x5, [x0, #16]
 	ldp	x4, x5, [x1, #16]
+	stnp	x6, x7, [x0, #32]
 	ldp	x6, x7, [x1, #32]
+	stnp	x8, x9, [x0, #48]
 	ldp	x8, x9, [x1, #48]
-	add	x1, x1, #64
-	prfm	pldl1strm, [x1, #64]
+	stnp	x10, x11, [x0, #64]
+	ldp	x10, x11, [x1, #64]
+	stnp	x12, x13, [x0, #80]
+	ldp	x12, x13, [x1, #80]
+	stnp	x14, x15, [x0, #96]
+	ldp	x14, x15, [x1, #96]
+	stnp	x16, x17, [x0, #112]
+	ldp	x16, x17, [x1, #112]
+
+	add	x0, x0, #128
+	add	x1, x1, #128
+
+	b.gt	1b
+
 	stnp	x2, x3, [x0]
 	stnp	x4, x5, [x0, #16]
 	stnp	x6, x7, [x0, #32]
 	stnp	x8, x9, [x0, #48]
-	add	x0, x0, #64
-	tst	x1, #(PAGE_SIZE - 1)
-	b.ne	1b
+	stnp	x10, x11, [x0, #64]
+	stnp	x12, x13, [x0, #80]
+	stnp	x14, x15, [x0, #96]
+	stnp	x16, x17, [x0, #112]
+
 	ret
 ENDPROC(copy_page)