[RFC] powerpc: e6500 optimised copy_to_user/copy_from_user

Subsystems: linux for powerpc (32-bit and 64-bit), the rest

2 messages, 2 authors, 2015-03-12 · open the first message on its own page

[RFC] powerpc: e6500 optimised copy_to_user/copy_from_user

From: Kim Phillips <hidden>
Date: 2015-03-12 23:06:36 

This mimics commit a66086b8197da8dc83b698642d5947ff850e708d "powerpc:
POWER7 optimised copy_to_user/copy_from_user using VMX", but for
the e6500, or BOOK3S_64.  Changes have been made for the smaller
cacheline size (64 bytes on e6500), and data cache block touch (dcbt)
instructions have been rewritten to prefetch 8 lines ahead, based
on preliminary benchmark results and perf -e L1-dcache-prefetches
and misses observations.

We see a gain of 5% in large netperf benchmarks between two T4240s,
both in terms of throughput, and latency.  The same netperf
lo(opback) test improves 27%.

Anton's microbenchmark results show a clear linear improvement path
with sizes 32KB and above, where, below that, the additional overhead
over the existing copyuser_64 implementation shows its head: e.g., 6%
for 1448 byte copies.  The observed transfer sizes under large,
netperf benchmarks show the TCP stack is invoking copies on the
order of a few 10's of KB, however. 1MB transfers are 30% better-off
on wall clock time.

RFC because of the following known issues:
- unsure if PPC_BOOK3E_64 vs. PPC_BOOK3S_64 build-time switch to
  re-target __copy_tofrom_user_vmx is appropriate (ppc64_defconfig
  builds fine, however)
- syscalls report deficits when folding vmx_unaligned_copy to a 64B
  cacheline (undone for this RFC)
- any consideration for the e5500?
- asm branch label re-enumeration
- ..I'm sure I've missed another couple of things, possibly
including how to fix lower-sized transfer performance

Signed-off-by: Kim Phillips <redacted>
---
applies to Linus' ToT today (commit 09d35919b06),
since Scott's tree seems a bit outdated.

 arch/powerpc/include/asm/cputable.h |   2 +-
 arch/powerpc/lib/Makefile           |   5 +-
 arch/powerpc/lib/copyuser_64.S      |   2 +-
 arch/powerpc/lib/copyuser_e6500.S   | 768 ++++++++++++++++++++++++++++++++++++
 arch/powerpc/lib/copyuser_power7.S  |   2 +-
 5 files changed, 775 insertions(+), 4 deletions(-)
 create mode 100644 arch/powerpc/lib/copyuser_e6500.S

diff --git a/arch/powerpc/include/asm/cputable.h b/arch/powerpc/include/asm/cputable.h
index 5cf5a6d..66ca94a 100644
--- a/arch/powerpc/include/asm/cputable.h
+++ b/arch/powerpc/include/asm/cputable.h
@@ -391,7 +391,7 @@ extern const char *powerpc_base_platform;
 	    CPU_FTR_L2CSR | CPU_FTR_LWSYNC | CPU_FTR_NOEXECUTE | \
 	    CPU_FTR_DBELL | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \
 	    CPU_FTR_DEBUG_LVL_EXC | CPU_FTR_EMB_HV | CPU_FTR_ALTIVEC_COMP | \
-	    CPU_FTR_CELL_TB_BUG | CPU_FTR_SMT)
+	    CPU_FTR_CELL_TB_BUG | CPU_FTR_SMT | CPU_FTR_VMX_COPY)
 #define CPU_FTRS_GENERIC_32	(CPU_FTR_COMMON | CPU_FTR_NODSISRALIGN)
 
 /* 64-bit CPUs */
diff --git a/arch/powerpc/lib/Makefile b/arch/powerpc/lib/Makefile
index 7902802..c85e05d 100644
--- a/arch/powerpc/lib/Makefile
+++ b/arch/powerpc/lib/Makefile
@@ -15,12 +15,15 @@ obj-y += string.o alloc.o crtsavres.o ppc_ksyms.o code-patching.o \
 obj-$(CONFIG_PPC32)	+= div64.o copy_32.o
 
 obj64-y	+= copypage_64.o copyuser_64.o usercopy_64.o mem_64.o hweight_64.o \
-	   copyuser_power7.o string_64.o copypage_power7.o memcpy_power7.o \
+	   string_64.o copypage_power7.o memcpy_power7.o \
 	   memcpy_64.o memcmp_64.o
 
 obj64-$(CONFIG_SMP)	+= locks.o
 obj64-$(CONFIG_ALTIVEC)	+= vmx-helper.o
 
+obj64-$(CONFIG_PPC_BOOK3S_64)	+= copyuser_power7.o
+obj64-$(CONFIG_PPC_BOOK3E_64)	+= copyuser_e6500.o
+
 ifeq ($(CONFIG_GENERIC_CSUM),)
 obj-y			+= checksum_$(CONFIG_WORD_SIZE).o
 obj-$(CONFIG_PPC64)	+= checksum_wrappers_64.o
diff --git a/arch/powerpc/lib/copyuser_64.S b/arch/powerpc/lib/copyuser_64.S
index f09899e..0cf9253 100644
--- a/arch/powerpc/lib/copyuser_64.S
+++ b/arch/powerpc/lib/copyuser_64.S
@@ -22,7 +22,7 @@ _GLOBAL_TOC(__copy_tofrom_user)
 BEGIN_FTR_SECTION
 	nop
 FTR_SECTION_ELSE
-	b	__copy_tofrom_user_power7
+	b	__copy_tofrom_user_vmx
 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY)
 _GLOBAL(__copy_tofrom_user_base)
 	/* first check for a whole page copy on a page boundary */
diff --git a/arch/powerpc/lib/copyuser_e6500.S b/arch/powerpc/lib/copyuser_e6500.S
new file mode 100644
index 0000000..efd4251
--- /dev/null
+++ b/arch/powerpc/lib/copyuser_e6500.S
@@ -0,0 +1,768 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Copyright Freescale Semiconductor, 2015
+ *
+ * based on code with:
+ * Copyright (C) IBM Corporation, 2011
+ *
+ * Author: Anton Blanchard <anton@au.ibm.com>
+ */
+#include <asm/ppc_asm.h>
+
+#ifdef __BIG_ENDIAN__
+#define LVS(VRT,RA,RB)		lvsl	VRT,RA,RB
+#define VPERM(VRT,VRA,VRB,VRC)	vperm	VRT,VRA,VRB,VRC
+#else
+#define LVS(VRT,RA,RB)		lvsr	VRT,RA,RB
+#define VPERM(VRT,VRA,VRB,VRC)	vperm	VRT,VRB,VRA,VRC
+#endif
+
+	.macro err1
+100:
+	.section __ex_table,"a"
+	.align 3
+	.llong 100b,.Ldo_err1
+	.previous
+	.endm
+
+	.macro err2
+200:
+	.section __ex_table,"a"
+	.align 3
+	.llong 200b,.Ldo_err2
+	.previous
+	.endm
+
+#ifdef CONFIG_ALTIVEC
+	.macro err3
+300:
+	.section __ex_table,"a"
+	.align 3
+	.llong 300b,.Ldo_err3
+	.previous
+	.endm
+
+	.macro err4
+400:
+	.section __ex_table,"a"
+	.align 3
+	.llong 400b,.Ldo_err4
+	.previous
+	.endm
+
+
+.Ldo_err4:
+.Ldo_err3:
+	bl	exit_vmx_usercopy
+	ld	r0,STACKFRAMESIZE+16(r1)
+	mtlr	r0
+	b	.Lexit
+#endif /* CONFIG_ALTIVEC */
+
+.Ldo_err2:
+	ld	r14,STK_REG(R15)(r1)
+	ld	r14,STK_REG(R14)(r1)
+.Lexit:
+	addi	r1,r1,STACKFRAMESIZE
+.Ldo_err1:
+	ld	r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
+	ld	r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
+	ld	r5,-STACKFRAMESIZE+STK_REG(R29)(r1)
+	b	__copy_tofrom_user_base
+
+
+_GLOBAL(__copy_tofrom_user_vmx)
+	dcbt	0,r4
+	dcbtst	0,r3
+#ifdef CONFIG_ALTIVEC
+	cmpldi	r5,64
+	cmpldi	cr1,r5,4096
+
+	std	r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
+	std	r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
+	std	r5,-STACKFRAMESIZE+STK_REG(R29)(r1)
+
+	blt	.Lshort_copy
+	bgt	cr1,.Lvmx_copy
+#else
+	cmpldi	r5,64
+
+	std	r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
+	std	r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
+	std	r5,-STACKFRAMESIZE+STK_REG(R29)(r1)
+
+	blt	.Lshort_copy
+#endif
+
+.Lnonvmx_copy:
+	/* Get the source 8B aligned */
+	neg	r6,r4
+	mtocrf	0x01,r6
+	clrldi	r6,r6,(64-3)
+
+	bf	cr7*4+3,1f
+err1;	lbz	r0,0(r4)
+	addi	r4,r4,1
+err1;	stb	r0,0(r3)
+	addi	r3,r3,1
+
+1:	bf	cr7*4+2,2f
+err1;	lhz	r0,0(r4)
+	addi	r4,r4,2
+err1;	sth	r0,0(r3)
+	addi	r3,r3,2
+
+2:	bf	cr7*4+1,3f
+err1;	lwz	r0,0(r4)
+	addi	r4,r4,4
+err1;	stw	r0,0(r3)
+	addi	r3,r3,4
+
+3:	sub	r5,r5,r6
+	cmpldi	r5,64
+	blt	5f
+
+	mflr	r0
+	stdu	r1,-STACKFRAMESIZE(r1)
+	std	r14,STK_REG(R14)(r1)
+	std	r15,STK_REG(R15)(r1)
+	std	r0,STACKFRAMESIZE+16(r1)
+
+#define LINES_AHEAD 8
+	/*
+	 * We prefetch both the source and destination using regular touch
+	 * instructions.
+	 */
+	clrrdi	r6,r4,6
+	clrrdi	r9,r3,6
+
+	srdi	r7,r5,6		/* length in cachelines, capped at LINES_AHEAD*/
+	cmpldi	r7,LINES_AHEAD
+	ble	111f		/* continue if cachelines <= LINES_AHEAD */
+	li	r7,LINES_AHEAD	/* otherwise, r7 <- LINES_AHEAD */
+111:	lis	r0,0
+
+	mtctr	r7
+222:
+	addi	r6,r6,64
+	addi	r9,r9,64
+
+	dcbt	r0,r6   /* addr from */
+	dcbtst	r0,r9   /* addr to */
+
+	bdnz	222b
+
+	li	r14,64*LINES_AHEAD
+
+	srdi	r15,r5,6	/* length in cachelines */
+	cmpldi	r15,LINES_AHEAD
+	ble	442f		/* don't prefetch if cachelines <= LINES_AHEAD */
+	subi	r15,r15,LINES_AHEAD	/* otherwise, r15 <- r15 - LINES_AHEAD */
+	mtctr	r15
+
+	/* Now do cacheline (64B) sized loads and stores WITH prefetches. */
+	.align	5
+4:
+err2;	ld	r0,0(r4)
+err2;	ld	r6,8(r4)
+err2;	ld	r7,16(r4)
+err2;	ld	r8,24(r4)
+err2;	ld	r9,32(r4)
+err2;	ld	r10,40(r4)
+err2;	ld	r11,48(r4)
+err2;	ld	r12,56(r4)
+
+	addi	r4,r4,64
+	dcbt	r14,r4
+err2;	std	r0,0(r3)
+err2;	std	r6,8(r3)
+err2;	std	r7,16(r3)
+err2;	std	r8,24(r3)
+err2;	std	r9,32(r3)
+err2;	std	r10,40(r3)
+err2;	std	r11,48(r3)
+err2;	std	r12,56(r3)
+
+	addi	r3,r3,64
+	dcbtst	r14,r3
+	bdnz	4b
+
+	srdi	r7,r5,6		/* length in cachelines */
+	subf	r15,r15,r7	/* r15 = r7 - r15 */
+442:
+	mtctr	r15
+
+	/* remainder cacheline (64B) sized loads and stores WITHOUT prefetches*/
+	.align	5
+444:
+err2;	ld	r0,0(r4)
+err2;	ld	r6,8(r4)
+err2;	ld	r7,16(r4)
+err2;	ld	r8,24(r4)
+err2;	ld	r9,32(r4)
+err2;	ld	r10,40(r4)
+err2;	ld	r11,48(r4)
+err2;	ld	r12,56(r4)
+
+	addi	r4,r4,64
+err2;	std	r0,0(r3)
+err2;	std	r6,8(r3)
+err2;	std	r7,16(r3)
+err2;	std	r8,24(r3)
+err2;	std	r9,32(r3)
+err2;	std	r10,40(r3)
+err2;	std	r11,48(r3)
+err2;	std	r12,56(r3)
+
+	addi	r3,r3,64
+	bdnz	444b
+
+	clrldi	r5,r5,(64-6)
+
+	ld	r14,STK_REG(R14)(r1)
+	ld	r15,STK_REG(R15)(r1)
+	addi	r1,r1,STACKFRAMESIZE
+
+	/* Up to 63B to go */
+.Lshort_copy:
+5:	srdi	r6,r5,4
+	mtocrf	0x01,r6
+
+7:	bf	cr7*4+2,8f
+err1;	ld	r0,0(r4)
+err1;	ld	r6,8(r4)
+err1;	ld	r7,16(r4)
+err1;	ld	r8,24(r4)
+	addi	r4,r4,32
+err1;	std	r0,0(r3)
+err1;	std	r6,8(r3)
+err1;	std	r7,16(r3)
+err1;	std	r8,24(r3)
+	addi	r3,r3,32
+
+	/* Up to 31B to go */
+8:	bf	cr7*4+3,9f
+err1;	ld	r0,0(r4)
+err1;	ld	r6,8(r4)
+	addi	r4,r4,16
+err1;	std	r0,0(r3)
+err1;	std	r6,8(r3)
+	addi	r3,r3,16
+
+9:	clrldi	r5,r5,(64-4)
+
+	/* Up to 15B to go */
+	mtocrf	0x01,r5
+	bf	cr7*4+0,12f
+err1;	lwz	r0,0(r4)	/* Less chance of a reject with word ops */
+err1;	lwz	r6,4(r4)
+	addi	r4,r4,8
+err1;	stw	r0,0(r3)
+err1;	stw	r6,4(r3)
+	addi	r3,r3,8
+
+12:	bf	cr7*4+1,13f
+err1;	lwz	r0,0(r4)
+	addi	r4,r4,4
+err1;	stw	r0,0(r3)
+	addi	r3,r3,4
+
+13:	bf	cr7*4+2,14f
+err1;	lhz	r0,0(r4)
+	addi	r4,r4,2
+err1;	sth	r0,0(r3)
+	addi	r3,r3,2
+
+14:	bf	cr7*4+3,15f
+err1;	lbz	r0,0(r4)
+err1;	stb	r0,0(r3)
+
+15:	li	r3,0
+	blr
+
+.Lunwind_stack_nonvmx_copy:
+	addi	r1,r1,STACKFRAMESIZE
+	b	.Lnonvmx_copy
+
+#ifdef CONFIG_ALTIVEC
+.Lvmx_copy:
+	mflr	r0
+	std	r0,16(r1)
+	stdu	r1,-STACKFRAMESIZE(r1)
+	bl	enter_vmx_usercopy
+	cmpwi	cr1,r3,0
+	ld	r0,STACKFRAMESIZE+16(r1)
+	ld	r3,STK_REG(R31)(r1)
+	ld	r4,STK_REG(R30)(r1)
+	ld	r5,STK_REG(R29)(r1)
+	mtlr	r0
+
+	/*
+	 * We prefetch both the source and destination using regular touch
+	 * instructions.
+	 */
+	clrrdi	r6,r4,6
+	clrrdi	r9,r3,6
+
+	srdi	r7,r5,6		/* length in cachelines, capped at LINES_AHEAD*/
+	cmpldi	r7,LINES_AHEAD
+	ble	333f
+	li	r7,LINES_AHEAD
+333:	lis	r0,0
+
+	mtctr	r7
+444:
+	addi	r6,r6,64
+	addi	r9,r9,64
+
+	dcbt	r0,r6   /* addr from */
+	dcbtst	r0,r9   /* addr to */
+
+	bdnz	444b
+
+	beq	cr1,.Lunwind_stack_nonvmx_copy
+
+	/*
+	 * If source and destination are not relatively aligned we use a
+	 * slower permute loop.
+	 */
+	xor	r6,r4,r3
+	rldicl.	r6,r6,0,(64-4)
+	bne	.Lvmx_unaligned_copy
+
+	/* Get the destination 16B aligned */
+	neg	r6,r3
+	mtocrf	0x01,r6
+	clrldi	r6,r6,(64-4)
+
+	bf	cr7*4+3,1f
+err3;	lbz	r0,0(r4)
+	addi	r4,r4,1
+err3;	stb	r0,0(r3)
+	addi	r3,r3,1
+
+1:	bf	cr7*4+2,2f
+err3;	lhz	r0,0(r4)
+	addi	r4,r4,2
+err3;	sth	r0,0(r3)
+	addi	r3,r3,2
+
+2:	bf	cr7*4+1,3f
+err3;	lwz	r0,0(r4)
+	addi	r4,r4,4
+err3;	stw	r0,0(r3)
+	addi	r3,r3,4
+
+3:	bf	cr7*4+0,4f
+err3;	ld	r0,0(r4)
+	addi	r4,r4,8
+err3;	std	r0,0(r3)
+	addi	r3,r3,8
+
+4:	sub	r5,r5,r6
+
+	/* Get the destination 64B aligned */
+	neg	r6,r3
+	srdi	r7,r6,4
+	mtocrf	0x01,r7
+	clrldi	r6,r6,(64-6)
+
+	li	r9,16
+	li	r10,32
+	li	r11,48
+
+	bf	cr7*4+3,5f
+err3;	lvx	vr1,r0,r4
+	addi	r4,r4,16
+err3;	stvx	vr1,r0,r3
+	addi	r3,r3,16
+
+5:	bf	cr7*4+2,7f
+err3;	lvx	vr1,r0,r4
+err3;	lvx	vr0,r4,r9
+	addi	r4,r4,32
+err3;	stvx	vr1,r0,r3
+err3;	stvx	vr0,r3,r9
+	addi	r3,r3,32
+
+7:	sub	r5,r5,r6	/* r5 <- r6 - r5 */
+	srdi	r6,r5,6
+
+	cmpldi	r6,LINES_AHEAD
+	ble	882f		/* branch if cachelines <= LINES_AHEAD */
+	subi	r6,r6,LINES_AHEAD	/* otherwise, r6 <- r6 - LINES_AHEAD */
+
+	li	r12,64*LINES_AHEAD
+
+	mtctr	r6
+
+	/*
+	 * Now do cacheline (64B) sized loads and stores WITH prefetches.
+	 * By this stage the cacheline stores are also cacheline aligned.
+	 */
+	.align	5
+8:
+err4;	lvx	vr7,r0,r4
+err4;	lvx	vr6,r4,r9
+err4;	lvx	vr5,r4,r10
+err4;	lvx	vr4,r4,r11
+
+	addi	r4,r4,64
+
+	dcbt	r12,r4
+
+err4;	stvx	vr7,r0,r3
+err4;	stvx	vr6,r3,r9
+err4;	stvx	vr5,r3,r10
+err4;	stvx	vr4,r3,r11
+
+	addi	r3,r3,64
+
+	dcbtst	r12,r3
+
+	bdnz	8b
+
+	srdi	r7,r5,6		/* length in cachelines */
+	subf	r6,r6,r7	/* r6 = r7 - r6 */
+882:
+	mtctr	r6
+	/* remainder cacheline (64B) sized loads and stores WITHOUT prefetches.  */
+	.align	5
+88:
+err4;	lvx	vr7,r0,r4
+err4;	lvx	vr6,r4,r9
+err4;	lvx	vr5,r4,r10
+err4;	lvx	vr4,r4,r11
+
+	addi	r4,r4,64
+
+err4;	stvx	vr7,r0,r3
+err4;	stvx	vr6,r3,r9
+err4;	stvx	vr5,r3,r10
+err4;	stvx	vr4,r3,r11
+
+	addi	r3,r3,64
+
+	bdnz	88b
+
+	/* Up to 63B to go */
+	clrldi	r5,r5,(64-6)
+	srdi	r6,r5,4
+	mtocrf	0x01,r6
+
+9:	bf	cr7*4+2,10f
+err3;	lvx	vr1,r0,r4
+err3;	lvx	vr0,r4,r9
+	addi	r4,r4,32
+err3;	stvx	vr1,r0,r3
+err3;	stvx	vr0,r3,r9
+	addi	r3,r3,32
+
+10:	bf	cr7*4+3,11f
+err3;	lvx	vr1,r0,r4
+	addi	r4,r4,16
+err3;	stvx	vr1,r0,r3
+	addi	r3,r3,16
+
+	/* Up to 15B to go */
+11:	clrldi	r5,r5,(64-4)
+	mtocrf	0x01,r5
+	bf	cr7*4+0,12f
+err3;	ld	r0,0(r4)
+	addi	r4,r4,8
+err3;	std	r0,0(r3)
+	addi	r3,r3,8
+
+12:	bf	cr7*4+1,13f
+err3;	lwz	r0,0(r4)
+	addi	r4,r4,4
+err3;	stw	r0,0(r3)
+	addi	r3,r3,4
+
+13:	bf	cr7*4+2,14f
+err3;	lhz	r0,0(r4)
+	addi	r4,r4,2
+err3;	sth	r0,0(r3)
+	addi	r3,r3,2
+
+14:	bf	cr7*4+3,15f
+err3;	lbz	r0,0(r4)
+err3;	stb	r0,0(r3)
+
+15:	addi	r1,r1,STACKFRAMESIZE
+	b	exit_vmx_usercopy	/* tail call optimise */
+
+.Lvmx_unaligned_copy:
+	/* Get the destination 16B aligned */
+	neg	r6,r3
+	mtocrf	0x01,r6
+	clrldi	r6,r6,(64-4)
+
+	bf	cr7*4+3,1f
+err3;	lbz	r0,0(r4)
+	addi	r4,r4,1
+err3;	stb	r0,0(r3)
+	addi	r3,r3,1
+
+1:	bf	cr7*4+2,2f
+err3;	lhz	r0,0(r4)
+	addi	r4,r4,2
+err3;	sth	r0,0(r3)
+	addi	r3,r3,2
+
+2:	bf	cr7*4+1,3f
+err3;	lwz	r0,0(r4)
+	addi	r4,r4,4
+err3;	stw	r0,0(r3)
+	addi	r3,r3,4
+
+3:	bf	cr7*4+0,4f
+err3;	lwz	r0,0(r4)	/* Less chance of a reject with word ops */
+err3;	lwz	r7,4(r4)
+	addi	r4,r4,8
+err3;	stw	r0,0(r3)
+err3;	stw	r7,4(r3)
+	addi	r3,r3,8
+
+4:	sub	r5,r5,r6
+
+	/* Get the destination 128B aligned */
+	neg	r6,r3
+	srdi	r7,r6,4
+	mtocrf	0x01,r7
+	clrldi	r6,r6,(64-7)
+
+	li	r9,16
+	li	r10,32
+	li	r11,48
+
+	LVS(vr16,0,r4)		/* Setup permute control vector */
+err3;	lvx	vr0,0,r4
+	addi	r4,r4,16
+
+	bf	cr7*4+3,5f
+err3;	lvx	vr1,r0,r4
+	VPERM(vr8,vr0,vr1,vr16)
+	addi	r4,r4,16
+err3;	stvx	vr8,r0,r3
+	addi	r3,r3,16
+	vor	vr0,vr1,vr1
+
+5:	bf	cr7*4+2,6f
+err3;	lvx	vr1,r0,r4
+	VPERM(vr8,vr0,vr1,vr16)
+err3;	lvx	vr0,r4,r9
+	VPERM(vr9,vr1,vr0,vr16)
+	addi	r4,r4,32
+err3;	stvx	vr8,r0,r3
+err3;	stvx	vr9,r3,r9
+	addi	r3,r3,32
+
+6:	bf	cr7*4+1,7f
+err3;	lvx	vr3,r0,r4
+	VPERM(vr8,vr0,vr3,vr16)
+err3;	lvx	vr2,r4,r9
+	VPERM(vr9,vr3,vr2,vr16)
+err3;	lvx	vr1,r4,r10
+	VPERM(vr10,vr2,vr1,vr16)
+err3;	lvx	vr0,r4,r11
+	VPERM(vr11,vr1,vr0,vr16)
+	addi	r4,r4,64
+err3;	stvx	vr8,r0,r3
+err3;	stvx	vr9,r3,r9
+err3;	stvx	vr10,r3,r10
+err3;	stvx	vr11,r3,r11
+	addi	r3,r3,64
+
+7:	sub	r5,r5,r6
+	srdi	r6,r5,7
+
+	cmpldi	r6,LINES_AHEAD
+	ble	982f		/* branch if cachelines <= LINES_AHEAD */
+	subi	r6,r6,LINES_AHEAD	/* otherwise, r6 <- r6 - LINES_AHEAD */
+
+	std	r14,STK_REG(R14)(r1)
+	std	r15,STK_REG(R15)(r1)
+	std	r16,STK_REG(R16)(r1)
+
+	li	r12,64
+	li	r14,80
+	li	r15,96
+	li	r16,112
+
+	li	r8,64*LINES_AHEAD
+
+	mtctr	r6
+
+	/*
+	 * Now do cacheline (128B) sized loads and stores WITH prefetches.
+	 * By this stage the cacheline stores are also cacheline aligned.
+	 */
+	.align	5
+8:
+err4;	lvx	vr7,r0,r4
+	VPERM(vr8,vr0,vr7,vr16)
+err4;	lvx	vr6,r4,r9
+	VPERM(vr9,vr7,vr6,vr16)
+err4;	lvx	vr5,r4,r10
+	VPERM(vr10,vr6,vr5,vr16)
+err4;	lvx	vr4,r4,r11
+	VPERM(vr11,vr5,vr4,vr16)
+err4;	lvx	vr3,r4,r12
+	VPERM(vr12,vr4,vr3,vr16)
+err4;	lvx	vr2,r4,r14
+	VPERM(vr13,vr3,vr2,vr16)
+err4;	lvx	vr1,r4,r15
+	VPERM(vr14,vr2,vr1,vr16)
+err4;	lvx	vr0,r4,r16
+	VPERM(vr15,vr1,vr0,vr16)
+
+	addi    r4,r4,64
+	dcbt	r8,r4
+	addi    r4,r4,64
+	dcbt	r8,r4
+
+err4;	stvx	vr8,r0,r3
+err4;	stvx	vr9,r3,r9
+err4;	stvx	vr10,r3,r10
+err4;	stvx	vr11,r3,r11
+err4;	stvx	vr12,r3,r12
+err4;	stvx	vr13,r3,r14
+err4;	stvx	vr14,r3,r15
+err4;	stvx	vr15,r3,r16
+ 
+	addi    r3,r3,64
+	dcbtst	r8,r3
+	addi    r3,r3,64
+	dcbtst	r8,r3
+
+	bdnz	8b
+
+	srdi	r7,r5,7		/* length in double-cachelines */
+	subf	r6,r6,r7	/* r6 = r7 - r6 */
+982:
+	mtctr	r6
+	/*
+	 * remainder double-cacheline (128B) sized loads and stores
+	 * WITHOUT prefetches.
+	 */
+	.align	5
+98:
+err4;	lvx	vr7,r0,r4
+	VPERM(vr8,vr0,vr7,vr16)
+err4;	lvx	vr6,r4,r9
+	VPERM(vr9,vr7,vr6,vr16)
+err4;	lvx	vr5,r4,r10
+	VPERM(vr10,vr6,vr5,vr16)
+err4;	lvx	vr4,r4,r11
+	VPERM(vr11,vr5,vr4,vr16)
+err4;	lvx	vr3,r4,r12
+	VPERM(vr12,vr4,vr3,vr16)
+err4;	lvx	vr2,r4,r14
+	VPERM(vr13,vr3,vr2,vr16)
+err4;	lvx	vr1,r4,r15
+	VPERM(vr14,vr2,vr1,vr16)
+err4;	lvx	vr0,r4,r16
+	VPERM(vr15,vr1,vr0,vr16)
+
+	addi    r4,r4,128
+
+err4;	stvx	vr8,r0,r3
+err4;	stvx	vr9,r3,r9
+err4;	stvx	vr10,r3,r10
+err4;	stvx	vr11,r3,r11
+err4;	stvx	vr12,r3,r12
+err4;	stvx	vr13,r3,r14
+err4;	stvx	vr14,r3,r15
+err4;	stvx	vr15,r3,r16
+ 
+	addi    r3,r3,128
+
+	bdnz	98b
+
+	ld	r14,STK_REG(R14)(r1)
+	ld	r15,STK_REG(R15)(r1)
+	ld	r16,STK_REG(R16)(r1)
+
+	/* Up to 127B to go */
+	clrldi	r5,r5,(64-7)
+	srdi	r6,r5,4
+	mtocrf	0x01,r6
+
+	bf      cr7*4+1,9f
+err3;	lvx     vr3,r0,r4
+	VPERM(vr8,vr0,vr3,vr16)
+err3;	lvx     vr2,r4,r9
+	VPERM(vr9,vr3,vr2,vr16)
+err3;	lvx     vr1,r4,r10
+	VPERM(vr10,vr2,vr1,vr16)
+err3;	lvx     vr0,r4,r11
+	VPERM(vr11,vr1,vr0,vr16)
+	addi    r4,r4,64
+err3;	stvx    vr8,r0,r3
+err3;	stvx    vr9,r3,r9
+err3;	stvx    vr10,r3,r10
+err3;	stvx    vr11,r3,r11
+	addi    r3,r3,64
+
+9:	bf	cr7*4+2,10f
+err3;	lvx	vr1,r0,r4
+	VPERM(vr8,vr0,vr1,vr16)
+err3;	lvx	vr0,r4,r9
+	VPERM(vr9,vr1,vr0,vr16)
+	addi	r4,r4,32
+err3;	stvx	vr8,r0,r3
+err3;	stvx	vr9,r3,r9
+	addi	r3,r3,32
+
+10:	bf	cr7*4+3,11f
+err3;	lvx	vr1,r0,r4
+	VPERM(vr8,vr0,vr1,vr16)
+	addi	r4,r4,16
+err3;	stvx	vr8,r0,r3
+	addi	r3,r3,16
+
+	/* Up to 15B to go */
+11:	clrldi	r5,r5,(64-4)
+	addi	r4,r4,-16	/* Unwind the +16 load offset */
+	mtocrf	0x01,r5
+	bf	cr7*4+0,12f
+err3;	lwz	r0,0(r4)	/* Less chance of a reject with word ops */
+err3;	lwz	r6,4(r4)
+	addi	r4,r4,8
+err3;	stw	r0,0(r3)
+err3;	stw	r6,4(r3)
+	addi	r3,r3,8
+
+12:	bf	cr7*4+1,13f
+err3;	lwz	r0,0(r4)
+	addi	r4,r4,4
+err3;	stw	r0,0(r3)
+	addi	r3,r3,4
+
+13:	bf	cr7*4+2,14f
+err3;	lhz	r0,0(r4)
+	addi	r4,r4,2
+err3;	sth	r0,0(r3)
+	addi	r3,r3,2
+
+14:	bf	cr7*4+3,15f
+err3;	lbz	r0,0(r4)
+err3;	stb	r0,0(r3)
+
+15:	addi	r1,r1,STACKFRAMESIZE
+	b	exit_vmx_usercopy	/* tail call optimise */
+#endif /* CONFIG_ALTIVEC */
diff --git a/arch/powerpc/lib/copyuser_power7.S b/arch/powerpc/lib/copyuser_power7.S
index 92ee840..258e6fc 100644
--- a/arch/powerpc/lib/copyuser_power7.S
+++ b/arch/powerpc/lib/copyuser_power7.S
@@ -91,7 +91,7 @@
 	b	__copy_tofrom_user_base
 
 
-_GLOBAL(__copy_tofrom_user_power7)
+_GLOBAL(__copy_tofrom_user_vmx)
 #ifdef CONFIG_ALTIVEC
 	cmpldi	r5,16
 	cmpldi	cr1,r5,4096
-- 
2.3.2

Re: [RFC] powerpc: e6500 optimised copy_to_user/copy_from_user

From: Michael Ellerman <mpe@ellerman.id.au>
Date: 2015-03-12 23:26:34 

On Thu, 2015-03-12 at 17:45 -0500, Kim Phillips wrote:
This mimics commit a66086b8197da8dc83b698642d5947ff850e708d "powerpc:
POWER7 optimised copy_to_user/copy_from_user using VMX", but for
the e6500, or BOOK3S_64.  Changes have been made for the smaller
cacheline size (64 bytes on e6500), and data cache block touch (dcbt)
instructions have been rewritten to prefetch 8 lines ahead, based
on preliminary benchmark results and perf -e L1-dcache-prefetches
and misses observations.

We see a gain of 5% in large netperf benchmarks between two T4240s,
both in terms of throughput, and latency.  The same netperf
lo(opback) test improves 27%.

Anton's microbenchmark results show a clear linear improvement path
with sizes 32KB and above, where, below that, the additional overhead
over the existing copyuser_64 implementation shows its head: e.g., 6%
for 1448 byte copies.  The observed transfer sizes under large,
netperf benchmarks show the TCP stack is invoking copies on the
order of a few 10's of KB, however. 1MB transfers are 30% better-off
on wall clock time.

RFC because of the following known issues:
- unsure if PPC_BOOK3E_64 vs. PPC_BOOK3S_64 build-time switch to
  re-target __copy_tofrom_user_vmx is appropriate (ppc64_defconfig
  builds fine, however)
That's fine, a combined kernel is not really on the horizon.

- syscalls report deficits when folding vmx_unaligned_copy to a 64B
  cacheline (undone for this RFC)
- any consideration for the e5500?
- asm branch label re-enumeration
- ..I'm sure I've missed another couple of things, possibly
including how to fix lower-sized transfer performance
Well the big issue for me is the code duplication. The diff between the
original and yours is not small, but it looks like doing a combined version
*should* be possible?

If you take out the 8 line prefetch changes it looks like it's just the
cacheline size that is the issue?

cheers
Keyboard shortcuts
hback out one level
jnext message in thread
kprevious message in thread
ldrill in
Escclose help / fold thread tree
?toggle this help