[PATCH 12/25] arm64: move the XOR code to lib/raid/
From: Christoph Hellwig <hch@lst.de>
Date: 2026-02-26 15:11:50
Also in:
linux-alpha, linux-arch, linux-btrfs, linux-crypto, linux-raid, linux-riscv, linux-s390, linux-um, lkml, loongarch, sparclinux
Subsystem:
arm64 port (aarch64 architecture), library code, the rest · Maintainers:
Catalin Marinas, Will Deacon, Andrew Morton, Linus Torvalds
Move the optimized XOR into lib/raid and include it it in the main
xor.ko instead of building a separate module for it.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
arch/arm64/include/asm/xor.h | 58 +------------------
arch/arm64/lib/Makefile | 6 --
lib/raid/xor/Makefile | 5 ++
lib/raid/xor/arm64/xor-neon-glue.c | 57 ++++++++++++++++++
.../lib => lib/raid/xor/arm64}/xor-neon.c | 21 ++-----
5 files changed, 69 insertions(+), 78 deletions(-)
create mode 100644 lib/raid/xor/arm64/xor-neon-glue.c
rename {arch/arm64/lib => lib/raid/xor/arm64}/xor-neon.c (95%)
diff --git a/arch/arm64/include/asm/xor.h b/arch/arm64/include/asm/xor.h
index bfa6122f55ce..81718f010761 100644
--- a/arch/arm64/include/asm/xor.h
+++ b/arch/arm64/include/asm/xor.h@@ -1,73 +1,21 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * arch/arm64/include/asm/xor.h - * * Authors: Jackie Liu <liuyun01@kylinos.cn> * Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd. */ -#include <linux/hardirq.h> #include <asm-generic/xor.h> -#include <asm/hwcap.h> #include <asm/simd.h> -#ifdef CONFIG_KERNEL_MODE_NEON - -extern struct xor_block_template const xor_block_inner_neon; - -static void -xor_neon_2(unsigned long bytes, unsigned long * __restrict p1, - const unsigned long * __restrict p2) -{ - scoped_ksimd() - xor_block_inner_neon.do_2(bytes, p1, p2); -} - -static void -xor_neon_3(unsigned long bytes, unsigned long * __restrict p1, - const unsigned long * __restrict p2, - const unsigned long * __restrict p3) -{ - scoped_ksimd() - xor_block_inner_neon.do_3(bytes, p1, p2, p3); -} - -static void -xor_neon_4(unsigned long bytes, unsigned long * __restrict p1, - const unsigned long * __restrict p2, - const unsigned long * __restrict p3, - const unsigned long * __restrict p4) -{ - scoped_ksimd() - xor_block_inner_neon.do_4(bytes, p1, p2, p3, p4); -} - -static void -xor_neon_5(unsigned long bytes, unsigned long * __restrict p1, - const unsigned long * __restrict p2, - const unsigned long * __restrict p3, - const unsigned long * __restrict p4, - const unsigned long * __restrict p5) -{ - scoped_ksimd() - xor_block_inner_neon.do_5(bytes, p1, p2, p3, p4, p5); -} - -static struct xor_block_template xor_block_arm64 = { - .name = "arm64_neon", - .do_2 = xor_neon_2, - .do_3 = xor_neon_3, - .do_4 = xor_neon_4, - .do_5 = xor_neon_5 -}; +extern struct xor_block_template xor_block_arm64; +void __init xor_neon_init(void); #define arch_xor_init arch_xor_init static __always_inline void __init arch_xor_init(void) { + xor_neon_init(); xor_register(&xor_block_8regs); xor_register(&xor_block_32regs); if (cpu_has_neon()) xor_register(&xor_block_arm64); } - -#endif /* ! CONFIG_KERNEL_MODE_NEON */
diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile
index 633e5223d944..448c917494f3 100644
--- a/arch/arm64/lib/Makefile
+++ b/arch/arm64/lib/Makefile@@ -5,12 +5,6 @@ lib-y := clear_user.o delay.o copy_from_user.o \ memset.o memcmp.o strcmp.o strncmp.o strlen.o \ strnlen.o strchr.o strrchr.o tishift.o -ifeq ($(CONFIG_KERNEL_MODE_NEON), y) -obj-$(CONFIG_XOR_BLOCKS) += xor-neon.o -CFLAGS_xor-neon.o += $(CC_FLAGS_FPU) -CFLAGS_REMOVE_xor-neon.o += $(CC_FLAGS_NO_FPU) -endif - lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
diff --git a/lib/raid/xor/Makefile b/lib/raid/xor/Makefile
index fb760edae54b..3c13851219e5 100644
--- a/lib/raid/xor/Makefile
+++ b/lib/raid/xor/Makefile@@ -12,8 +12,13 @@ xor-$(CONFIG_ALPHA) += alpha/xor.o xor-$(CONFIG_ARM) += arm/xor.o ifeq ($(CONFIG_ARM),y) xor-$(CONFIG_KERNEL_MODE_NEON) += arm/xor-neon.o arm/xor-neon-glue.o +else +xor-$(CONFIG_KERNEL_MODE_NEON) += arm64/xor-neon.o arm64/xor-neon-glue.o endif CFLAGS_arm/xor-neon.o += $(CC_FLAGS_FPU) CFLAGS_REMOVE_arm/xor-neon.o += $(CC_FLAGS_NO_FPU) + +CFLAGS_arm64/xor-neon.o += $(CC_FLAGS_FPU) +CFLAGS_REMOVE_arm64/xor-neon.o += $(CC_FLAGS_NO_FPU)
diff --git a/lib/raid/xor/arm64/xor-neon-glue.c b/lib/raid/xor/arm64/xor-neon-glue.c
new file mode 100644
index 000000000000..067a2095659a
--- /dev/null
+++ b/lib/raid/xor/arm64/xor-neon-glue.c@@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Authors: Jackie Liu <liuyun01@kylinos.cn> + * Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd. + */ + +#include <linux/raid/xor_impl.h> +#include <asm/simd.h> +#include <asm/xor.h> + +extern struct xor_block_template const xor_block_inner_neon; + +static void +xor_neon_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) +{ + scoped_ksimd() + xor_block_inner_neon.do_2(bytes, p1, p2); +} + +static void +xor_neon_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) +{ + scoped_ksimd() + xor_block_inner_neon.do_3(bytes, p1, p2, p3); +} + +static void +xor_neon_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) +{ + scoped_ksimd() + xor_block_inner_neon.do_4(bytes, p1, p2, p3, p4); +} + +static void +xor_neon_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) +{ + scoped_ksimd() + xor_block_inner_neon.do_5(bytes, p1, p2, p3, p4, p5); +} + +struct xor_block_template xor_block_arm64 = { + .name = "arm64_neon", + .do_2 = xor_neon_2, + .do_3 = xor_neon_3, + .do_4 = xor_neon_4, + .do_5 = xor_neon_5 +};
diff --git a/arch/arm64/lib/xor-neon.c b/lib/raid/xor/arm64/xor-neon.c
similarity index 95%
rename from arch/arm64/lib/xor-neon.c
rename to lib/raid/xor/arm64/xor-neon.c
index 8fffebfa17b2..8d2d185090db 100644
--- a/arch/arm64/lib/xor-neon.c
+++ b/lib/raid/xor/arm64/xor-neon.c@@ -1,14 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * arch/arm64/lib/xor-neon.c - * * Authors: Jackie Liu <liuyun01@kylinos.cn> * Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd. */ -#include <linux/raid/xor.h> -#include <linux/module.h> +#include <linux/raid/xor_impl.h> +#include <linux/cache.h> #include <asm/neon-intrinsics.h> +#include <asm/xor.h> static void xor_arm64_neon_2(unsigned long bytes, unsigned long * __restrict p1, const unsigned long * __restrict p2)
@@ -179,7 +178,6 @@ struct xor_block_template xor_block_inner_neon __ro_after_init = { .do_4 = xor_arm64_neon_4, .do_5 = xor_arm64_neon_5, }; -EXPORT_SYMBOL(xor_block_inner_neon); static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r) {
@@ -317,22 +315,11 @@ static void xor_arm64_eor3_5(unsigned long bytes, } while (--lines > 0); } -static int __init xor_neon_init(void) +void __init xor_neon_init(void) { if (cpu_have_named_feature(SHA3)) { xor_block_inner_neon.do_3 = xor_arm64_eor3_3; xor_block_inner_neon.do_4 = xor_arm64_eor3_4; xor_block_inner_neon.do_5 = xor_arm64_eor3_5; } - return 0; } -module_init(xor_neon_init); - -static void __exit xor_neon_exit(void) -{ -} -module_exit(xor_neon_exit); - -MODULE_AUTHOR("Jackie Liu <liuyun01@kylinos.cn>"); -MODULE_DESCRIPTION("ARMv8 XOR Extensions"); -MODULE_LICENSE("GPL");
--
2.47.3