[RFC 10/12] arm64: Support TLBIP instructions
From: Kristina Martšenko <hidden>
Date: 2026-10-01 11:08:16
Also in:
kvmarm, linux-acpi, linux-efi
Subsystem:
arm64 port (aarch64 architecture), the rest · Maintainers:
Catalin Marinas, Will Deacon, Linus Torvalds
Modify the TLBI helper functions in tlbflush.h to issue TLBIP instructions instead of TLBI when requested. These will be used to pass a TLBI domain ID to TLBI operations. FEAT_TLBID introduces TLBIP instructions that behave like TLBI but take a 128-bit value as an argument (passed in two consecutive GPRs). TLBID uses these to pass a TLBI domain ID to *VA* and *IPA* type instructions. TLBID does not introduce non-shareable TLBIP instructions, so assert that those are not requested. Use local register variables to ensure that the two GPRs are consecutive. Alternatively we could use the %H operand modifier but LLVM doesn't support it. Check for assembler support of TLBIP instructions. TLBIP won't be used (nor assembled) when CONFIG_ARM64_TLBID=n. For now there should be no functional change, as no callers request TLBIP. A later patch will change this. Signed-off-by: Kristina Martšenko <redacted> --- arch/arm64/Kconfig | 9 +- arch/arm64/include/asm/tlbflush.h | 131 ++++++++++++++++++++++-------- 2 files changed, 103 insertions(+), 37 deletions(-)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 46701449fef2..a6525754492b 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig@@ -2329,9 +2329,16 @@ config ARM64_LSUI endmenu # "ARMv9.6 architectural feature" +config AS_HAS_TLBID + def_bool $(as-instr,.arch_extension tlbid) + # https://github.com/llvm/llvm-project/pull/178913 + depends on AS_IS_GNU || (AS_IS_LLVM && AS_VERSION >= 230000) + help + Supported by LLVM 23+ and binutils 2.46+. + config ARM64_TLBID def_bool y - depends on ACPI + depends on ACPI && AS_HAS_TLBID select ACPI_TLBI config ARM64_SVE
diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
index b51df8e0ce63..dac35568570f 100644
--- a/arch/arm64/include/asm/tlbflush.h
+++ b/arch/arm64/include/asm/tlbflush.h@@ -41,11 +41,26 @@ #define __tlbi(op, ...) __TLBI_N(op, ##__VA_ARGS__, 1, 0) +#define __tlbip(op, arg) do { \ + register u64 low asm ("x0") = (u64)arg; \ + register u64 high asm ("x1") = (u64)(arg >> 64); \ + \ + asm (ARM64_ASM_PREAMBLE ".arch_extension tlbid\n" \ + "tlbip " #op ", %0, %1\n" \ + : : "r" (low), "r" (high)); \ +} while (0) + #define __tlbi_user(op, arg) do { \ if (arm64_kernel_unmapped_at_el0()) \ __tlbi(op, (arg) | USER_ASID_FLAG); \ } while (0) +#define __tlbip_user(op, arg) do { \ + if (arm64_kernel_unmapped_at_el0()) { \ + __tlbip(op, (arg) | (u128)USER_ASID_FLAG); \ + } \ +} while (0) + #define TLBI_TLBID_MASK GENMASK_ULL(15, 0) /* This macro creates a properly formatted VA operand for the TLBI */
@@ -132,49 +147,75 @@ static inline void sme_dvmsync_batch(void) #define TLBI_TTL_UNKNOWN INT_MAX -typedef void (*tlbi_op)(u64 arg); +typedef void (*tlbi_op)(bool tlbip, u128 arg); -static __always_inline void vae1is(u64 arg) +static __always_inline void vae1is(bool tlbip, u128 arg) { - __tlbi(vae1is, arg); - __tlbi_user(vae1is, arg); + if (tlbip) { + __tlbip(vae1is, arg); + __tlbip_user(vae1is, arg); + } else { + __tlbi(vae1is, (u64)arg); + __tlbi_user(vae1is, (u64)arg); + } } -static __always_inline void vae2is(u64 arg) +static __always_inline void vae2is(bool tlbip, u128 arg) { - __tlbi(vae2is, arg); + if (tlbip) + __tlbip(vae2is, arg); + else + __tlbi(vae2is, (u64)arg); } -static __always_inline void vale1(u64 arg) +static __always_inline void vale1(bool tlbip, u128 arg) { - __tlbi(vale1, arg); - __tlbi_user(vale1, arg); + compiletime_assert(!tlbip, "Unsupported TLBIP operand"); + + __tlbi(vale1, (u64)arg); + __tlbi_user(vale1, (u64)arg); } -static __always_inline void vale1is(u64 arg) +static __always_inline void vale1is(bool tlbip, u128 arg) { - __tlbi(vale1is, arg); - __tlbi_user(vale1is, arg); + if (tlbip) { + __tlbip(vale1is, arg); + __tlbip_user(vale1is, arg); + } else { + __tlbi(vale1is, (u64)arg); + __tlbi_user(vale1is, (u64)arg); + } } -static __always_inline void vale2is(u64 arg) +static __always_inline void vale2is(bool tlbip, u128 arg) { - __tlbi(vale2is, arg); + if (tlbip) + __tlbip(vale2is, arg); + else + __tlbi(vale2is, (u64)arg); } -static __always_inline void vaale1is(u64 arg) +static __always_inline void vaale1is(bool tlbip, u128 arg) { - __tlbi(vaale1is, arg); + if (tlbip) + __tlbip(vaale1is, arg); + else + __tlbi(vaale1is, (u64)arg); } -static __always_inline void ipas2e1(u64 arg) +static __always_inline void ipas2e1(bool tlbip, u128 arg) { - __tlbi(ipas2e1, arg); + compiletime_assert(!tlbip, "Unsupported TLBIP operand"); + + __tlbi(ipas2e1, (u64)arg); } -static __always_inline void ipas2e1is(u64 arg) +static __always_inline void ipas2e1is(bool tlbip, u128 arg) { - __tlbi(ipas2e1is, arg); + if (tlbip) + __tlbi(ipas2e1is, arg); + else + __tlbi(ipas2e1is, (u64)arg); } static __always_inline void __tlbi_level_asid(tlbi_op op, u64 addr, u32 level,
@@ -188,7 +229,7 @@ static __always_inline void __tlbi_level_asid(tlbi_op op, u64 addr, u32 level, FIELD_MODIFY(TLBI_TTL_MASK, &arg, ttl); } - op(arg); + op(false, arg); } static inline void __tlbi_level(tlbi_op op, u64 addr, u32 level)
@@ -445,32 +486,50 @@ static inline void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) * operations can only span an even number of pages. We save this for last to * ensure 64KB start alignment is maintained for the LPA2 case. */ -static __always_inline void rvae1is(u64 arg) -{ - __tlbi(rvae1is, arg); - __tlbi_user(rvae1is, arg); +static __always_inline void rvae1is(bool tlbip, u128 arg) +{ + if (tlbip) { + __tlbip(rvae1is, arg); + __tlbip_user(rvae1is, arg); + } else { + __tlbi(rvae1is, (u64)arg); + __tlbi_user(rvae1is, (u64)arg); + } } -static __always_inline void rvale1(u64 arg) +static __always_inline void rvale1(bool tlbip, u128 arg) { - __tlbi(rvale1, arg); - __tlbi_user(rvale1, arg); + compiletime_assert(!tlbip, "Unsupported TLBIP operand"); + + __tlbi(rvale1, (u64)arg); + __tlbi_user(rvale1, (u64)arg); } -static __always_inline void rvale1is(u64 arg) +static __always_inline void rvale1is(bool tlbip, u128 arg) { - __tlbi(rvale1is, arg); - __tlbi_user(rvale1is, arg); + if (tlbip) { + __tlbip(rvale1is, arg); + __tlbip_user(rvale1is, arg); + } else { + __tlbi(rvale1is, (u64)arg); + __tlbi_user(rvale1is, (u64)arg); + } } -static __always_inline void rvaale1is(u64 arg) +static __always_inline void rvaale1is(bool tlbip, u128 arg) { - __tlbi(rvaale1is, arg); + if (tlbip) + __tlbip(rvaale1is, arg); + else + __tlbi(rvaale1is, (u64)arg); } -static __always_inline void ripas2e1is(u64 arg) +static __always_inline void ripas2e1is(bool tlbip, u128 arg) { - __tlbi(ripas2e1is, arg); + if (tlbip) + __tlbip(ripas2e1is, arg); + else + __tlbi(ripas2e1is, (u64)arg); } static __always_inline void __tlbi_range(tlbi_op op, u64 addr,
@@ -486,7 +545,7 @@ static __always_inline void __tlbi_range(tlbi_op op, u64 addr, arg |= FIELD_PREP(TLBIR_TG_MASK, get_trans_granule()); arg |= FIELD_PREP(TLBIR_ASID_MASK, asid); - op(arg); + op(false, arg); } static __always_inline void __flush_tlb_range_op(tlbi_op lop, tlbi_op rop,
--
2.43.0