[PATCH 5/7] mips: bpf: Add JIT workarounds for CPU errata
From: Johan Almbladh <johan.almbladh@anyfinetworks.com>
Date: 2021-10-05 16:54:36
Also in:
bpf, netdev
Subsystem:
bpf jit for mips (32-bit and 64-bit), bpf [general] (safe dynamic programs and tools), mips, the rest · Maintainers:
Johan Almbladh, Paul Burton, Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, Eduard Zingerman, Kumar Kartikeya Dwivedi, Thomas Bogendoerfer, Linus Torvalds
This patch adds workarounds for the following CPU errata to the MIPS eBPF JIT, if enabled in the kernel configuration. - R10000 ll/sc weak ordering - Loongson-3 ll/sc weak ordering - Loongson-2F jump hang The Loongson-2F nop errata is implemented in uasm, which the JIT uses, so no additional mitigations are needed for that. Signed-off-by: Johan Almbladh <johan.almbladh@anyfinetworks.com> --- arch/mips/net/bpf_jit_comp.c | 6 ++++-- arch/mips/net/bpf_jit_comp.h | 26 +++++++++++++++++++++++++- arch/mips/net/bpf_jit_comp64.c | 10 ++++++---- 3 files changed, 35 insertions(+), 7 deletions(-)
diff --git a/arch/mips/net/bpf_jit_comp.c b/arch/mips/net/bpf_jit_comp.c
index 7eb95fc57710..b17130d510d4 100644
--- a/arch/mips/net/bpf_jit_comp.c
+++ b/arch/mips/net/bpf_jit_comp.c@@ -404,6 +404,7 @@ void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op) /* Atomic read-modify-write (32-bit) */ void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code) { + LLSC_sync(ctx); emit(ctx, ll, MIPS_R_T9, off, dst); switch (code) { case BPF_ADD:
@@ -427,7 +428,7 @@ void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code) break; } emit(ctx, sc, MIPS_R_T8, off, dst); - emit(ctx, beqz, MIPS_R_T8, -16); + emit(ctx, LLSC_beqz, MIPS_R_T8, -16 - LLSC_offset); emit(ctx, nop); /* Delay slot */ if (code & BPF_FETCH) {
@@ -439,11 +440,12 @@ void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code) /* Atomic compare-and-exchange (32-bit) */ void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off) { + LLSC_sync(ctx); emit(ctx, ll, MIPS_R_T9, off, dst); emit(ctx, bne, MIPS_R_T9, res, 12); emit(ctx, move, MIPS_R_T8, src); /* Delay slot */ emit(ctx, sc, MIPS_R_T8, off, dst); - emit(ctx, beqz, MIPS_R_T8, -20); + emit(ctx, LLSC_beqz, MIPS_R_T8, -20 - LLSC_offset); emit(ctx, move, res, MIPS_R_T9); /* Delay slot */ clobber_reg(ctx, res); }
diff --git a/arch/mips/net/bpf_jit_comp.h b/arch/mips/net/bpf_jit_comp.h
index 44787cf377dd..6f3a7b07294b 100644
--- a/arch/mips/net/bpf_jit_comp.h
+++ b/arch/mips/net/bpf_jit_comp.h@@ -87,7 +87,7 @@ struct jit_context { }; /* Emit the instruction if the JIT memory space has been allocated */ -#define emit(ctx, func, ...) \ +#define __emit(ctx, func, ...) \ do { \ if ((ctx)->target != NULL) { \ u32 *p = &(ctx)->target[ctx->jit_index]; \
@@ -95,6 +95,30 @@ do { \ } \ (ctx)->jit_index++; \ } while (0) +#define emit(...) __emit(__VA_ARGS__) + +/* Workaround for R10000 ll/sc errata */ +#ifdef CONFIG_WAR_R10000 +#define LLSC_beqz beqzl +#else +#define LLSC_beqz beqz +#endif + +/* Workaround for Loongson-3 ll/sc errata */ +#ifdef CONFIG_CPU_LOONGSON3_WORKAROUNDS +#define LLSC_sync(ctx) emit(ctx, sync, 0) +#define LLSC_offset 4 +#else +#define LLSC_sync(ctx) +#define LLSC_offset 0 +#endif + +/* Workaround for Loongson-2F jump errata */ +#ifdef CONFIG_CPU_JUMP_WORKAROUNDS +#define JALR_MASK 0xffffffffcfffffffULL +#else +#define JALR_MASK (~0ULL) +#endif /* * Mark a BPF register as accessed, it needs to be
diff --git a/arch/mips/net/bpf_jit_comp64.c b/arch/mips/net/bpf_jit_comp64.c
index ca49d3ef7ff4..1f1f7b87f213 100644
--- a/arch/mips/net/bpf_jit_comp64.c
+++ b/arch/mips/net/bpf_jit_comp64.c@@ -375,6 +375,7 @@ static void emit_atomic_r64(struct jit_context *ctx, u8 t1 = MIPS_R_T6; u8 t2 = MIPS_R_T7; + LLSC_sync(ctx); emit(ctx, lld, t1, off, dst); switch (code) { case BPF_ADD:
@@ -398,7 +399,7 @@ static void emit_atomic_r64(struct jit_context *ctx, break; } emit(ctx, scd, t2, off, dst); - emit(ctx, beqz, t2, -16); + emit(ctx, LLSC_beqz, t2, -16 - LLSC_offset); emit(ctx, nop); /* Delay slot */ if (code & BPF_FETCH) {
@@ -414,12 +415,13 @@ static void emit_cmpxchg_r64(struct jit_context *ctx, u8 dst, u8 src, s16 off) u8 t1 = MIPS_R_T6; u8 t2 = MIPS_R_T7; + LLSC_sync(ctx); emit(ctx, lld, t1, off, dst); emit(ctx, bne, t1, r0, 12); emit(ctx, move, t2, src); /* Delay slot */ emit(ctx, scd, t2, off, dst); - emit(ctx, beqz, t2, -20); - emit(ctx, move, r0, t1); /* Delay slot */ + emit(ctx, LLSC_beqz, t2, -20 - LLSC_offset); + emit(ctx, move, r0, t1); /* Delay slot */ clobber_reg(ctx, r0); }
@@ -443,7 +445,7 @@ static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn) push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0); /* Emit function call */ - emit_mov_i64(ctx, tmp, addr); + emit_mov_i64(ctx, tmp, addr & JALR_MASK); emit(ctx, jalr, MIPS_R_RA, tmp); emit(ctx, nop); /* Delay slot */
--
2.30.2