[PATCH v2 0/8] bpf powerpc: Add BPF_PROBE_MEM support in powerpc JIT compiler

---

* No changes in v2.


  arch/powerpc/net/bpf_jit.h | 3 +--
  1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/arch/powerpc/net/bpf_jit.h b/arch/powerpc/net/bpf_jit.h
index 99fad093f43e..d6267e93027a 100644
--- a/arch/powerpc/net/bpf_jit.h
+++ b/arch/powerpc/net/bpf_jit.h

@@ -116,8 +116,7 @@ static inline bool is_nearbranch(int offset)
  #define COND_LE		(CR0_GT | COND_CMP_FALSE)
  
  #define SEEN_FUNC	0x20000000 /* might call external helpers */
-#define SEEN_STACK	0x40000000 /* uses BPF stack */
-#define SEEN_TAILCALL	0x80000000 /* uses tail calls */
+#define SEEN_TAILCALL	0x40000000 /* uses tail calls */
  
  #define SEEN_VREG_MASK	0x1ff80000 /* Volatile registers r3-r12 */
  #define SEEN_NVREG_MASK	0x0003ffff /* Non volatile registers r14-r31 */

---

Changes in v2:
* New patch to introduce PPC_RAW_BRANCH() macro.


  arch/powerpc/include/asm/ppc-opcode.h | 2 ++
  arch/powerpc/net/bpf_jit.h            | 4 ++--
  2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h
index baea657bc868..f50213e2a3e0 100644
--- a/arch/powerpc/include/asm/ppc-opcode.h
+++ b/arch/powerpc/include/asm/ppc-opcode.h

@@ -566,6 +566,8 @@
  #define PPC_RAW_MTSPR(spr, d)		(0x7c0003a6 | ___PPC_RS(d) | __PPC_SPR(spr))
  #define PPC_RAW_EIEIO()			(0x7c0006ac)
  
+#define PPC_RAW_BRANCH(addr)		(PPC_INST_BRANCH | ((addr) & 0x03fffffc))
+
  /* Deal with instructions that older assemblers aren't aware of */
  #define	PPC_BCCTR_FLUSH		stringify_in_c(.long PPC_INST_BCCTR_FLUSH)
  #define	PPC_CP_ABORT		stringify_in_c(.long PPC_RAW_CP_ABORT)

diff --git a/arch/powerpc/net/bpf_jit.h b/arch/powerpc/net/bpf_jit.h
index 411c63d945c7..0c8f885b8f48 100644
--- a/arch/powerpc/net/bpf_jit.h
+++ b/arch/powerpc/net/bpf_jit.h

@@ -24,8 +24,8 @@
  #define EMIT(instr)		PLANT_INSTR(image, ctx->idx, instr)
  
  /* Long jump; (unconditional 'branch') */
-#define PPC_JMP(dest)		EMIT(PPC_INST_BRANCH |			      \
-				     (((dest) - (ctx->idx * 4)) & 0x03fffffc))
+#define PPC_JMP(dest)		EMIT(PPC_RAW_BRANCH((dest) - (ctx->idx * 4)))
+
  /* blr; (unconditional 'branch' with link) to absolute address */
  #define PPC_BL_ABS(dest)	EMIT(PPC_INST_BL |			      \
  				     (((dest) - (unsigned long)(image + ctx->idx)) & 0x03fffffc))

From: Ravi Bangoria <redacted>

BPF load instruction with BPF_PROBE_MEM mode can cause a fault
inside kernel. Append exception table for such instructions
within BPF program.

Unlike other archs which uses extable 'fixup' field to pass dest_reg
and nip, BPF exception table on PowerPC follows the generic PowerPC
exception table design, where it populates both fixup and extable
sections within BPF program. fixup section contains two instructions,
first instruction clears dest_reg and 2nd jumps to next instruction
in the BPF code. extable 'insn' field contains relative offset of
the instruction and 'fixup' field contains relative offset of the
fixup entry. Example layout of BPF program with extable present:

              +------------------+
              |                  |
              |                  |
    0x4020 -->| ld   r27,4(r3)   |
              |                  |
              |                  |
    0x40ac -->| lwz  r3,0(r4)    |
              |                  |
              |                  |
              |------------------|
    0x4280 -->| li  r27,0        |  \ fixup entry
              | b   0x4024       |  /
    0x4288 -->| li  r3,0         |
              | b   0x40b0       |
              |------------------|
    0x4290 -->| insn=0xfffffd90  |  \ extable entry
              | fixup=0xffffffec |  /
    0x4298 -->| insn=0xfffffe14  |
              | fixup=0xffffffec |
              +------------------+

    (Addresses shown here are chosen random, not real)

Signed-off-by: Ravi Bangoria <redacted>
Signed-off-by: Hari Bathini <hbathini@linux.ibm.com>
---

Changes in v2:
* Used JITing code after refactoring.
* Replaced 'xor reg,reg,reg' with 'li reg,0' where appropriate.
* Avoided unnecessary init during declaration.


  arch/powerpc/net/bpf_jit.h        |  5 ++-
  arch/powerpc/net/bpf_jit_comp.c   | 25 ++++++++++----
  arch/powerpc/net/bpf_jit_comp32.c |  2 +-
  arch/powerpc/net/bpf_jit_comp64.c | 57 ++++++++++++++++++++++++++++++-
  4 files changed, 80 insertions(+), 9 deletions(-)

diff --git a/arch/powerpc/net/bpf_jit.h b/arch/powerpc/net/bpf_jit.h
index 0c8f885b8f48..6357c71c26eb 100644
--- a/arch/powerpc/net/bpf_jit.h
+++ b/arch/powerpc/net/bpf_jit.h

@@ -141,8 +141,11 @@ struct codegen_context {
  	unsigned int idx;
  	unsigned int stack_size;
  	int b2p[ARRAY_SIZE(b2p)];
+	unsigned int exentry_idx;
  };
  
+#define BPF_FIXUP_LEN	8 /* Two instructions */
+
  static inline void bpf_flush_icache(void *start, void *end)
  {
  	smp_wmb();	/* smp write barrier */

@@ -166,7 +169,7 @@ static inline void bpf_clear_seen_register(struct codegen_context *ctx, int i)
  
  void bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, u64 func);
  int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx,
-		       u32 *addrs);
+		       u32 *addrs, int pass);
  void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx);
  void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx);
  void bpf_jit_realloc_regs(struct codegen_context *ctx);

diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c
index c5c9e8ad1de7..e92bd79d3bac 100644
--- a/arch/powerpc/net/bpf_jit_comp.c
+++ b/arch/powerpc/net/bpf_jit_comp.c

@@ -101,6 +101,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
  	struct bpf_prog *tmp_fp;
  	bool bpf_blinded = false;
  	bool extra_pass = false;
+	u32 extable_len;
+	u32 fixup_len;
  
  	if (!fp->jit_requested)
  		return org_fp;

@@ -131,7 +133,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
  		image = jit_data->image;
  		bpf_hdr = jit_data->header;
  		proglen = jit_data->proglen;
-		alloclen = proglen + FUNCTION_DESCR_SIZE;
  		extra_pass = true;
  		goto skip_init_ctx;
  	}

@@ -149,7 +150,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
  	cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
  
  	/* Scouting faux-generate pass 0 */
-	if (bpf_jit_build_body(fp, 0, &cgctx, addrs)) {
+	if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0)) {
  		/* We hit something illegal or unsupported. */
  		fp = org_fp;
  		goto out_addrs;

@@ -162,7 +163,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
  	 */
  	if (cgctx.seen & SEEN_TAILCALL) {
  		cgctx.idx = 0;
-		if (bpf_jit_build_body(fp, 0, &cgctx, addrs)) {
+		if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0)) {
  			fp = org_fp;
  			goto out_addrs;
  		}

@@ -177,8 +178,11 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
  	bpf_jit_build_prologue(0, &cgctx);
  	bpf_jit_build_epilogue(0, &cgctx);
  
+	fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN;
+	extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry);
+
  	proglen = cgctx.idx * 4;
-	alloclen = proglen + FUNCTION_DESCR_SIZE;
+	alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len;
  
  	bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4, bpf_jit_fill_ill_insns);
  	if (!bpf_hdr) {

@@ -186,6 +190,11 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
  		goto out_addrs;
  	}
  
+	if (extable_len) {
+		fp->aux->extable = (void *)image + FUNCTION_DESCR_SIZE +
+				   proglen + fixup_len;
+	}

+
  skip_init_ctx:
  	code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
  

@@ -210,7 +219,11 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
  		/* Now build the prologue, body code & epilogue for real. */
  		cgctx.idx = 0;
  		bpf_jit_build_prologue(code_base, &cgctx);
-		bpf_jit_build_body(fp, code_base, &cgctx, addrs);
+		if (bpf_jit_build_body(fp, code_base, &cgctx, addrs, pass)) {
+			bpf_jit_binary_free(bpf_hdr);
+			fp = org_fp;
+			goto out_addrs;
+		}
  		bpf_jit_build_epilogue(code_base, &cgctx);
  
  		if (bpf_jit_enable > 1)

@@ -234,7 +247,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
  
  	fp->bpf_func = (void *)image;
  	fp->jited = 1;
-	fp->jited_len = alloclen;
+	fp->jited_len = proglen + FUNCTION_DESCR_SIZE;
  
  	bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + (bpf_hdr->pages * PAGE_SIZE));
  	bpf_jit_binary_lock_ro(bpf_hdr);

diff --git a/arch/powerpc/net/bpf_jit_comp32.c b/arch/powerpc/net/bpf_jit_comp32.c
index c8ae14c316e3..94641b7be387 100644
--- a/arch/powerpc/net/bpf_jit_comp32.c
+++ b/arch/powerpc/net/bpf_jit_comp32.c

@@ -266,7 +266,7 @@ static void bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32
  
  /* Assemble the body code between the prologue & epilogue */
  int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx,
-		       u32 *addrs)
+		       u32 *addrs, int pass)
  {
  	const struct bpf_insn *insn = fp->insnsi;
  	int flen = fp->len;

diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c
index 78b28f25555c..2fc10995f243 100644
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c

@@ -270,9 +270,54 @@ static void bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32
  	/* out: */
  }
  
+/*
+ * The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling
+ * this function, as this only applies to BPF_PROBE_MEM, for now.
+ */
+static int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, int pass,
+				 struct codegen_context *ctx, int dst_reg)

+{
+	off_t offset;
+	unsigned long pc;
+	struct exception_table_entry *ex;
+	u32 *fixup;
+
+	/* Populate extable entries only in the last pass */
+	if (pass != 2)
+		return 0;
+
+	if (!fp->aux->extable ||
+	    WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries))
+		return -EINVAL;
+
+	pc = (unsigned long)&image[ctx->idx - 1];
+
+	fixup = (void *)fp->aux->extable -
+		(fp->aux->num_exentries * BPF_FIXUP_LEN) +
+		(ctx->exentry_idx * BPF_FIXUP_LEN);
+
+	fixup[0] = PPC_RAW_LI(dst_reg, 0);
+	fixup[1] = PPC_RAW_BRANCH((long)(pc + 4) - (long)&fixup[1]);
+
+	ex = &fp->aux->extable[ctx->exentry_idx];
+
+	offset = pc - (long)&ex->insn;
+	if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
+		return -ERANGE;
+	ex->insn = offset;
+
+	offset = (long)fixup - (long)&ex->fixup;
+	if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
+		return -ERANGE;
+	ex->fixup = offset;
+
+	ctx->exentry_idx++;
+	return 0;
+}
+
  /* Assemble the body code between the prologue & epilogue */
  int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx,
-		       u32 *addrs)
+		       u32 *addrs, int pass)
  {
  	const struct bpf_insn *insn = fp->insnsi;
  	int flen = fp->len;

@@ -714,12 +759,16 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
  		 */
  		/* dst = *(u8 *)(ul) (src + off) */
  		case BPF_LDX | BPF_MEM | BPF_B:
+		case BPF_LDX | BPF_PROBE_MEM | BPF_B:
  		/* dst = *(u16 *)(ul) (src + off) */
  		case BPF_LDX | BPF_MEM | BPF_H:
+		case BPF_LDX | BPF_PROBE_MEM | BPF_H:
  		/* dst = *(u32 *)(ul) (src + off) */
  		case BPF_LDX | BPF_MEM | BPF_W:
+		case BPF_LDX | BPF_PROBE_MEM | BPF_W:
  		/* dst = *(u64 *)(ul) (src + off) */
  		case BPF_LDX | BPF_MEM | BPF_DW:
+		case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
  			switch (size) {
  			case BPF_B:
  				EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));

@@ -737,6 +786,12 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
  
  			if ((size != BPF_DW) && insn_is_zext(&insn[i + 1]))
  				addrs[++i] = ctx->idx * 4;
+
+			if (BPF_MODE(code) == BPF_PROBE_MEM) {
+				ret = bpf_add_extable_entry(fp, image, pass, ctx, dst_reg);
+				if (ret)
+					return ret;
+			}
  			break;
  
  		/*

This will catch NULL, valid or invalid userspace pointers. Only bad
kernel pointer will be handled by BPF exception table.

[Alexei suggested for x86]
Suggested-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Ravi Bangoria <redacted>
Signed-off-by: Hari Bathini <hbathini@linux.ibm.com>
---

Changes in v2:
* Refactored the code based on Christophe's comments.


  arch/powerpc/net/bpf_jit_comp64.c | 23 +++++++++++++++++++++++
  1 file changed, 23 insertions(+)

diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c
index 2fc10995f243..eb28dbc67151 100644
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c

@@ -769,6 +769,29 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
  		/* dst = *(u64 *)(ul) (src + off) */
  		case BPF_LDX | BPF_MEM | BPF_DW:
  		case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
+			/*
+			 * As PTR_TO_BTF_ID that uses BPF_PROBE_MEM mode could either be a valid
+			 * kernel pointer or NULL but not a userspace address, execute BPF_PROBE_MEM
+			 * load only if addr > TASK_SIZE_MAX, otherwise set dst_reg=0 and move on.
+			 */
+			if (BPF_MODE(code) == BPF_PROBE_MEM) {
+				unsigned int adjusted_idx;
+
+				/*
+				 * Check if 'off' is word aligned because PPC_BPF_LL()
+				 * (BPF_DW case) generates two instructions if 'off' is not
+				 * word-aligned and one instruction otherwise.
+				 */
+				adjusted_idx = ((BPF_SIZE(code) == BPF_DW) && (off & 3)) ? 1 : 0;

Signed-off-by: Hari Bathini <hbathini@linux.ibm.com>
---

Changes in v2:
* New patch to refactor a bit of JITing code.


  arch/powerpc/net/bpf_jit_comp32.c | 50 +++++++++++---------
  arch/powerpc/net/bpf_jit_comp64.c | 76 ++++++++++++++++---------------
  2 files changed, 68 insertions(+), 58 deletions(-)

diff --git a/arch/powerpc/net/bpf_jit_comp32.c b/arch/powerpc/net/bpf_jit_comp32.c
index b60b59426a24..c8ae14c316e3 100644
--- a/arch/powerpc/net/bpf_jit_comp32.c
+++ b/arch/powerpc/net/bpf_jit_comp32.c

@@ -276,17 +276,17 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
  	u32 exit_addr = addrs[flen];
  
  	for (i = 0; i < flen; i++) {
-		u32 code = insn[i].code;
  		u32 dst_reg = bpf_to_ppc(ctx, insn[i].dst_reg);
-		u32 dst_reg_h = dst_reg - 1;
  		u32 src_reg = bpf_to_ppc(ctx, insn[i].src_reg);
-		u32 src_reg_h = src_reg - 1;
  		u32 tmp_reg = bpf_to_ppc(ctx, TMP_REG);
+		u32 true_cond, code = insn[i].code;
+		u32 dst_reg_h = dst_reg - 1;
+		u32 src_reg_h = src_reg - 1;

+		u32 size = BPF_SIZE(code);
  		s16 off = insn[i].off;
  		s32 imm = insn[i].imm;
  		bool func_addr_fixed;
  		u64 func_addr;
-		u32 true_cond;
  
  		/*
  		 * addrs[] maps a BPF bytecode address into a real offset from

@@ -809,25 +809,33 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
  		/*
  		 * BPF_LDX
  		 */
-		case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */
-			EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));
-			if (!fp->aux->verifier_zext)
-				EMIT(PPC_RAW_LI(dst_reg_h, 0));
-			break;
-		case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */
-			EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off));
-			if (!fp->aux->verifier_zext)
-				EMIT(PPC_RAW_LI(dst_reg_h, 0));
-			break;
-		case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */
-			EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off));
-			if (!fp->aux->verifier_zext)
+		/* dst = *(u8 *)(ul) (src + off) */
+		case BPF_LDX | BPF_MEM | BPF_B:
+		/* dst = *(u16 *)(ul) (src + off) */
+		case BPF_LDX | BPF_MEM | BPF_H:
+		/* dst = *(u32 *)(ul) (src + off) */
+		case BPF_LDX | BPF_MEM | BPF_W:
+		/* dst = *(u64 *)(ul) (src + off) */
+		case BPF_LDX | BPF_MEM | BPF_DW:

  				EMIT(PPC_RAW_LI(dst_reg_h, 0));
  			break;
-		case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */
-			EMIT(PPC_RAW_LWZ(dst_reg_h, src_reg, off));
-			EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off + 4));
-			break;
  
  		/*
  		 * Doubleword load

diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c
index 2a87da50d9a4..78b28f25555c 100644
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c

@@ -282,16 +282,15 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
  	u32 exit_addr = addrs[flen];
  
  	for (i = 0; i < flen; i++) {
-		u32 code = insn[i].code;
  		u32 dst_reg = b2p[insn[i].dst_reg];
  		u32 src_reg = b2p[insn[i].src_reg];
+		u32 tmp_idx, code = insn[i].code;
+		u32 size = BPF_SIZE(code);
  		s16 off = insn[i].off;
  		s32 imm = insn[i].imm;
  		bool func_addr_fixed;
-		u64 func_addr;
-		u64 imm64;
+		u64 func_addr, imm64;
  		u32 true_cond;
-		u32 tmp_idx;

  
  		/*
  		 * addrs[] maps a BPF bytecode address into a real offset from

@@ -638,35 +637,34 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
  		 */
  		case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */
  		case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */
-			if (BPF_CLASS(code) == BPF_ST) {
-				EMIT(PPC_RAW_LI(b2p[TMP_REG_1], imm));
-				src_reg = b2p[TMP_REG_1];
-			}
-			EMIT(PPC_RAW_STB(src_reg, dst_reg, off));
-			break;
  		case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
  		case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */
-			if (BPF_CLASS(code) == BPF_ST) {
-				EMIT(PPC_RAW_LI(b2p[TMP_REG_1], imm));
-				src_reg = b2p[TMP_REG_1];
-			}
-			EMIT(PPC_RAW_STH(src_reg, dst_reg, off));
-			break;
  		case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
  		case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */
-			if (BPF_CLASS(code) == BPF_ST) {
-				PPC_LI32(b2p[TMP_REG_1], imm);
-				src_reg = b2p[TMP_REG_1];
-			}
-			EMIT(PPC_RAW_STW(src_reg, dst_reg, off));
-			break;
  		case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
  		case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */
  			if (BPF_CLASS(code) == BPF_ST) {
-				PPC_LI32(b2p[TMP_REG_1], imm);
+				if ((size == BPF_B) || (size == BPF_H))
+					EMIT(PPC_RAW_LI(b2p[TMP_REG_1], imm));
+				else /* size == BPF_W || size == BPF_DW */
+					PPC_LI32(b2p[TMP_REG_1], imm);

  				src_reg = b2p[TMP_REG_1];
  			}
-			PPC_BPF_STL(src_reg, dst_reg, off);
+
+			switch (size) {
+			case BPF_B:
+				EMIT(PPC_RAW_STB(src_reg, dst_reg, off));
+				break;
+			case BPF_H:
+				EMIT(PPC_RAW_STH(src_reg, dst_reg, off));
+				break;
+			case BPF_W:
+				EMIT(PPC_RAW_STW(src_reg, dst_reg, off));
+				break;
+			case BPF_DW:
+				PPC_BPF_STL(src_reg, dst_reg, off);
+				break;
+			}
  			break;
  
  		/*

@@ -716,25 +714,29 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
  		 */
  		/* dst = *(u8 *)(ul) (src + off) */
  		case BPF_LDX | BPF_MEM | BPF_B:
-			EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));
-			if (insn_is_zext(&insn[i + 1]))
-				addrs[++i] = ctx->idx * 4;
-			break;
  		/* dst = *(u16 *)(ul) (src + off) */
  		case BPF_LDX | BPF_MEM | BPF_H:
-			EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off));
-			if (insn_is_zext(&insn[i + 1]))
-				addrs[++i] = ctx->idx * 4;
-			break;
  		/* dst = *(u32 *)(ul) (src + off) */
  		case BPF_LDX | BPF_MEM | BPF_W:
-			EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off));
-			if (insn_is_zext(&insn[i + 1]))
-				addrs[++i] = ctx->idx * 4;
-			break;
  		/* dst = *(u64 *)(ul) (src + off) */
  		case BPF_LDX | BPF_MEM | BPF_DW:
-			PPC_BPF_LL(dst_reg, src_reg, off);
+			switch (size) {
+			case BPF_B:
+				EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));
+				break;
+			case BPF_H:
+				EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off));
+				break;
+			case BPF_W:
+				EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off));
+				break;
+			case BPF_DW:
+				PPC_BPF_LL(dst_reg, src_reg, off);
+				break;
+			}
+
+			if ((size != BPF_DW) && insn_is_zext(&insn[i + 1]))
+				addrs[++i] = ctx->idx * 4;
  			break;
  
  		/*

Refactor powerpc JITing. This simplifies adding BPF_PROBE_MEM support.

Signed-off-by: Hari Bathini <hbathini@linux.ibm.com>
---

Changes in v2:
* New patch to refactor a bit of JITing code.


  arch/powerpc/net/bpf_jit_comp32.c | 50 +++++++++++---------
  arch/powerpc/net/bpf_jit_comp64.c | 76 ++++++++++++++++---------------
  2 files changed, 68 insertions(+), 58 deletions(-)

diff --git a/arch/powerpc/net/bpf_jit_comp32.c b/arch/powerpc/net/bpf_jit_comp32.c
index b60b59426a24..c8ae14c316e3 100644
--- a/arch/powerpc/net/bpf_jit_comp32.c
+++ b/arch/powerpc/net/bpf_jit_comp32.c

@@ -276,17 +276,17 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
  	u32 exit_addr = addrs[flen];
  
  	for (i = 0; i < flen; i++) {
-		u32 code = insn[i].code;
  		u32 dst_reg = bpf_to_ppc(ctx, insn[i].dst_reg);
-		u32 dst_reg_h = dst_reg - 1;
  		u32 src_reg = bpf_to_ppc(ctx, insn[i].src_reg);
-		u32 src_reg_h = src_reg - 1;
  		u32 tmp_reg = bpf_to_ppc(ctx, TMP_REG);
+		u32 true_cond, code = insn[i].code;
+		u32 dst_reg_h = dst_reg - 1;
+		u32 src_reg_h = src_reg - 1;
+		u32 size = BPF_SIZE(code);
  		s16 off = insn[i].off;
  		s32 imm = insn[i].imm;
  		bool func_addr_fixed;
  		u64 func_addr;
-		u32 true_cond;
  
  		/*
  		 * addrs[] maps a BPF bytecode address into a real offset from

@@ -809,25 +809,33 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
  		/*
  		 * BPF_LDX
  		 */
-		case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */
-			EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));
-			if (!fp->aux->verifier_zext)
-				EMIT(PPC_RAW_LI(dst_reg_h, 0));
-			break;
-		case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */
-			EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off));
-			if (!fp->aux->verifier_zext)
-				EMIT(PPC_RAW_LI(dst_reg_h, 0));
-			break;
-		case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */
-			EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off));
-			if (!fp->aux->verifier_zext)
+		/* dst = *(u8 *)(ul) (src + off) */
+		case BPF_LDX | BPF_MEM | BPF_B:
+		/* dst = *(u16 *)(ul) (src + off) */
+		case BPF_LDX | BPF_MEM | BPF_H:
+		/* dst = *(u32 *)(ul) (src + off) */
+		case BPF_LDX | BPF_MEM | BPF_W:
+		/* dst = *(u64 *)(ul) (src + off) */
+		case BPF_LDX | BPF_MEM | BPF_DW: