--- vrfc
+++ v1
@@ -1,21 +1,33 @@
-Instruction slot for detour buffer is allocated from
-the reserved area. For the time being 64KB is reserved
-in memory for this purpose. ppc_get_optinsn_slot() and
-ppc_free_optinsn_slot() are geared towards the allocation and freeing
-of memory from this area.
+Instructions which can be emulated are suppliants for optimization.
+Before optimization ensure that the address range between the detour
+buffer allocated and the instruction being probed is within ± 32MB.
-Signed-off-by: Anju T <anju@linux.vnet.ibm.com>
+Signed-off-by: Anju T Sudhakar <anju@linux.vnet.ibm.com>
---
- arch/powerpc/kernel/optprobes.c | 463 ++++++++++++++++++++++++++++++++++++++++
- 1 file changed, 463 insertions(+)
+ arch/powerpc/include/asm/sstep.h | 1 +
+ arch/powerpc/kernel/optprobes.c | 329 +++++++++++++++++++++++++++++++++++++++
+ arch/powerpc/lib/sstep.c | 21 +++
+ 3 files changed, 351 insertions(+)
create mode 100644 arch/powerpc/kernel/optprobes.c
+diff --git a/arch/powerpc/include/asm/sstep.h b/arch/powerpc/include/asm/sstep.h
+index d3a42cc..cd5f6ab 100644
+--- a/arch/powerpc/include/asm/sstep.h
++++ b/arch/powerpc/include/asm/sstep.h
+@@ -25,6 +25,7 @@ struct pt_regs;
+
+ /* Emulate instructions that cause a transfer of control. */
+ extern int emulate_step(struct pt_regs *regs, unsigned int instr);
++extern int optprobe_conditional_branch_check(unsigned int instr);
+
+ enum instruction_type {
+ COMPUTE, /* arith/logical/CR op, etc. */
diff --git a/arch/powerpc/kernel/optprobes.c b/arch/powerpc/kernel/optprobes.c
new file mode 100644
-index 0000000..50a60c1
+index 0000000..7983d07
--- /dev/null
+++ b/arch/powerpc/kernel/optprobes.c
-@@ -0,0 +1,463 @@
+@@ -0,0 +1,329 @@
+/*
+ * Code for Kernel probes Jump optimization.
+ *
@@ -38,148 +50,46 @@
+#include <asm/code-patching.h>
+#include <asm/sstep.h>
+
-+/* Reserve an area to allocate slots for detour buffer */
-+extern void optprobe_trampoline_holder(void)
-+{
-+ asm volatile(".global optinsn_slot\n"
-+ "optinsn_slot:\n"
-+ ".space 65536");
-+}
-+
-+#define SLOT_SIZE 65536
++DEFINE_INSN_CACHE_OPS(ppc_optinsn)
++
+#define TMPL_CALL_HDLR_IDX \
+ (optprobe_template_call_handler - optprobe_template_entry)
+#define TMPL_EMULATE_IDX \
+ (optprobe_template_call_emulate - optprobe_template_entry)
-+#define TMPL_RET_BRANCH_IDX \
-+ (optprobe_template_ret_branch - optprobe_template_entry)
+#define TMPL_RET_IDX \
+ (optprobe_template_ret - optprobe_template_entry)
++#define TMPL_KP_IDX \
++ (optprobe_template_kp_addr - optprobe_template_entry)
+#define TMPL_OP1_IDX \
+ (optprobe_template_op_address1 - optprobe_template_entry)
-+#define TMPL_OP2_IDX \
-+ (optprobe_template_op_address2 - optprobe_template_entry)
+#define TMPL_INSN_IDX \
+ (optprobe_template_insn - optprobe_template_entry)
+#define TMPL_END_IDX \
+ (optprobe_template_end - optprobe_template_entry)
+
-+struct kprobe_ppc_insn_page {
-+ struct list_head list;
-+ kprobe_opcode_t *insns; /* Page of instruction slots */
-+ struct kprobe_insn_cache *cache;
-+ int nused;
-+ int ngarbage;
-+ char slot_used[];
-+};
-+
-+#define PPC_KPROBE_INSN_PAGE_SIZE(slots) \
-+ (offsetof(struct kprobe_ppc_insn_page, slot_used) + \
-+ (sizeof(char) * (slots)))
-+
-+enum ppc_kprobe_slot_state {
-+ SLOT_CLEAN = 0,
-+ SLOT_DIRTY = 1,
-+ SLOT_USED = 2,
-+};
-+
-+static struct kprobe_insn_cache kprobe_ppc_optinsn_slots = {
++static bool insn_page_in_use;
++
++static void *__ppc_alloc_insn_page(void)
++{
++ if (insn_page_in_use)
++ return NULL;
++ insn_page_in_use = true;
++ return &optinsn_slot;
++}
++
++static void __ppc_free_insn_page(void *page __maybe_unused)
++{
++ insn_page_in_use = false;
++}
++
++struct kprobe_insn_cache kprobe_ppc_optinsn_slots = {
+ .mutex = __MUTEX_INITIALIZER(kprobe_ppc_optinsn_slots.mutex),
+ .pages = LIST_HEAD_INIT(kprobe_ppc_optinsn_slots.pages),
-+ /* .insn_size is initialized later */
++ /* insn_size initialized later */
++ .alloc = __ppc_alloc_insn_page,
++ .free = __ppc_free_insn_page,
+ .nr_garbage = 0,
+};
-+
-+static int ppc_slots_per_page(struct kprobe_insn_cache *c)
-+{
-+ /*
-+ * Here the #slots per page differs from x86 as we have
-+ * only 64KB reserved.
-+ */
-+ return SLOT_SIZE / (c->insn_size * sizeof(kprobe_opcode_t));
-+}
-+
-+/* Return 1 if all garbages are collected, otherwise 0. */
-+static int collect_one_slot(struct kprobe_ppc_insn_page *kip, int idx)
-+{
-+ kip->slot_used[idx] = SLOT_CLEAN;
-+ kip->nused--;
-+ return 0;
-+}
-+
-+static int collect_garbage_slots(struct kprobe_insn_cache *c)
-+{
-+ struct kprobe_ppc_insn_page *kip, *next;
-+
-+ /* Ensure no-one is interrupted on the garbages */
-+ synchronize_sched();
-+
-+ list_for_each_entry_safe(kip, next, &c->pages, list) {
-+ int i;
-+
-+ if (kip->ngarbage == 0)
-+ continue;
-+ kip->ngarbage = 0; /* we will collect all garbages */
-+ for (i = 0; i < ppc_slots_per_page(c); i++) {
-+ if (kip->slot_used[i] == SLOT_DIRTY &&
-+ collect_one_slot(kip, i))
-+ break;
-+ }
-+ }
-+ c->nr_garbage = 0;
-+ return 0;
-+}
-+
-+kprobe_opcode_t *__ppc_get_optinsn_slot(struct kprobe_insn_cache *c)
-+{
-+ struct kprobe_ppc_insn_page *kip;
-+ kprobe_opcode_t *slot = NULL;
-+
-+ mutex_lock(&c->mutex);
-+ list_for_each_entry(kip, &c->pages, list) {
-+ if (kip->nused < ppc_slots_per_page(c)) {
-+ int i;
-+
-+ for (i = 0; i < ppc_slots_per_page(c); i++) {
-+ if (kip->slot_used[i] == SLOT_CLEAN) {
-+ kip->slot_used[i] = SLOT_USED;
-+ kip->nused++;
-+ slot = kip->insns + (i * c->insn_size);
-+ goto out;
-+ }
-+ }
-+ /* kip->nused reached max value. */
-+ kip->nused = ppc_slots_per_page(c);
-+ pr_info("No more slots to allocate\n");
-+ WARN_ON(1);
-+ goto out;
-+ }
-+ }
-+ kip = kmalloc(PPC_KPROBE_INSN_PAGE_SIZE(ppc_slots_per_page(c)),
-+ GFP_KERNEL);
-+ if (!kip)
-+ goto out;
-+ /*
-+ * Allocate from the reserved area so as to
-+ * ensure the range will be within +/-32MB
-+ */
-+ kip->insns = &optinsn_slot;
-+ if (!kip->insns) {
-+ kfree(kip);
-+ goto out;
-+ }
-+ INIT_LIST_HEAD(&kip->list);
-+ memset(kip->slot_used, SLOT_CLEAN, ppc_slots_per_page(c));
-+ kip->slot_used[0] = SLOT_USED;
-+ kip->nused = 1;
-+ kip->ngarbage = 0;
-+ kip->cache = c;
-+ list_add(&kip->list, &c->pages);
-+ slot = kip->insns;
-+out:
-+ mutex_unlock(&c->mutex);
-+ return slot;
-+}
+
+kprobe_opcode_t *ppc_get_optinsn_slot(struct optimized_kprobe *op)
+{
@@ -188,42 +98,9 @@
+ * area(ie &optinsn_slot).We are not optimizing probes
+ * at module_addr now.
+ */
-+ kprobe_opcode_t *slot = NULL;
-+
-+ if (is_kernel_addr(op->kp.addr))
-+ slot = __ppc_get_optinsn_slot(&kprobe_ppc_optinsn_slots);
-+ else
-+ pr_info("Kprobe can not be optimized\n");
-+ return slot;
-+}
-+NOKPROBE_SYMBOL(ppc_get_optinsn_slot);
-+
-+void __ppc_free_optinsn_slot(struct kprobe_insn_cache *c,
-+ kprobe_opcode_t *slot, int dirty)
-+{
-+ struct kprobe_ppc_insn_page *kip;
-+
-+ mutex_lock(&c->mutex);
-+
-+ list_for_each_entry(kip, &c->pages, list) {
-+ long idx = ((long)slot - (long)kip->insns) /
-+ (c->insn_size * sizeof(kprobe_opcode_t));
-+ if (idx >= 0 && idx < ppc_slots_per_page(c)) {
-+ WARN_ON(kip->slot_used[idx] != SLOT_USED);
-+ if (dirty) {
-+ kip->slot_used[idx] = SLOT_DIRTY;
-+ kip->ngarbage++;
-+ if (++c->nr_garbage > ppc_slots_per_page(c))
-+ collect_garbage_slots(c);
-+ } else
-+ collect_one_slot(kip, idx);
-+ goto out;
-+ }
-+ }
-+ /* Could not free this slot. */
-+ WARN_ON(1);
-+out:
-+ mutex_unlock(&c->mutex);
++ if (is_kernel_addr((unsigned long)op->kp.addr))
++ return get_ppc_optinsn_slot();
++ return NULL;
+}
+
+static void ppc_free_optinsn_slot(struct optimized_kprobe *op)
@@ -231,63 +108,37 @@
+ if (!op->optinsn.insn)
+ return;
+ if (is_kernel_addr((unsigned long)op->kp.addr))
-+ __ppc_free_optinsn_slot(&kprobe_ppc_optinsn_slots,
-+ op->optinsn.insn, 0);
-+}
-+NOKPROBE_SYMBOL(ppc_free_optinsn_slot);
-+
-+static void
-+__arch_remove_optimized_kprobe(struct optimized_kprobe *op, int dirty)
-+{
-+ if (op->optinsn.insn) {
-+ ppc_free_optinsn_slot(op);
-+ op->optinsn.insn = NULL;
-+ }
-+}
-+
-+static int can_optimize(struct kprobe *p)
-+{
++ free_ppc_optinsn_slot(op->optinsn.insn, 0);
++}
++
++static unsigned long can_optimize(struct kprobe *p)
++{
++ struct pt_regs *regs;
++ unsigned int instr;
++
+ /*
+ * Not optimizing the kprobe placed by
+ * kretprobe during boot time
+ */
-+ struct pt_regs *regs;
-+ unsigned int instr;
-+ int r;
-+
-+ if ((kprobe_opcode_t)p->addr == (kprobe_opcode_t)&kretprobe_trampoline)
++ if (p->addr == (kprobe_opcode_t *)&kretprobe_trampoline)
+ return 0;
+
-+ regs = current_pt_regs();
-+ instr = *(p->ainsn.insn);
-+
-+ /* Ensure the instruction can be emulated*/
-+ r = emulate_step(regs, instr);
-+ if (r != 1)
++ regs = kmalloc(sizeof(*regs), GFP_KERNEL);
++ if (!regs)
++ return -ENOMEM;
++ memset(regs, 0, sizeof(struct pt_regs));
++ memcpy(regs, current_pt_regs(), sizeof(struct pt_regs));
++ regs->nip = (unsigned long)p->addr;
++ instr = *p->ainsn.insn;
++
++ /* Ensure the instruction can be emulated */
++ if (emulate_step(regs, instr) != 1)
+ return 0;
-+
-+ return 1;
-+}
-+
-+static void
-+create_return_branch(struct optimized_kprobe *op, struct pt_regs *regs)
-+{
-+ /*
-+ * Create a branch back to the return address
-+ * after the probed instruction is emulated
-+ */
-+
-+ kprobe_opcode_t branch, *buff;
-+ unsigned long ret;
-+
-+ ret = regs->nip;
-+ buff = op->optinsn.insn;
-+
-+ branch = create_branch((unsigned int *)buff + TMPL_RET_IDX,
-+ (unsigned long)ret, 0);
-+ buff[TMPL_RET_IDX] = branch;
-+}
-+NOKPROBE_SYMBOL(create_return_branch);
++ /* Conditional branches are not optimized */
++ if (optprobe_conditional_branch_check(instr) != 1)
++ return 0;
++ return regs->nip;
++}
+
+static void
+optimized_callback(struct optimized_kprobe *op, struct pt_regs *regs)
@@ -297,9 +148,9 @@
+
+ local_irq_save(flags);
+
-+ if (kprobe_running())
++ if (kprobe_running()) {
+ kprobes_inc_nmissed_count(&op->kp);
-+ else {
++ } else {
+ __this_cpu_write(current_kprobe, &op->kp);
+ kcb->kprobe_status = KPROBE_HIT_ACTIVE;
+ opt_pre_handler(&op->kp, regs);
@@ -307,62 +158,83 @@
+ }
+ local_irq_restore(flags);
+}
++NOKPROBE_SYMBOL(optimized_callback);
+
+void arch_remove_optimized_kprobe(struct optimized_kprobe *op)
+{
-+ __arch_remove_optimized_kprobe(op, 1);
-+}
-+
-+void create_insn(unsigned int insn, kprobe_opcode_t *addr)
-+{
-+ unsigned int instr, instr2;
-+
-+ instr = 0x3c000000 | 0x800000 | ((insn >> 16) & 0xffff);
++ ppc_free_optinsn_slot(op);
++ op->optinsn.insn = NULL;
++}
++
++/*
++ * emulate_step() requires insn to be emulated as
++ * second parameter. Load register 'r4' with the
++ * instruction.
++ */
++void create_load_emulate_insn(unsigned int insn, kprobe_opcode_t *addr)
++{
++ u32 instr, instr2;
++
++ /* synthesize addis r4,0,(insn)@h */
++ instr = 0x3c000000 | 0x800000 | ((insn >> 16) & 0xffff);
+ *addr++ = instr;
++
++ /* ori r4,r4,(insn)@l */
+ instr2 = 0x60000000 | 0x40000 | 0x800000;
+ instr2 = instr2 | (insn & 0xffff);
+ *addr = instr2;
+}
+
-+void create_load_address_insn(struct optimized_kprobe *op,
-+ kprobe_opcode_t *addr, kprobe_opcode_t *addr2)
-+{
-+ unsigned int instr1, instr2, instr3, instr4, instr5;
-+ unsigned long val;
-+
-+ val = op;
-+
++/*
++ * optimized_kprobe structure is required as a parameter
++ * for invoking optimized_callback() from detour buffer.
++ * Load this value into register 'r3'.
++ */
++void create_load_address_insn(unsigned long val, kprobe_opcode_t *addr)
++{
++ u32 instr1, instr2, instr3, instr4, instr5;
++ /*
++ * 64bit immediate load into r3.
++ * lis r3,(op)@highest
++ */
+ instr1 = 0x3c000000 | 0x600000 | ((val >> 48) & 0xffff);
+ *addr++ = instr1;
-+ *addr2++ = instr1;
++
++ /* ori r3,r3,(op)@higher */
+ instr2 = 0x60000000 | 0x30000 | 0x600000 | ((val >> 32) & 0xffff);
+ *addr++ = instr2;
-+ *addr2++ = instr2;
++
++ /* rldicr r3,r3,32,31 */
+ instr3 = 0x78000004 | 0x30000 | 0x600000 | ((32 & 0x1f) << 11);
+ instr3 = instr3 | ((31 & 0x1f) << 6) | ((32 & 0x20) >> 4);
+ *addr++ = instr3;
-+ *addr2++ = instr3;
++
++ /* oris r3,r3,(op)@h */
+ instr4 = 0x64000000 | 0x30000 | 0x600000 | ((val >> 16) & 0xffff);
+ *addr++ = instr4;
-+ *addr2++ = instr4;
-+ instr5 = 0x60000000 | 0x30000 | 0x600000 | (val & 0xffff);
++
++ /* ori r3,r3,(op)@l */
++ instr5 = 0x60000000 | 0x30000 | 0x600000 | (val & 0xffff);
+ *addr = instr5;
-+ *addr2 = instr5;
+}
+
+int arch_prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *p)
+{
+ kprobe_opcode_t *buff, branch, branch2, branch3;
+ long rel_chk, ret_chk;
++ unsigned long nip;
+
+ kprobe_ppc_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
+ op->optinsn.insn = NULL;
-+
-+ if (!can_optimize(p))
++ nip = can_optimize(p);
++
++ if (!nip)
+ return -EILSEQ;
+
-+ /* Allocate instruction slot for detour buffer*/
++ /* Allocate instruction slot for detour buffer */
+ buff = ppc_get_optinsn_slot(op);
++ if (!buff)
++ return -ENOMEM;
+
+ /*
+ * OPTPROBE use a 'b' instruction to branch to optinsn.insn.
@@ -373,57 +245,63 @@
+ * in the opcode specify the address. Therefore the address gap should
+ * be 32MB on either side of the current instruction.
+ */
-+ rel_chk = (long)buff -
-+ (unsigned long)p->addr;
++ rel_chk = (long)buff - (unsigned long)p->addr;
+ if (rel_chk < -0x2000000 || rel_chk > 0x1fffffc || rel_chk & 0x3) {
+ ppc_free_optinsn_slot(op);
+ return -ERANGE;
+ }
-+ /*
-+ * For the time being assume that the return address is NIP+4.
-+ * TODO : check the return address is also within 32MB range for
-+ * cases where NIP is not NIP+4.ie the NIP is decided after emulation.
-+ */
-+ ret_chk = (long)(buff + TMPL_RET_IDX) - (unsigned long)(p->addr) + 4;
-+
++ /* Check the return address is also within 32MB range */
++ ret_chk = (long)(buff + TMPL_RET_IDX) - (unsigned long)nip;
+ if (ret_chk < -0x2000000 || ret_chk > 0x1fffffc || ret_chk & 0x3) {
+ ppc_free_optinsn_slot(op);
+ return -ERANGE;
+ }
+
-+ /* Do Copy arch specific instance from template*/
++ /* Do Copy arch specific instance from template */
+ memcpy(buff, optprobe_template_entry,
+ TMPL_END_IDX * sizeof(kprobe_opcode_t));
-+ create_load_address_insn(op, buff + TMPL_OP1_IDX, buff + TMPL_OP2_IDX);
-+
-+ /* Create a branch to the optimized_callback function */
++
++ /* Load address into register */
++ create_load_address_insn((unsigned long)p->addr, buff + TMPL_KP_IDX);
++ create_load_address_insn((unsigned long)op, buff + TMPL_OP1_IDX);
++
++ /*
++ * Create a branch to the optimized_callback function.
++ * optimized_callback, points to the global entry point.
++ * Add +8, to create a branch to the LEP of the function.
++ */
+ branch = create_branch((unsigned int *)buff + TMPL_CALL_HDLR_IDX,
+ (unsigned long)optimized_callback + 8,
+ BRANCH_SET_LINK);
+
+ /* Place the branch instr into the trampoline */
+ buff[TMPL_CALL_HDLR_IDX] = branch;
-+ create_insn(*(p->ainsn.insn), buff + TMPL_INSN_IDX);
-+
-+ /*Create a branch instruction into the emulate_step*/
++
++ /* Load instruction to be emulated into relevant register */
++ create_load_emulate_insn(*p->ainsn.insn, buff + TMPL_INSN_IDX);
++
++ /*
++ * Create a branch instruction into the emulate_step.
++ * Add +8, to create the branch to LEP of emulate_step().
++ */
+ branch3 = create_branch((unsigned int *)buff + TMPL_EMULATE_IDX,
+ (unsigned long)emulate_step + 8,
+ BRANCH_SET_LINK);
+ buff[TMPL_EMULATE_IDX] = branch3;
+
-+ /* Create a branch for jumping back*/
-+ branch2 = create_branch((unsigned int *)buff + TMPL_RET_BRANCH_IDX,
-+ (unsigned long)create_return_branch + 8,
-+ BRANCH_SET_LINK);
-+ buff[TMPL_RET_BRANCH_IDX] = branch2;
++ /* Create a branch for jumping back */
++ branch2 = create_branch((unsigned int *)buff + TMPL_RET_IDX,
++ (unsigned long)nip, 0);
++ buff[TMPL_RET_IDX] = branch2;
+
+ op->optinsn.insn = buff;
++ smp_mb();
+ return 0;
+}
+
+int arch_prepared_optinsn(struct arch_optimized_insn *optinsn)
+{
-+ return optinsn->insn;
++ return optinsn->insn != NULL;
+}
+
+/*
@@ -444,16 +322,16 @@
+ unsigned int branch;
+
+ list_for_each_entry_safe(op, tmp, oplist, list) {
-+ /*
-+ * Backup instructions which will be replaced
-+ * by jump address
-+ */
-+ memcpy(op->optinsn.copied_insn, op->kp.addr,
-+ RELATIVEJUMP_SIZE);
-+ branch = create_branch((unsigned int *)op->kp.addr,
-+ (unsigned long)op->optinsn.insn, 0);
-+ *op->kp.addr = branch;
-+ list_del_init(&op->list);
++ /*
++ * Backup instructions which will be replaced
++ * by jump address
++ */
++ memcpy(op->optinsn.copied_insn, op->kp.addr,
++ RELATIVEJUMP_SIZE);
++ branch = create_branch((unsigned int *)op->kp.addr,
++ (unsigned long)op->optinsn.insn, 0);
++ *op->kp.addr = branch;
++ list_del_init(&op->list);
+ }
+}
+
@@ -479,5 +357,34 @@
+{
+ return 0;
+}
+diff --git a/arch/powerpc/lib/sstep.c b/arch/powerpc/lib/sstep.c
+index 3362299..c4b8259 100644
+--- a/arch/powerpc/lib/sstep.c
++++ b/arch/powerpc/lib/sstep.c
+@@ -2018,3 +2018,24 @@ int __kprobes emulate_step(struct pt_regs *regs, unsigned int instr)
+ regs->nip = truncate_if_32bit(regs->msr, regs->nip + 4);
+ return 1;
+ }
++
++/* Before optimizing, ensure that the probed instruction is not a
++ * conditional branch instruction
++ */
++int __kprobes optprobe_conditional_branch_check(unsigned int instr)
++{
++ unsigned int opcode;
++
++ opcode = instr >> 26;
++ if (opcode == 16)
++ return 0;
++ if (opcode == 19) {
++ switch ((instr >> 1) & 0x3ff) {
++ case 16: /* bclr, bclrl */
++ case 528: /* bcctr, bcctrl */
++ case 560: /* bctar, bctarl */
++ return 0;
++ }
++ }
++ return 1;
++}
--
-2.1.0
+1.8.3.1