--- v4
+++ v8
@@ -1,256 +1,207 @@
-At least POWER7/8 have MMUs that don't completely autoload;
-a normal, recoverable memory fault might pass through these functions.
-If a dynamic tracer function causes such a fault, any of these functions
-being traced with -mprofile-kernel may cause an endless recursion.
+ * create the appropriate files+functions
+ arch/powerpc/include/asm/livepatch.h
+ klp_check_compiler_support,
+ klp_arch_set_pc
+ arch/powerpc/kernel/livepatch.c with a stub for
+ klp_write_module_reloc
+ This is architecture-independent work in progress.
+ * introduce a fixup in arch/powerpc/kernel/entry_64.S
+ for local calls that are becoming global due to live patching.
+ And of course do the main KLP thing: return to a maybe different
+ address, possibly altered by the live patching ftrace op.
Signed-off-by: Torsten Duwe <duwe@suse.de>
---
- arch/powerpc/kernel/process.c | 2 +-
- arch/powerpc/mm/fault.c | 2 +-
- arch/powerpc/mm/hash_utils_64.c | 18 +++++++++---------
- arch/powerpc/mm/hugetlbpage-hash64.c | 2 +-
- arch/powerpc/mm/hugetlbpage.c | 4 ++--
- arch/powerpc/mm/mem.c | 2 +-
- arch/powerpc/mm/pgtable_64.c | 2 +-
- arch/powerpc/mm/slb.c | 6 +++---
- arch/powerpc/mm/slice.c | 8 ++++----
- 9 files changed, 23 insertions(+), 23 deletions(-)
+ arch/powerpc/include/asm/livepatch.h | 45 +++++++++++++++++++++++++++++++
+ arch/powerpc/kernel/entry_64.S | 51 +++++++++++++++++++++++++++++++++---
+ arch/powerpc/kernel/livepatch.c | 38 +++++++++++++++++++++++++++
+ 3 files changed, 130 insertions(+), 4 deletions(-)
+ create mode 100644 arch/powerpc/include/asm/livepatch.h
+ create mode 100644 arch/powerpc/kernel/livepatch.c
-diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
-index 75b6676..c2900b9 100644
---- a/arch/powerpc/kernel/process.c
-+++ b/arch/powerpc/kernel/process.c
-@@ -715,7 +715,7 @@ static inline void __switch_to_tm(struct task_struct *prev)
- * don't know which of the checkpointed state and the transactional
- * state to use.
- */
--void restore_tm_state(struct pt_regs *regs)
-+notrace void restore_tm_state(struct pt_regs *regs)
- {
- unsigned long msr_diff;
-
-diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
-index a67c6d7..125be37 100644
---- a/arch/powerpc/mm/fault.c
-+++ b/arch/powerpc/mm/fault.c
-@@ -205,7 +205,7 @@ static int mm_fault_error(struct pt_regs *regs, unsigned long addr, int fault)
- * The return value is 0 if the fault was handled, or the signal
- * number if this is a kernel fault that can't be handled here.
- */
--int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address,
-+notrace int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address,
- unsigned long error_code)
- {
- enum ctx_state prev_state = exception_enter();
-diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
-index aee7017..90e89e7 100644
---- a/arch/powerpc/mm/hash_utils_64.c
-+++ b/arch/powerpc/mm/hash_utils_64.c
-@@ -849,7 +849,7 @@ void early_init_mmu_secondary(void)
- /*
- * Called by asm hashtable.S for doing lazy icache flush
- */
--unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap)
-+notrace unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap)
- {
- struct page *page;
-
-@@ -870,7 +870,7 @@ unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap)
- }
-
- #ifdef CONFIG_PPC_MM_SLICES
--static unsigned int get_paca_psize(unsigned long addr)
-+static notrace unsigned int get_paca_psize(unsigned long addr)
- {
- u64 lpsizes;
- unsigned char *hpsizes;
-@@ -899,7 +899,7 @@ unsigned int get_paca_psize(unsigned long addr)
- * For now this makes the whole process use 4k pages.
- */
- #ifdef CONFIG_PPC_64K_PAGES
--void demote_segment_4k(struct mm_struct *mm, unsigned long addr)
-+notrace void demote_segment_4k(struct mm_struct *mm, unsigned long addr)
- {
- if (get_slice_psize(mm, addr) == MMU_PAGE_4K)
- return;
-@@ -920,7 +920,7 @@ void demote_segment_4k(struct mm_struct *mm, unsigned long addr)
- * Result is 0: full permissions, _PAGE_RW: read-only,
- * _PAGE_USER or _PAGE_USER|_PAGE_RW: no access.
- */
--static int subpage_protection(struct mm_struct *mm, unsigned long ea)
-+static notrace int subpage_protection(struct mm_struct *mm, unsigned long ea)
- {
- struct subpage_prot_table *spt = &mm->context.spt;
- u32 spp = 0;
-@@ -968,7 +968,7 @@ void hash_failure_debug(unsigned long ea, unsigned long access,
- trap, vsid, ssize, psize, lpsize, pte);
- }
-
--static void check_paca_psize(unsigned long ea, struct mm_struct *mm,
-+static notrace void check_paca_psize(unsigned long ea, struct mm_struct *mm,
- int psize, bool user_region)
- {
- if (user_region) {
-@@ -990,7 +990,7 @@ static void check_paca_psize(unsigned long ea, struct mm_struct *mm,
- * -1 - critical hash insertion error
- * -2 - access not permitted by subpage protection mechanism
- */
--int hash_page_mm(struct mm_struct *mm, unsigned long ea,
-+notrace int hash_page_mm(struct mm_struct *mm, unsigned long ea,
- unsigned long access, unsigned long trap,
- unsigned long flags)
- {
-@@ -1186,7 +1186,7 @@ bail:
- }
- EXPORT_SYMBOL_GPL(hash_page_mm);
-
--int hash_page(unsigned long ea, unsigned long access, unsigned long trap,
-+notrace int hash_page(unsigned long ea, unsigned long access, unsigned long trap,
- unsigned long dsisr)
- {
- unsigned long flags = 0;
-@@ -1288,7 +1288,7 @@ out_exit:
- /* WARNING: This is called from hash_low_64.S, if you change this prototype,
- * do not forget to update the assembly call site !
- */
--void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, int ssize,
-+notrace void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, int ssize,
- unsigned long flags)
- {
- unsigned long hash, index, shift, hidx, slot;
-@@ -1436,7 +1436,7 @@ void low_hash_fault(struct pt_regs *regs, unsigned long address, int rc)
- exception_exit(prev_state);
- }
-
--long hpte_insert_repeating(unsigned long hash, unsigned long vpn,
-+notrace long hpte_insert_repeating(unsigned long hash, unsigned long vpn,
- unsigned long pa, unsigned long rflags,
- unsigned long vflags, int psize, int ssize)
- {
-diff --git a/arch/powerpc/mm/hugetlbpage-hash64.c b/arch/powerpc/mm/hugetlbpage-hash64.c
-index d94b1af..50b8c6f 100644
---- a/arch/powerpc/mm/hugetlbpage-hash64.c
-+++ b/arch/powerpc/mm/hugetlbpage-hash64.c
-@@ -18,7 +18,7 @@ extern long hpte_insert_repeating(unsigned long hash, unsigned long vpn,
- unsigned long pa, unsigned long rlags,
- unsigned long vflags, int psize, int ssize);
-
--int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
-+notrace int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
- pte_t *ptep, unsigned long trap, unsigned long flags,
- int ssize, unsigned int shift, unsigned int mmu_psize)
- {
-diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c
-index 06c1452..bc2f459 100644
---- a/arch/powerpc/mm/hugetlbpage.c
-+++ b/arch/powerpc/mm/hugetlbpage.c
-@@ -922,7 +922,7 @@ static int __init hugetlbpage_init(void)
+diff --git a/arch/powerpc/include/asm/livepatch.h b/arch/powerpc/include/asm/livepatch.h
+new file mode 100644
+index 0000000..44e8a2d
+--- /dev/null
++++ b/arch/powerpc/include/asm/livepatch.h
+@@ -0,0 +1,45 @@
++/*
++ * livepatch.h - powerpc-specific Kernel Live Patching Core
++ *
++ * Copyright (C) 2015 SUSE
++ *
++ * This program is free software; you can redistribute it and/or
++ * modify it under the terms of the GNU General Public License
++ * as published by the Free Software Foundation; either version 2
++ * of the License, or (at your option) any later version.
++ *
++ * This program is distributed in the hope that it will be useful,
++ * but WITHOUT ANY WARRANTY; without even the implied warranty of
++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
++ * GNU General Public License for more details.
++ *
++ * You should have received a copy of the GNU General Public License
++ * along with this program; if not, see <http://www.gnu.org/licenses/>.
++ */
++#ifndef _ASM_POWERPC64_LIVEPATCH_H
++#define _ASM_POWERPC64_LIVEPATCH_H
++
++#include <linux/module.h>
++#include <linux/ftrace.h>
++
++#ifdef CONFIG_LIVEPATCH
++static inline int klp_check_compiler_support(void)
++{
++#if !defined(_CALL_ELF) || _CALL_ELF != 2 || !defined(CC_USING_MPROFILE_KERNEL)
++ return 1;
++#endif
++ return 0;
++}
++
++extern int klp_write_module_reloc(struct module *mod, unsigned long type,
++ unsigned long loc, unsigned long value);
++
++static inline void klp_arch_set_pc(struct pt_regs *regs, unsigned long ip)
++{
++ regs->nip = ip;
++}
++#else
++#error Live patching support is disabled; check CONFIG_LIVEPATCH
++#endif
++
++#endif /* _ASM_POWERPC64_LIVEPATCH_H */
+diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S
+index c063564..52c7a15 100644
+--- a/arch/powerpc/kernel/entry_64.S
++++ b/arch/powerpc/kernel/entry_64.S
+@@ -1202,6 +1202,9 @@ _GLOBAL(ftrace_caller)
+ mflr r3
+ std r3, _NIP(r1)
+ std r3, 16(r1)
++#ifdef CONFIG_LIVEPATCH
++ mr r14,r3 /* remember old NIP */
++#endif
+ subi r3, r3, MCOUNT_INSN_SIZE
+ mfmsr r4
+ std r4, _MSR(r1)
+@@ -1218,7 +1221,10 @@ ftrace_call:
+ nop
+
+ ld r3, _NIP(r1)
+- mtlr r3
++ mtctr r3 /* prepare to jump there */
++#ifdef CONFIG_LIVEPATCH
++ cmpd r14,r3 /* has NIP been altered? */
++#endif
+
+ REST_8GPRS(0,r1)
+ REST_8GPRS(8,r1)
+@@ -1231,6 +1237,27 @@ ftrace_call:
+ mtlr r12
+ mr r2,r0 /* restore callee's TOC */
+
++#ifdef CONFIG_LIVEPATCH
++ beq+ 4f /* likely(old_NIP == new_NIP) */
++
++ /* For a local call, restore this TOC after calling the patch function.
++ * For a global call, it does not matter what we restore here,
++ * since the global caller does its own restore right afterwards,
++ * anyway. Just insert a KLP_return_helper frame in any case,
++ * so a patch function can always count on the changed stack offsets.
++ */
++ stdu r1,-32(r1) /* open new mini stack frame */
++ std r0,24(r1) /* save TOC now, unconditionally. */
++ bl 5f
++5: mflr r12
++ addi r12,r12,(KLP_return_helper+4-.)@l
++ std r12,LRSAVE(r1)
++ mtlr r12
++ mfctr r12 /* allow for TOC calculation in newfunc */
++ bctr
++4:
++#endif
++
+ #ifdef CONFIG_FUNCTION_GRAPH_TRACER
+ stdu r1, -112(r1)
+ .globl ftrace_graph_call
+@@ -1240,15 +1267,31 @@ _GLOBAL(ftrace_graph_stub)
+ addi r1, r1, 112
#endif
- arch_initcall(hugetlbpage_init);
-
--void flush_dcache_icache_hugepage(struct page *page)
-+notrace void flush_dcache_icache_hugepage(struct page *page)
- {
- int i;
- void *start;
-@@ -955,7 +955,7 @@ void flush_dcache_icache_hugepage(struct page *page)
- * when we have MSR[EE] = 0 but the paca->soft_enabled = 1
- */
-
--pte_t *__find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea,
-+notrace pte_t *__find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea,
- unsigned *shift)
- {
- pgd_t pgd, *pgdp;
-diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
-index 22d94c3..f690e8a 100644
---- a/arch/powerpc/mm/mem.c
-+++ b/arch/powerpc/mm/mem.c
-@@ -406,7 +406,7 @@ void flush_dcache_page(struct page *page)
- }
- EXPORT_SYMBOL(flush_dcache_page);
-
--void flush_dcache_icache_page(struct page *page)
-+notrace void flush_dcache_icache_page(struct page *page)
- {
- #ifdef CONFIG_HUGETLB_PAGE
- if (PageCompound(page)) {
-diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c
-index e92cb21..c74050b 100644
---- a/arch/powerpc/mm/pgtable_64.c
-+++ b/arch/powerpc/mm/pgtable_64.c
-@@ -442,7 +442,7 @@ static void page_table_free_rcu(void *table)
- }
- }
-
--void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift)
-+notrace void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift)
- {
- unsigned long pgf = (unsigned long)table;
-
-diff --git a/arch/powerpc/mm/slb.c b/arch/powerpc/mm/slb.c
-index 8a32a2b..5b05754 100644
---- a/arch/powerpc/mm/slb.c
-+++ b/arch/powerpc/mm/slb.c
-@@ -91,7 +91,7 @@ static inline void create_shadowed_slbe(unsigned long ea, int ssize,
- : "memory" );
- }
-
--static void __slb_flush_and_rebolt(void)
-+static notrace void __slb_flush_and_rebolt(void)
- {
- /* If you change this make sure you change SLB_NUM_BOLTED
- * and PR KVM appropriately too. */
-@@ -131,7 +131,7 @@ static void __slb_flush_and_rebolt(void)
- : "memory");
- }
-
--void slb_flush_and_rebolt(void)
-+notrace void slb_flush_and_rebolt(void)
- {
-
- WARN_ON(!irqs_disabled());
-@@ -146,7 +146,7 @@ void slb_flush_and_rebolt(void)
- get_paca()->slb_cache_ptr = 0;
- }
-
--void slb_vmalloc_update(void)
-+notrace void slb_vmalloc_update(void)
- {
- unsigned long vflags;
-
-diff --git a/arch/powerpc/mm/slice.c b/arch/powerpc/mm/slice.c
-index 0f432a7..f92f0f0 100644
---- a/arch/powerpc/mm/slice.c
-+++ b/arch/powerpc/mm/slice.c
-@@ -76,8 +76,8 @@ static void slice_print_mask(const char *label, struct slice_mask mask) {}
-
- #endif
-
--static struct slice_mask slice_range_to_mask(unsigned long start,
-- unsigned long len)
-+static notrace struct slice_mask slice_range_to_mask(unsigned long start,
-+ unsigned long len)
- {
- unsigned long end = start + len - 1;
- struct slice_mask ret = { 0, 0 };
-@@ -564,7 +564,7 @@ unsigned long arch_get_unmapped_area_topdown(struct file *filp,
- current->mm->context.user_psize, 1);
- }
-
--unsigned int get_slice_psize(struct mm_struct *mm, unsigned long addr)
-+notrace unsigned int get_slice_psize(struct mm_struct *mm, unsigned long addr)
- {
- unsigned char *hpsizes;
- int index, mask_index;
-@@ -645,7 +645,7 @@ void slice_set_user_psize(struct mm_struct *mm, unsigned int psize)
- spin_unlock_irqrestore(&slice_convert_lock, flags);
- }
-
--void slice_set_range_psize(struct mm_struct *mm, unsigned long start,
-+notrace void slice_set_range_psize(struct mm_struct *mm, unsigned long start,
- unsigned long len, unsigned int psize)
- {
- struct slice_mask mask = slice_range_to_mask(start, len);
+
+- mflr r0 /* move this LR to CTR */
+- mtctr r0
+-
+ ld r0,LRSAVE(r1) /* restore callee's lr at _mcount site */
+ mtlr r0
+ bctr /* jump after _mcount site */
+ #endif /* CC_USING_MPROFILE_KERNEL */
+ _GLOBAL(ftrace_stub)
+ blr
++
++#ifdef CONFIG_LIVEPATCH
++/* Helper function for local calls that are becoming global
++ due to live patching.
++ We can't simply patch the NOP after the original call,
++ because, depending on the consistency model, some kernel
++ threads may still have called the original, local function
++ *without* saving their TOC in the respective stack frame slot,
++ so the decision is made per-thread during function return by
++ maybe inserting a KLP_return_helper frame or not.
++*/
++KLP_return_helper:
++ ld r2,24(r1) /* restore TOC (saved by ftrace_caller) */
++ addi r1, r1, 32 /* destroy mini stack frame */
++ ld r0,LRSAVE(r1) /* get the real return address */
++ mtlr r0
++ blr
++#endif
++
+ #else
+ _GLOBAL_TOC(_mcount)
+ /* Taken from output of objdump from lib64/glibc */
+diff --git a/arch/powerpc/kernel/livepatch.c b/arch/powerpc/kernel/livepatch.c
+new file mode 100644
+index 0000000..cdd15f1
+--- /dev/null
++++ b/arch/powerpc/kernel/livepatch.c
+@@ -0,0 +1,38 @@
++/*
++ * livepatch.c - powerpc-specific Kernel Live Patching Core
++ *
++ * Copyright (C) 2015 SUSE
++ *
++ * This program is free software; you can redistribute it and/or
++ * modify it under the terms of the GNU General Public License
++ * as published by the Free Software Foundation; either version 2
++ * of the License, or (at your option) any later version.
++ *
++ * This program is distributed in the hope that it will be useful,
++ * but WITHOUT ANY WARRANTY; without even the implied warranty of
++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
++ * GNU General Public License for more details.
++ *
++ * You should have received a copy of the GNU General Public License
++ * along with this program; if not, see <http://www.gnu.org/licenses/>.
++ */
++#include <linux/module.h>
++#include <asm/livepatch.h>
++
++/**
++ * klp_write_module_reloc() - write a relocation in a module
++ * @mod: module in which the section to be modified is found
++ * @type: ELF relocation type (see asm/elf.h)
++ * @loc: address that the relocation should be written to
++ * @value: relocation value (sym address + addend)
++ *
++ * This function writes a relocation to the specified location for
++ * a particular module.
++ */
++int klp_write_module_reloc(struct module *mod, unsigned long type,
++ unsigned long loc, unsigned long value)
++{
++ /* This requires infrastructure changes; we need the loadinfos. */
++ pr_err("klp_write_module_reloc not yet supported\n");
++ return -ENOSYS;
++}
--
1.8.5.6