[PATCH v3 00/17] LPAE fixes and extensions for Keystone

@@ -307,6 +307,8 @@ static inline __deprecated void *bus_to_virt(unsigned long x)
 #define virt_to_page(kaddr)	pfn_to_page(__pa(kaddr) >> PAGE_SHIFT)
 #define virt_addr_valid(kaddr)	((unsigned long)(kaddr) >= PAGE_OFFSET && (unsigned long)(kaddr) < (unsigned long)high_memory)
 
+#define ARCH_LOW_ADDRESS_LIMIT		PHYS_MASK
+
 #endif
 
 #include <asm-generic/memory_model.h>

@@ -116,13 +116,27 @@ extern void cpu_resume(void);
 #define cpu_switch_mm(pgd,mm) cpu_do_switch_mm(virt_to_phys(pgd),mm)
 
 #ifdef CONFIG_ARM_LPAE
+
+#define cpu_get_ttbr(nr)					\
+	({							\
+		u64 ttbr;					\
+		__asm__("mrrc	p15, " #nr ", %Q0, %R0, c2"	\
+			: "=r" (ttbr)				\
+			: : );					\
+		ttbr;						\
+	})
+
+#define cpu_set_ttbr(nr, val)					\
+	do {							\
+		u64 ttbr = val;					\
+		__asm__("mcrr	p15, " #nr ", %Q0, %R0, c2"	\
+			: : "r" (ttbr)				\
+			: "cc");				\
+	} while (0)
+
 #define cpu_get_pgd()	\
 	({						\
-		unsigned long pg, pg2;			\
-		__asm__("mrrc	p15, 0, %0, %1, c2"	\
-			: "=r" (pg), "=r" (pg2)		\
-			:				\
-			: "cc");			\
+		u64 pg = cpu_get_ttbr(0);		\
 		pg &= ~(PTRS_PER_PGD*sizeof(pgd_t)-1);	\
 		(pgd_t *)phys_to_virt(pg);		\
 	})

@@ -16,6 +16,7 @@
 #include <asm/mmu_context.h>
 #include <asm/thread_notify.h>
 #include <asm/tlbflush.h>
+#include <asm/proc-fns.h>
 
 static DEFINE_RAW_SPINLOCK(cpu_asid_lock);
 unsigned int cpu_last_asid = ASID_FIRST_VERSION;

@@ -23,17 +24,11 @@ unsigned int cpu_last_asid = ASID_FIRST_VERSION;
 #ifdef CONFIG_ARM_LPAE
 void cpu_set_reserved_ttbr0(void)
 {
-	unsigned long ttbl = __pa(swapper_pg_dir);
-	unsigned long ttbh = 0;
-
 	/*
 	 * Set TTBR0 to swapper_pg_dir which contains only global entries. The
 	 * ASID is set to 0.
 	 */
-	asm volatile(
-	"	mcrr	p15, 0, %0, %1, c2		@ set TTBR0\n"
-	:
-	: "r" (ttbl), "r" (ttbh));
+	cpu_set_ttbr(0, __pa(swapper_pg_dir));
 	isb();
 }
 #else

@@ -457,7 +457,7 @@ static inline void
 free_memmap(unsigned long start_pfn, unsigned long end_pfn)
 {
 	struct page *start_pg, *end_pg;
-	unsigned long pg, pgend;
+	phys_addr_t pg, pgend;
 
 	/*
 	 * Convert start_pfn/end_pfn to a struct page pointer.

@@ -469,8 +469,8 @@ free_memmap(unsigned long start_pfn, unsigned long end_pfn)
 	 * Convert to physical addresses, and
 	 * round start upwards and end downwards.
 	 */
-	pg = (unsigned long)PAGE_ALIGN(__pa(start_pg));
-	pgend = (unsigned long)__pa(end_pg) & PAGE_MASK;
+	pg = PAGE_ALIGN(__pa(start_pg));
+	pgend = __pa(end_pg) & PAGE_MASK;
 
 	/*
 	 * If there are free pages between these,

@@ -157,22 +157,32 @@ extern unsigned long	__pv_offset;
 extern unsigned long	__pv_phys_offset;
 #define PHYS_OFFSET	__virt_to_phys(PAGE_OFFSET)
 
-static inline unsigned long __virt_to_phys(unsigned long x)
+static inline phys_addr_t __virt_to_phys(unsigned long x)
 {
 	unsigned long t;
 	early_patch_imm8("add", t, x, __pv_offset, 0);
 	return t;
 }
 
-static inline unsigned long __phys_to_virt(unsigned long x)
+static inline unsigned long __phys_to_virt(phys_addr_t x)
 {
-	unsigned long t;
-	early_patch_imm8("sub", t, x, __pv_offset, 0);
+	unsigned long t, xlo = x;
+	early_patch_imm8("sub", t, xlo, __pv_offset, 0);
 	return t;
 }
+
 #else
-#define __virt_to_phys(x)	((x) - PAGE_OFFSET + PHYS_OFFSET)
-#define __phys_to_virt(x)	((x) - PHYS_OFFSET + PAGE_OFFSET)
+
+static inline phys_addr_t __virt_to_phys(unsigned long x)
+{
+	return (phys_addr_t)x - PAGE_OFFSET + PHYS_OFFSET;
+}
+
+static inline unsigned long __phys_to_virt(phys_addr_t x)
+{
+	return x - PHYS_OFFSET + PAGE_OFFSET;
+}
+
 #endif
 #endif
 #endif /* __ASSEMBLY__ */

@@ -210,14 +220,14 @@ static inline phys_addr_t virt_to_phys(const volatile void *x)
 
 static inline void *phys_to_virt(phys_addr_t x)
 {
-	return (void *)(__phys_to_virt((unsigned long)(x)));
+	return (void *)__phys_to_virt(x);
 }
 
 /*
  * Drivers should NOT use these either.
  */
 #define __pa(x)			__virt_to_phys((unsigned long)(x))
-#define __va(x)			((void *)__phys_to_virt((unsigned long)(x)))
+#define __va(x)			((void *)__phys_to_virt((phys_addr_t)(x)))
 #define pfn_to_kaddr(pfn)	__va((pfn) << PAGE_SHIFT)
 
 /*

@@ -36,12 +36,13 @@
 
 #include "mm.h"
 
-static unsigned long phys_initrd_start __initdata = 0;
+static phys_addr_t phys_initrd_start __initdata = 0;
 static unsigned long phys_initrd_size __initdata = 0;
 
 static int __init early_initrd(char *p)
 {
-	unsigned long start, size;
+	phys_addr_t start;
+	unsigned long size;
 	char *endp;
 
 	start = memparse(p, &endp);

@@ -347,14 +348,14 @@ void __init arm_memblock_init(struct meminfo *mi, struct machine_desc *mdesc)
 #ifdef CONFIG_BLK_DEV_INITRD
 	if (phys_initrd_size &&
 	    !memblock_is_region_memory(phys_initrd_start, phys_initrd_size)) {
-		pr_err("INITRD: 0x%08lx+0x%08lx is not a memory region - disabling initrd\n",
-		       phys_initrd_start, phys_initrd_size);
+		pr_err("INITRD: 0x%08llx+0x%08lx is not a memory region - disabling initrd\n",
+		       (u64)phys_initrd_start, phys_initrd_size);
 		phys_initrd_start = phys_initrd_size = 0;
 	}
 	if (phys_initrd_size &&
 	    memblock_is_region_reserved(phys_initrd_start, phys_initrd_size)) {
-		pr_err("INITRD: 0x%08lx+0x%08lx overlaps in-use memory region - disabling initrd\n",
-		       phys_initrd_start, phys_initrd_size);
+		pr_err("INITRD: 0x%08llx+0x%08lx overlaps in-use memory region - disabling initrd\n",
+		       (u64)phys_initrd_start, phys_initrd_size);
 		phys_initrd_start = phys_initrd_size = 0;
 	}
 	if (phys_initrd_size) {

@@ -919,15 +919,12 @@ void __init sanity_check_meminfo(void)
 		struct membank *bank = &meminfo.bank[j];
 		*bank = meminfo.bank[i];
 
-		if (bank->start > ULONG_MAX)
-			highmem = 1;
-
-#ifdef CONFIG_HIGHMEM
 		if (bank->start >= vmalloc_limit)
 			highmem = 1;
 
 		bank->highmem = highmem;
 
+#ifdef CONFIG_HIGHMEM
 		/*
 		 * Split those memory banks which are partially overlapping
 		 * the vmalloc area greatly simplifying things later.

@@ -950,8 +947,6 @@ void __init sanity_check_meminfo(void)
 			bank->size = vmalloc_limit - bank->start;
 		}
 #else
-		bank->highmem = highmem;
-
 		/*
 		 * Highmem banks not allowed with !CONFIG_HIGHMEM.
 		 */

@@ -964,18 +959,6 @@ void __init sanity_check_meminfo(void)
 		}
 
 		/*
-		 * Check whether this memory bank would entirely overlap
-		 * the vmalloc area.
-		 */
-		if (bank->start >= vmalloc_limit) {
-			printk(KERN_NOTICE "Ignoring RAM at %.8llx-%.8llx "
-			       "(vmalloc region overlap).\n",
-			       (unsigned long long)bank->start,
-			       (unsigned long long)bank->start + bank->size - 1);
-			continue;
-		}
-
-		/*
 		 * Check whether this memory bank would partially overlap
 		 * the vmalloc area.
 		 */

@@ -207,6 +207,18 @@ config ARM_PATCH_PHYS_VIRT
 	  this feature (eg, building a kernel for a single machine) and
 	  you need to shrink the kernel to the minimal size.
 
+config ARM_RUNTIME_PATCH_TEST
+	bool "Self test runtime patching mechanism" if ARM_RUNTIME_PATCH
+	default y
+	help
+	  Select this to enable init time self checking for the runtime kernel
+	  patching mechanism.  This enables an ISA specific set of tests that
+	  ensure that the instructions generated by the patch process are
+	  consistent with those generated by the assembler at compile time.
+
+	  Only disable this option if you need to shrink the kernel to the
+	  minimal size.
+
 config NEED_MACH_IO_H
 	bool
 	help

@@ -168,6 +168,78 @@ static int apply_patch_imm8(const struct patch_info *p)
 	return 0;
 }
 
+#ifdef CONFIG_ARM_RUNTIME_PATCH_TEST
+
+struct patch_test_imm8 {
+	u16	imm;
+	u16	shift;
+	u32	insn;
+};
+
+static void __init __used __naked __patch_test_code_imm8(void)
+{
+	__asm__ __volatile__ (
+
+		/* a single test case */
+		"	.macro		test_one, imm, sft\n"
+		"	.hword		\\imm\n"
+		"	.hword		\\sft\n"
+		"	add		r1, r2, #(\\imm << \\sft)\n"
+		"	.endm\n"
+
+		/* a sequence of tests at 'inc' increments of shift */
+		"	.macro		test_seq, imm, sft, max, inc\n"
+		"	test_one	\\imm, \\sft\n"
+		"	.if		\\sft < \\max\n"
+		"	test_seq	\\imm, (\\sft + \\inc), \\max, \\inc\n"
+		"	.endif\n"
+		"	.endm\n"
+
+		/* an empty record to mark the end */
+		"	.macro		test_end\n"
+		"	.hword		0, 0\n"
+		"	.word		0\n"
+		"	.endm\n"
+
+		/* finally generate the test sequences */
+		"	test_seq	0x41, 0, 24, 1\n"
+		"	test_seq	0x81, 0, 24, 2\n"
+		"	test_end\n"
+		: : : "r1", "r2", "cc");
+}
+
+static void __init test_patch_imm8(void)
+{
+	u32 test_code_addr = (u32)(&__patch_test_code_imm8);
+	struct patch_test_imm8 *test = (void *)(test_code_addr & ~1);
+	u32 ninsn, insn, patched_insn;
+	int i, err;
+
+	insn = test[0].insn;
+	for (i = 0; test[i].insn; i++) {
+		err = do_patch_imm8(insn, test[i].imm << test[i].shift, &ninsn);
+		__patch_text(&patched_insn, ninsn);
+
+		if (err) {
+			pr_err("rtpatch imm8: failed@imm %x, shift %d\n",
+			       test[i].imm, test[i].shift);
+		} else if (patched_insn != test[i].insn) {
+			pr_err("rtpatch imm8: failed, need %x got %x\n",
+			       test[i].insn, patched_insn);
+		} else {
+			pr_debug("rtpatch imm8: imm %x, shift %d, %x -> %x\n",
+				 test[i].imm, test[i].shift, insn,
+				 patched_insn);
+		}
+	}
+}
+
+static void __init runtime_patch_test(void)
+{
+	test_patch_imm8();
+}
+#endif
+
 int runtime_patch(const void *table, unsigned size)
 {
 	const struct patch_info *p = table, *end = (table + size);

@@ -189,5 +261,8 @@ void __init runtime_patch_kernel(void)
 	const void *start = &__runtime_patch_table_begin;
 	const void *end   = &__runtime_patch_table_end;
 
+#ifdef CONFIG_ARM_RUNTIME_PATCH_TEST
+	runtime_patch_test();
+#endif
 	BUG_ON(runtime_patch(start, end - start));
 }

@@ -612,7 +612,8 @@ static void __init alloc_init_section(pud_t *pud, unsigned long addr,
 }
 
 static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
-	unsigned long end, unsigned long phys, const struct mem_type *type)
+				  unsigned long end, phys_addr_t phys,
+				  const struct mem_type *type)
 {
 	pud_t *pud = pud_offset(pgd, addr);
 	unsigned long next;

@@ -62,6 +62,9 @@ config ARM
 config ARM_HAS_SG_CHAIN
 	bool
 
+config ARM_RUNTIME_PATCH
+	bool
+
 config NEED_SG_DMA_LENGTH
 	bool
 

@@ -39,9 +39,16 @@ struct mod_arch_specific {
 #define MODULE_ARCH_VERMAGIC_ARMTHUMB ""
 #endif
 
+#ifdef CONFIG_ARM_RUNTIME_PATCH
+#define MODULE_ARCH_VERMAGIC_RT_PATCH "rt-patch "
+#else
+#define MODULE_ARCH_VERMAGIC_RT_PATCH ""
+#endif
+
 #define MODULE_ARCH_VERMAGIC \
 	MODULE_ARCH_VERMAGIC_ARMVSN \
 	MODULE_ARCH_VERMAGIC_ARMTHUMB \
+	MODULE_ARCH_VERMAGIC_RT_PATCH \
 	MODULE_ARCH_VERMAGIC_P2V
 
 #endif /* _ASM_ARM_MODULE_H */

@@ -0,0 +1,208 @@
+/*
+ * arch/arm/include/asm/runtime-patch.h
+ * Note: this file should not be included by non-asm/.h files
+ *
+ * Copyright 2012 Texas Instruments, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef __ASM_ARM_RUNTIME_PATCH_H
+#define __ASM_ARM_RUNTIME_PATCH_H
+
+#include <linux/stringify.h>
+
+#ifndef __ASSEMBLY__
+
+#ifdef CONFIG_ARM_RUNTIME_PATCH
+
+struct patch_info {
+	void		*insn;
+	u16		 type;
+	u8		 insn_size;
+	u8		 data_size;
+	u32		 data[0];
+};
+
+#define PATCH_IMM8		0x0001
+struct patch_info_imm8 {
+	u32		*imm;
+	u32		 insn;
+};
+
+#define patch_next(p)	((void *)(p) + sizeof(*(p)) + (p)->data_size)
+#define patch_data(p)	((void *)&(p)->data[0])
+
+#define patch_stub(type, code, patch_data, ...)				\
+	__asm__("@ patch stub\n"					\
+		"1:\n"							\
+		code							\
+		"2:\n"							\
+		"	.pushsection .runtime.patch.table, \"a\"\n"	\
+		"3:\n"							\
+		"	.word 1b\n"					\
+		"	.hword (" __stringify(type) ")\n"		\
+		"	.byte (2b-1b)\n"				\
+		"	.byte (5f-4f)\n"				\
+		"4:\n"							\
+		patch_data						\
+		"	.align\n"					\
+		"5:\n"							\
+		"	.popsection\n"					\
+		__VA_ARGS__)
+
+#define early_patch_stub(type, code, pad, patch_data, ...)		\
+	__asm__("@ patch stub\n"					\
+		"1:\n"							\
+		"	b	6f\n"					\
+		"	.fill	" __stringify(pad) ", 1, 0\n"		\
+		"2:\n"							\
+		"	.pushsection .runtime.patch.table, \"a\"\n"	\
+		"3:\n"							\
+		"	.word 1b\n"					\
+		"	.hword (" __stringify(type) ")\n"		\
+		"	.byte (2b-1b)\n"				\
+		"	.byte (5f-4f)\n"				\
+		"4:\n"							\
+		patch_data						\
+		"	.align\n"					\
+		"5:\n"							\
+		"	.popsection\n"					\
+		"	.pushsection .runtime.patch.code, \"ax\"\n"	\
+		"6:\n"							\
+		code							\
+		"	b 2b\n"						\
+		"	.popsection\n"					\
+		__VA_ARGS__)
+
+/* constant used to force encoding */
+#define __IMM8		(0x81 << 24)
+
+/*
+ * patch_imm8() - init-time specialized binary operation (imm8 operand)
+ *		  This effectively does: to = from "insn" sym,
+ *		  where the value of sym is fixed@init-time, and is patched
+ *		  in as an immediate operand.  This value must be
+ *		  representible as an 8-bit quantity with an optional
+ *		  rotation.
+ *
+ *		  The stub code produced by this variant is non-functional
+ *		  prior to patching.  Use early_patch_imm8() if you need the
+ *		  code to be functional early on in the init sequence.
+ */
+#define patch_imm8(_insn, _to, _from, _sym, _ofs)			\
+	patch_stub(							\
+		/* type */						\
+			PATCH_IMM8,					\
+		/* code */						\
+			_insn "	%[to], %[from], %[imm]\n",		\
+		/* patch_data */					\
+			".long " __stringify(_sym + _ofs) "\n"		\
+			_insn "	%[to], %[from], %[imm]\n",		\
+		/* operands */						\
+			: [to]	 "=r" (_to)				\
+			: [from] "r"  (_from),				\
+			  [imm]	 "I"  (__IMM8),				\
+				 "i"  (&(_sym))				\
+			: "cc")
+
+/*
+ * patch_imm8_mov() - same as patch_imm8(), but for mov/mvn instructions
+ */
+#define patch_imm8_mov(_insn, _to, _sym, _ofs)				\
+	patch_stub(							\
+		/* type */						\
+			PATCH_IMM8,					\
+		/* code */						\
+			_insn "	%[to], %[imm]\n",			\
+		/* patch_data */					\
+			".long " __stringify(_sym + _ofs) "\n"		\
+			_insn "	%[to], %[imm]\n",			\
+		/* operands */						\
+			: [to]	"=r" (_to)				\
+			: [imm]	"I"  (__IMM8),				\
+				"i"  (&(_sym))				\
+			: "cc")
+
+/*
+ * early_patch_imm8() - early functional variant of patch_imm8() above.  The
+ *			same restrictions on the constant apply here.  This
+ *			version emits workable (albeit inefficient) code at
+ *			compile-time, and therefore functions even prior to
+ *			patch application.
+ */
+#define early_patch_imm8(_insn, _to, _from, _sym, _ofs)			\
+do {									\
+	unsigned long __tmp;						\
+	early_patch_stub(						\
+		/* type */						\
+			PATCH_IMM8,					\
+		/* code */						\
+			"ldr	%[tmp], =" __stringify(_sym + _ofs) "\n"\
+			"ldr	%[tmp], [%[tmp]]\n"			\
+			_insn "	%[to], %[from], %[tmp]\n",		\
+		/* pad */						\
+			0,						\
+		/* patch_data */					\
+			".long " __stringify(_sym + _ofs) "\n"		\
+			_insn "	%[to], %[from], %[imm]\n",		\
+		/* operands */						\
+			: [to]	 "=r"  (_to),				\
+			  [tmp]	 "=&r" (__tmp)				\
+			: [from] "r"   (_from),				\
+			  [imm]	 "I"   (__IMM8),			\
+				 "i"   (&(_sym))			\
+			: "cc");					\
+} while (0)
+
+#define early_patch_imm8_mov(_insn, _to, _sym, _ofs)			\
+do {									\
+	unsigned long __tmp;						\
+	early_patch_stub(						\
+		/* type */						\
+			PATCH_IMM8					\
+		/* code */						\
+			"ldr	%[tmp], =" __stringify(_sym + _ofs) "\n"\
+			"ldr	%[tmp], [%[tmp]]\n"			\
+			_insn "	%[to], %[tmp]\n",			\
+		/* pad */						\
+			0,						\
+		/* patch_data */					\
+			".long " __stringify(_sym + _ofs) "\n"		\
+			_insn " %[to], %[imm]\n",			\
+		/* operands */						\
+			: [to]	"=r"  (_to),				\
+			  [tmp]	"=&r" (__tmp)				\
+			: [imm]	"I"   (__IMM8),				\
+				"i"   (&(_sym))				\
+			: "cc");					\
+} while (0)
+
+int runtime_patch(const void *table, unsigned size);
+void runtime_patch_kernel(void);
+
+#else
+
+static inline int runtime_patch(const void *table, unsigned size)
+{
+	return 0;
+}
+
+static inline void runtime_patch_kernel(void)
+{
+}
+
+#endif /* CONFIG_ARM_RUNTIME_PATCH */
+
+#endif /* __ASSEMBLY__ */
+
+#endif /* __ASM_ARM_RUNTIME_PATCH_H */

@@ -80,5 +80,6 @@ endif
 head-y			:= head$(MMUEXT).o
 obj-$(CONFIG_DEBUG_LL)	+= debug.o
 obj-$(CONFIG_EARLY_PRINTK)	+= early_printk.o
+obj-$(CONFIG_ARM_RUNTIME_PATCH)	+= runtime-patch.o patch.o
 
 extra-y := $(head-y) vmlinux.lds

@@ -24,6 +24,7 @@
 #include <asm/sections.h>
 #include <asm/smp_plat.h>
 #include <asm/unwind.h>
+#include <asm/runtime-patch.h>
 
 #ifdef CONFIG_XIP_KERNEL
 /*

@@ -276,7 +277,7 @@ int module_finalize(const Elf32_Ehdr *hdr, const Elf_Shdr *sechdrs,
 	const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
 	const Elf_Shdr *sechdrs_end = sechdrs + hdr->e_shnum;
 	struct mod_unwind_map maps[ARM_SEC_MAX];
-	int i;
+	int i, err;
 
 	memset(maps, 0, sizeof(maps));
 

@@ -321,6 +322,12 @@ int module_finalize(const Elf32_Ehdr *hdr, const Elf_Shdr *sechdrs,
 	if (s)
 		fixup_pv_table((void *)s->sh_addr, s->sh_size);
 #endif
+	s = find_mod_section(hdr, sechdrs, ".runtime.patch.table");
+	if (s) {
+		err = runtime_patch((void *)s->sh_addr, s->sh_size);
+		if (err)
+			return err;
+	}
 	s = find_mod_section(hdr, sechdrs, ".alt.smp.init");
 	if (s && !is_smp())
 #ifdef CONFIG_SMP_ON_UP

@@ -0,0 +1,193 @@
+/*
+ * arch/arm/kernel/runtime-patch.c
+ *
+ * Copyright 2012 Texas Instruments, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <linux/kernel.h>
+#include <linux/sched.h>
+
+#include <asm/opcodes.h>
+#include <asm/cacheflush.h>
+#include <asm/runtime-patch.h>
+
+#include "patch.h"
+
+static inline void flush_icache_insn(void *insn_ptr, int bytes)
+{
+	unsigned long insn_addr = (unsigned long)insn_ptr;
+	flush_icache_range(insn_addr, insn_addr + bytes - 1);
+}
+
+#ifdef CONFIG_THUMB2_KERNEL
+
+static int do_patch_imm8(u32 insn, u32 imm, u32 *ninsn)
+{
+	u32 op, rot, val;
+	const u32 supported_ops = (BIT(0)  | /* and */
+				   BIT(1)  | /* bic */
+				   BIT(2)  | /* orr/mov */
+				   BIT(3)  | /* orn/mvn */
+				   BIT(4)  | /* eor */
+				   BIT(8)  | /* add */
+				   BIT(10) | /* adc */
+				   BIT(11) | /* sbc */
+				   BIT(12) | /* sub */
+				   BIT(13)); /* rsb */
+
+	insn = __mem_to_opcode_thumb32(insn);
+
+	if (!__opcode_is_thumb32(insn)) {
+		pr_err("patch: invalid thumb2 insn %08x\n", insn);
+		return -EINVAL;
+	}
+
+	/* allow only data processing (immediate)
+	 * 1111 0x0x xxx0 xxxx 0xxx xxxx xxxx xxxx */
+	if ((insn & 0xfa008000) != 0xf0000000) {
+		pr_err("patch: unknown insn %08x\n", insn);
+		return -EINVAL;
+	}
+
+	/* extract op code */
+	op = (insn >> 21) & 0xf;
+
+	/* disallow unsupported opcodes */
+	if ((supported_ops & BIT(op)) == 0) {
+		pr_err("patch: unsupported opcode %x\n", op);
+		return -EINVAL;
+	}
+
+	if (imm <= 0xff) {
+		rot = 0;
+		val = imm;
+	} else {
+		rot = 32 - fls(imm); /* clz */
+		if (imm & ~(0xff000000 >> rot)) {
+			pr_err("patch: constant overflow %08x\n", imm);
+			return -EINVAL;
+		}
+		val  = (imm >> (24 - rot)) & 0x7f;
+		rot += 8; /* encoded i:imm3:a */
+
+		/* pack least-sig rot bit into most-sig val bit */
+		val |= (rot & 1) << 7;
+		rot >>= 1;
+	}
+
+	*ninsn  = insn & ~(BIT(26) | 0x7 << 12 | 0xff);
+	*ninsn |= (rot >> 3) << 26;	/* field "i" */
+	*ninsn |= (rot & 0x7) << 12;	/* field "imm3" */
+	*ninsn |= val;
+
+	return 0;
+}
+
+#else
+
+static int do_patch_imm8(u32 insn, u32 imm, u32 *ninsn)
+{
+	u32 rot, val, op;
+
+	insn = __mem_to_opcode_arm(insn);
+
+	/* disallow special unconditional instructions
+	 * 1111 xxxx xxxx xxxx xxxx xxxx xxxx xxxx */
+	if ((insn >> 24) == 0xf) {
+		pr_err("patch: unconditional insn %08x\n", insn);
+		return -EINVAL;
+	}
+
+	/* allow only data processing (immediate)
+	 * xxxx 001x xxxx xxxx xxxx xxxx xxxx xxxx */
+	if (((insn >> 25) & 0x3) != 1) {
+		pr_err("patch: unknown insn %08x\n", insn);
+		return -EINVAL;
+	}
+
+	/* extract op code */
+	op = (insn >> 20) & 0x1f;
+
+	/* disallow unsupported 10xxx op codes */
+	if (((op >> 3) & 0x3) == 2) {
+		pr_err("patch: unsupported opcode %08x\n", insn);
+		return -EINVAL;
+	}
+
+	rot = imm ? __ffs(imm) / 2 : 0;
+	val = imm >> (rot * 2);
+	rot = (-rot) & 0xf;
+
+	/* does this fit in 8-bit? */
+	if (val > 0xff) {
+		pr_err("patch: constant overflow %08x\n", imm);
+		return -EINVAL;
+	}
+
+	/* patch in new immediate and rotation */
+	*ninsn = (insn & ~0xfff) | (rot << 8) | val;
+
+	return 0;
+}
+
+#endif	/* CONFIG_THUMB2_KERNEL */
+
+static int apply_patch_imm8(const struct patch_info *p)
+{
+	u32 *insn_ptr = p->insn, ninsn;
+	int count = p->insn_size / sizeof(u32);
+	const struct patch_info_imm8 *info;
+	int err;
+
+
+	if (count <= 0 || p->data_size != count * sizeof(*info)) {
+		pr_err("patch: bad patch, insn size %d, data size %d\n",
+		       p->insn_size, p->data_size);
+		return -EINVAL;
+	}
+
+	for (info = patch_data(p); count; count--, info++, insn_ptr++) {
+		err = do_patch_imm8(info->insn, *info->imm, &ninsn);
+		if (err)
+			return err;
+		__patch_text(insn_ptr, ninsn);
+	}
+
+
+	return 0;
+}
+
+int runtime_patch(const void *table, unsigned size)
+{
+	const struct patch_info *p = table, *end = (table + size);
+
+	for (p = table; p < end; p = patch_next(p)) {
+		int err = -EINVAL;
+
+		if (p->type == PATCH_IMM8)
+			err = apply_patch_imm8(p);
+		if (err)
+			return err;
+	}
+	return 0;
+}
+
+void __init runtime_patch_kernel(void)
+{
+	extern unsigned __runtime_patch_table_begin, __runtime_patch_table_end;
+	const void *start = &__runtime_patch_table_begin;
+	const void *end   = &__runtime_patch_table_end;
+
+	BUG_ON(runtime_patch(start, end - start));
+}

@@ -53,6 +53,7 @@
 #include <asm/traps.h>
 #include <asm/unwind.h>
 #include <asm/memblock.h>
+#include <asm/runtime-patch.h>
 
 #include "atags.h"
 #include "tcm.h"

@@ -764,6 +765,8 @@ void __init setup_arch(char **cmdline_p)
 
 	if (mdesc->init_early)
 		mdesc->init_early();
+
+	runtime_patch_kernel();
 }
 
 

@@ -167,6 +167,16 @@ SECTIONS
 		*(.pv_table)
 		__pv_table_end = .;
 	}
+	.init.runtime_patch_table : {
+		__runtime_patch_table_begin = .;
+		*(.runtime.patch.table)
+		__runtime_patch_table_end = .;
+	}
+	.init.runtime_patch_code : {
+		__runtime_patch_code_begin = .;
+		*(.runtime.patch.code)
+		__runtime_patch_code_end = .;
+	}
 	.init.data : {
 #ifndef CONFIG_XIP_KERNEL
 		INIT_DATA

@@ -195,6 +195,7 @@ config ARM_PATCH_PHYS_VIRT
 	default y
 	depends on !XIP_KERNEL && MMU
 	depends on !ARCH_REALVIEW || !SPARSEMEM
+	select ARM_RUNTIME_PATCH
 	help
 	  Patch phys-to-virt and virt-to-phys translation functions at
 	  boot and module load time according to the position of the

@@ -18,6 +18,8 @@
 #include <linux/types.h>
 #include <linux/sizes.h>
 
+#include <asm/runtime-patch.h>
+
 #ifdef CONFIG_NEED_MACH_MEMORY_H
 #include <mach/memory.h>
 #endif

@@ -151,35 +153,21 @@
 #ifndef __virt_to_phys
 #ifdef CONFIG_ARM_PATCH_PHYS_VIRT
 
-/*
- * Constants used to force the right instruction encodings and shifts
- * so that all we need to do is modify the 8-bit constant field.
- */
-#define __PV_BITS_31_24	0x81000000
-
-extern unsigned long __pv_phys_offset;
-#define PHYS_OFFSET __pv_phys_offset
-
-#define __pv_stub(from,to,instr,type)			\
-	__asm__("@ __pv_stub\n"				\
-	"1:	" instr "	%0, %1, %2\n"		\
-	"	.pushsection .pv_table,\"a\"\n"		\
-	"	.long	1b\n"				\
-	"	.popsection\n"				\
-	: "=r" (to)					\
-	: "r" (from), "I" (type))
+extern unsigned long	__pv_offset;
+extern unsigned long	__pv_phys_offset;
+#define PHYS_OFFSET	__virt_to_phys(PAGE_OFFSET)
 
 static inline unsigned long __virt_to_phys(unsigned long x)
 {
 	unsigned long t;
-	__pv_stub(x, t, "add", __PV_BITS_31_24);
+	early_patch_imm8("add", t, x, __pv_offset, 0);
 	return t;
 }
 
 static inline unsigned long __phys_to_virt(unsigned long x)
 {
 	unsigned long t;
-	__pv_stub(x, t, "sub", __PV_BITS_31_24);
+	early_patch_imm8("sub", t, x, __pv_offset, 0);
 	return t;
 }
 #else

@@ -152,7 +152,3 @@ EXPORT_SYMBOL(mcount);
 #endif
 EXPORT_SYMBOL(__gnu_mcount_nc);
 #endif
-
-#ifdef CONFIG_ARM_PATCH_PHYS_VIRT
-EXPORT_SYMBOL(__pv_phys_offset);
-#endif

@@ -117,7 +117,7 @@ ENTRY(stext)
 	bl	__fixup_smp
 #endif
 #ifdef CONFIG_ARM_PATCH_PHYS_VIRT
-	bl	__fixup_pv_table
+	bl	__fixup_pv_offsets
 #endif
 	bl	__create_page_tables
 

@@ -511,92 +511,29 @@ ENDPROC(fixup_smp)
 
 #ifdef CONFIG_ARM_PATCH_PHYS_VIRT
 
-/* __fixup_pv_table - patch the stub instructions with the delta between
- * PHYS_OFFSET and PAGE_OFFSET, which is assumed to be 16MiB aligned and
- * can be expressed by an immediate shifter operand. The stub instruction
- * has a form of '(add|sub) rd, rn, #imm'.
+/*
+ * __fixup_pv_offsets - update __pv_offset and __pv_phys_offset based on the
+ * runtime location of the kernel.
  */
 	__HEAD
-__fixup_pv_table:
+__fixup_pv_offsets:
 	adr	r0, 1f
-	ldmia	r0, {r3-r5, r7}
+	ldmia	r0, {r3-r6}
 	sub	r3, r0, r3	@ PHYS_OFFSET - PAGE_OFFSET
-	add	r4, r4, r3	@ adjust table start address
-	add	r5, r5, r3	@ adjust table end address
-	add	r7, r7, r3	@ adjust __pv_phys_offset address
-	str	r8, [r7]	@ save computed PHYS_OFFSET to __pv_phys_offset
-	mov	r6, r3, lsr #24	@ constant for add/sub instructions
-	teq	r3, r6, lsl #24 @ must be 16MiB aligned
-THUMB(	it	ne		@ cross section branch )
-	bne	__error
-	str	r6, [r7, #4]	@ save to __pv_offset
-	b	__fixup_a_pv_table
-ENDPROC(__fixup_pv_table)
+	add	r4, r4, r3	@ virt_to_phys(__pv_phys_offset)
+	add	r5, r5, r3	@ virt_to_phys(__pv_offset)
+	add	r6, r6, r3	@ virt_to_phys(PAGE_OFFSET) = PHYS_OFFSET
+	str	r6, [r4]	@ save __pv_phys_offset
+	str	r3, [r5]	@ save __pv_offset
+	mov	pc, lr
+ENDPROC(__fixup_pv_offsets)
 
 	.align
 1:	.long	.
-	.long	__pv_table_begin
-	.long	__pv_table_end
-2:	.long	__pv_phys_offset
-
-	.text
-__fixup_a_pv_table:
-#ifdef CONFIG_THUMB2_KERNEL
-	lsls	r6, #24
-	beq	2f
-	clz	r7, r6
-	lsr	r6, #24
-	lsl	r6, r7
-	bic	r6, #0x0080
-	lsrs	r7, #1
-	orrcs	r6, #0x0080
-	orr	r6, r6, r7, lsl #12
-	orr	r6, #0x4000
-	b	2f
-1:	add     r7, r3
-	ldrh	ip, [r7, #2]
-	and	ip, 0x8f00
-	orr	ip, r6	@ mask in offset bits 31-24
-	strh	ip, [r7, #2]
-2:	cmp	r4, r5
-	ldrcc	r7, [r4], #4	@ use branch for delay slot
-	bcc	1b
-	bx	lr
-#else
-	b	2f
-1:	ldr	ip, [r7, r3]
-	bic	ip, ip, #0x000000ff
-	orr	ip, ip, r6	@ mask in offset bits 31-24
-	str	ip, [r7, r3]
-2:	cmp	r4, r5
-	ldrcc	r7, [r4], #4	@ use branch for delay slot
-	bcc	1b
-	mov	pc, lr
+	.long	__pv_phys_offset
+	.long	__pv_offset
+	.long	PAGE_OFFSET
 #endif
-ENDPROC(__fixup_a_pv_table)
-
-ENTRY(fixup_pv_table)
-	stmfd	sp!, {r4 - r7, lr}
-	ldr	r2, 2f			@ get address of __pv_phys_offset
-	mov	r3, #0			@ no offset
-	mov	r4, r0			@ r0 = table start
-	add	r5, r0, r1		@ r1 = table size
-	ldr	r6, [r2, #4]		@ get __pv_offset
-	bl	__fixup_a_pv_table
-	ldmfd	sp!, {r4 - r7, pc}
-ENDPROC(fixup_pv_table)
 
-	.align
-2:	.long	__pv_phys_offset
-
-	.data
-	.globl	__pv_phys_offset
-	.type	__pv_phys_offset, %object
-__pv_phys_offset:
-	.long	0
-	.size	__pv_phys_offset, . - __pv_phys_offset
-__pv_offset:
-	.long	0
-#endif
 
 #include "head-common.S"