ptwrite uprobes v2

@@ -90,6 +90,7 @@ interactions.
 .. toctree::
    :maxdepth: 1
 
+   ptwrite-uprobes
    user_events
    uprobetracer
 

@@ -0,0 +1,387 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+===============
+ptwrite uprobes
+===============
+
+.. contents:: :local:
+
+Introduction
+============
+
+A classic uprobe enters the kernel for each probe. That causes overhead
+when the probe is executed frequently.
+
+ptwrite uprobes instead rely on hardware tracing that doesn't enter
+the kernel. It uses the ``PTWRITE`` instruction available on modern
+Intel CPUs to log data to the Processor Trace buffer. Processor
+Trace is configured and recorded by Linux perf.
+
+There are some limitations of the scheme (see below)
+but it is a lot faster than classic uprobes.
+
+Performance
+===========
+
+Measured on the development kernel in a KVM guest (2 vCPUs) running
+on an Alder Lake laptop with a probe on a hot function called in a
+tight loop.
+
+    +---------------------------------------+----------------------+--------------------+------------------------+
+    | mode (2-arg probe)                    | PT off (% of classic)| full (% of classic)| snapshot (% of classic)|
+    +=======================================+======================+====================+========================+
+    | classic uprobe (tracefs)              | 100%                 | 100%               | 100%                   |
+    | classic uprobe (perf probe, tracefs)  | 104%                 |                    |                        |
+    | classic uprobe (perf probe, perf ring)|                      | 156%               |                        |
+    | ptwrite %nopace                       | 2%                   | 2%                 | 8%                     |
+    | ptwrite default                       | 7%                   | 7%                 | 11%                    |
+    | perf probe ``--ptwrite``              | 7%                   | 7%                 | 12%                    |
+    +---------------------------------------+----------------------+--------------------+------------------------+
+
+The percentages are normalized to the classic tracefs uprobe in each
+recording mode; lower values represent lower cost per hit.
+
+A ``%nopace`` probe costs roughly **2%** as much as a classic uprobe
+with PT off (about 98% less). The default pacing (on unless ``%nopace``
+is given) costs roughly 7% as much (about 93% less) in the same mode.
+The default pacing slows down the probes to avoid data loss when they are
+too tightly spaced.
+
+``snapshot`` refers to ``perf record`` snapshot mode (``-S``) which doesn't
+save the PT ring buffer constantly.
+
+
+Requirements
+============
+
+- An Intel CPU with Intel PT and PTWRITE. When running as a guest Intel PT
+  needs to be exposed to the guest.
+  PT/PTWRITE are available when ``/sys/devices/intel_pt/format/ptw`` exists.
+- A kernel with ``CONFIG_UPROBE_EVENTS`` enabled.
+
+Quick start (tracefs)
+=====================
+
+Pick a probe site, register a probe at its file offset, enable it, run the
+program under PT, decode.
+
+Example 1: probe an existing instruction (punning)
+--------------------------------------------------
+
+Build a small program and probe the entry of ``main``::
+
+    $ cat > t.c <<'EOF'
+    #include <stdio.h>
+
+    __attribute__((noinline, noipa)) static unsigned long
+    target(unsigned long a, unsigned long b)
+    {
+        return a * 31 + b;
+    }
+
+    int main(void)
+    {
+        unsigned long i, acc = 0;
+        for (i = 0; i < 100; i++)
+            acc += target(i, i + 1);
+        printf("acc=%lu\n", acc);
+        return 0;
+    }
+    EOF
+    $ gcc -O2 -no-pie -fno-inline -o t t.c
+
+``objdump -F`` prints the file offset of every instruction::
+
+    $ objdump -d -F t | sed -n "/<main> (File Offset/,+1p"
+    0000000000401040 <main> (File Offset: 0x1040):
+      401040:	55			push   %rbp
+
+``1040`` is the file offset of ``main``'s first instruction, exactly
+what the probe line needs. Register the probe there, enable it, run
+the program under PT and decode::
+
+    # echo "ptw:e t:0x1040 %di %si" > /sys/kernel/tracing/uprobe_events
+    # echo 1 > /sys/kernel/tracing/events/uprobes/e/enable
+    # perf record -e intel_pt/ptw=1,fup_on_ptw=1/u -o perf.data ./t
+    # perf script --itrace=qwe -s uprobe-ptwrite-decode.py -i perf.data
+    record 1: event=uprobes/e id=0x6c2 args=[1, 140728356930600]
+    summary: records=1 dropped=0 stray=0 unknown=0 errors=0
+
+Since it isn't a nop the instruction is "punned": byte 0 is changed
+to a jump to a trampoline that logs the data and returns to the
+previous execution.
+
+Punning is a probabilistic method that depends on the existing
+instruction bytes and the placement of the executable in memory.
+It has a high chance of success on PIE/PIC binaries, but tends
+to work poorly on non PIE main executables.
+When punning is not possible the probe is rejected at install
+time. Options in this case:
+
+- Move the probe site to a different instruction which may work.
+- Rebuild with -fPIE if the main executable is not PIE.
+- Enable or disable /proc/sys/kernel/randomize_va_space. If the
+  randomization is enabled it may also just work on a rerun of
+  the program.
+- Fall back to a classic uprobe.
+- Insert a 5-byte NOP which is always supported (see below).
+
+Example 2: an explicit 5-byte NOP (inline assembly)
+---------------------------------------------------
+
+Add a 5-byte NOP at the probe point::
+
+    $ cat > t.c <<'EOF'
+    #include <stdio.h>
+
+    __attribute__((noinline)) static unsigned long
+    target(unsigned long a, unsigned long b)
+    {
+        asm volatile(".byte 0x0f, 0x1f, 0x44, 0x00, 0x00"); /* nopl */
+        return a * 31 + b;
+    }
+
+    int main(void)
+    {
+        unsigned long i, acc = 0;
+        for (i = 0; i < 100; i++)
+            acc += target(i, i + 1);
+        printf("acc=%lu\n", acc);
+        return 0;
+    }
+    EOF
+    $ gcc -O2 -no-pie -o t t.c
+
+    $ objdump -d -F t | sed -n "/<target> (File Offset/,+1p"
+    0000000000401170 <target> (File Offset: 0x1170):
+      401170:	0f 1f 44 00 00		nopl   0x0(%rax,%rax,1)
+
+Probe it exactly like example 1::
+
+    # echo "ptw:e t:0x1170 %di %si" > /sys/kernel/tracing/uprobe_events
+    # echo 1 > /sys/kernel/tracing/events/uprobes/e/enable
+    # perf record -e intel_pt/ptw=1,fup_on_ptw=1/u -o perf.data ./t
+    # perf script --itrace=qwe -s uprobe-ptwrite-decode.py -i perf.data
+    record 99: event=uprobes/e id=0x6c2 args=[98, 99]
+    record 100: event=uprobes/e id=0x6c2 args=[99, 100]
+    summary: records=100 dropped=0 stray=0 unknown=0 errors=0
+
+Configuring ptwrite uprobes
+===========================
+
+ptwrite uprobes is configured like normal uprobes by writing
+commands to ``/sys/kernel/tracing/uprobe_events``.
+
+  ptw[:[GRP/][EVENT]] PATH:OFFSET [FETCHARGS] : set a ptwrite probe
+  -:[GRP/][EVENT]                             : clear a probe
+
+  GRP      : group name. If omitted, "uprobes" is the default (the
+             event appears under events/uprobes/).
+  EVENT    : event name. If omitted, one is generated from PATH+OFFSET.
+  PATH     : path to an executable or a library.
+  OFFSET   : file offset of the probe site (0x-prefixed hex, see above).
+  FETCHARGS: probe arguments, up to 8 (see "Argument syntax" below).
+
+After creating the ptwrite uprobe it becomes available with its name
+in ``/sys/kernel/tracing/uprobe_events``. There it can be enabled
+by writing 1 to its enable field. However it only logs data
+when a Linux perf PT recording session with ptw=1 is active.
+
+perf probe
+----------
+
+``perf probe --ptwrite -x <file>`` creates ptwrite uprobes instead of
+the classic trap-based ones. The example below uses SDT probes.
+
+(this requires installing systemtap-devel or an equivalent package)
+
+    $ cat > t.c <<'EOF'
+    #include <stdio.h>
+    #include <sys/sdt.h>
+    __attribute__((noinline, noclone)) static unsigned long
+    target(unsigned long a, unsigned long b)
+    {
+        unsigned long local = a * 2;
+        STAP_PROBE1(test, rarg, a);
+        STAP_PROBE1(test, carg, 42);
+        STAP_PROBE2(test, marg, &local, b);
+        return a * 31 + b;
+    }
+    int main(void)
+    {
+        unsigned long i, acc = 0;
+        for (i = 0; i < 20; i++) {
+            acc += target(i, i + 1);
+            asm volatile("pause");
+        }
+        printf("acc=%lu\n", acc);
+        return 0;
+    }
+    EOF
+    $ gcc -O2 -no-pie -o t t.c
+
+The first probe point (``rarg``) is a nop 9 bytes into ``target``::
+
+    $ objdump -d t | sed -n "/<target>:/,+3p"
+    0000000000401180 <target>:
+      401180:	48 8d 04 3f		lea    (%rdi,%rdi,1),%rax
+      401184:	48 89 44 24 f8		mov    %rax,-0x8(%rsp)
+      401189:	90			nop
+
+Probe it with ``perf probe --ptwrite`` using the function+offset
+form, then enable, capture and delete it like any ptwrite probe::
+
+    $ perf probe --ptwrite -x ./t --add "target+9 %di %si"
+    Added new event:
+      probe_t:target      (on target+9 in ./t with %di %si)
+
+    # the tracefs line it wrote:
+    # ptw:probe_t/target ./t:0x1189 arg1=%di arg2=%si
+
+    # echo 1 > /sys/kernel/tracing/events/probe_t/target/enable
+    # perf record -e intel_pt/ptw=1,fup_on_ptw=1/u -o perf.data ./t
+    # perf script --itrace=qwe -s uprobe-ptwrite-decode.py -i perf.data
+    record 1: event=probe_t/target id=0x6a9 args=[0, 1]
+    record 20: event=probe_t/target id=0x6a9 args=[19, 20]
+    summary: records=20 dropped=0 stray=0 unknown=0 errors=0
+    # perf probe -d probe_t:target
+
+A ``nop`` instruction, as used by SDT probes, is not guaranteed to
+be ptwrite patchable. It needs a 5-byte NOP, but it can
+often be punned. If punning fails, the kernel reports
+``failed to install`` and the probe has to be moved to another site.
+
+GCC's ``-fpatchable-function-entry=5`` may emit five one-byte NOPs.
+To use that site, add ``%multinop`` to the tracefs probe offset::
+
+    # echo "ptw:e t:0x1170%multinop %di %si" > /sys/kernel/tracing/uprobe_events
+
+The five-byte run must start at an 8-byte-aligned address because it is
+patched with an atomic eight-byte store. An unaligned ``%multinop`` site is
+rejected; without ``%multinop``, the run is treated as a pun and may not
+always succeed.
+
+perf probe uses the standard argument syntax for the ptwrite subset
+(registers, ``$stack``/``$stackN``, ``+disp(%reg)`` memory reads, and
+``\0x2a``-style constants). Strings, arrays and typed suffixes are not
+supported by the ptwrite stub and are rejected by the kernel.
+``%return`` is refused (ptwrite probes are entry-only), and the mode
+requires ``-x``. The probes are enabled, captured and deleted like
+classic probes (``perf probe -l``, ``perf probe -d``).
+They carry the default (LFENCE) pacing. ``%nopace`` cannot be selected
+through perf probe. Write the tracefs line by hand for that.
+
+Argument syntax
+---------------
+
+ptwrite uprobes only support a limited number of argument types
+compared to classic uprobes.
+
+``ptw:<name> <path>:<offset> <arg> ... [options]`` where each ``<arg>`` is one
+of:
+
+- ``%di``, ``%si``, ``%ax`` ...: a live register.
+- ``\IMM``: a fixed constant (stored in the stub), e.g. ``\0x42``.
+- ``$stack``: the stack pointer value (never faults).
+- ``$stackN``: the Nth stack slot (``[%rsp + 8N]``). ``u64`` uses an
+  8-byte load on the fault-fixup path; ``u32``/``s32``/``x32`` use a 4-byte
+  load.
+- ``+<disp>(%reg)``: read memory at ``[reg + disp]``. ``u64`` uses an
+  8-byte load; ``u32``/``s32``/``x32`` use a 4-byte load (``ptwritel``).
+  A bad address writes ``0``.
+
+Options
+-------
+
+``%nopace`` disables artificial slowdown of the probes. This can cause
+data loss when they are tightly spaced or have many arguments, but
+speeds up the probes (see the benchmark section above).
+
+``%multinop`` lets users probe a 5-byte nop sequence that is not one
+instruction. A program could jump to a later nop, which would break when
+the probe rewrites the site.
+
+However there is a common case where gcc's -fpatchable-function-entry=5
+generates 5 nops for each function that are convenient points
+for patching, and nobody jumps into the middle of them.
+
+The five one-byte NOP sequence must be 8-byte aligned.
+
+The encoding format
+===================
+
+Each probe writes a header and the arguments to the PT stream.
+
+The header is a 64-bit word. Each argument is one PTWRITE payload exposed by
+perf as a ``u64`` value.
+
+    header word:      bits 63..48  event id (matches the tracefs id in sysfs)
+                      bits 47..40  number of argument words
+                      bits 39..0   fixed magic 0x5054525731 ("PTRW1")
+    arguments:        one PTWRITE payload per FETCHARG
+
+If the program itself also executes own ``PTWRITE``, those values mix with the
+uprobe output in the stream. The decoder uses the header magic to identify
+uprobe records. Other values are printed as ``manual ptwrite:`` lines (with
+their IP when ``fup_on_ptw`` is set) and counted in the summary's ``stray``
+field.
+
+To also print the decoded branch stream alongside the records, add
+``b`` to the itrace options and drop the ``q``
+
+    # ``perf script --itrace=web -s uprobe-ptwrite-decode.py -i perf.data``
+
+Each decoded branch prints as a ``branch:`` line (from => to, with
+symbols where resolvable), interleaved with the probe records and any
+manual ptwrites in delivery order.
+
+Other events in the recording, including classic uprobes, tracepoints,
+and sample events, are printed as ``event:`` lines unless disabled
+by the decoder.
+
+Unsupported instructions for probes
+===================================
+
+The following instructions are always refused for instrumentation:
+
+- Traps: ``int3``, ``int1``, ``int imm8``, ``into``, ``iret``
+  because they save the IP.
+- System instructions: ``syscall``, ``sysenter``, ``sysexit``
+  for similar reasons.
+- Far control flow: ``jmp far``, ``call far``, and the indirect far
+  forms (call-far, jmp-far).
+- Relative branches: ``jmp rel8/rel32``, ``jcc rel8/rel32``,
+  ``loop*``, ``jecxz/jrcxz``, and ``call rel32``. Out-of-line execution
+  would change their relative targets, and a call's return address would
+  point into the stub.
+- Indirect ``call``: the return-address problem also applies to the
+  register and memory forms.
+- If the 5 byte area of the instruction crosses a page boundary it
+  currently cannot be probed (this applies to nop probes too).
+
+Other restrictions
+==================
+
+- Each probe needs 4K of process memory and roughly 1K extra in the kernel.
+- The probe pages are currently only freed on process exit.
+- Return probes (``%return``/``r:``): ptwrite probes are entry-only
+  and ``%return`` is refused.
+- The SDT reference counter (``(REF)``).
+- EBPF, perf actions, filters, event predicates, histograms, triggers,
+  profiling and similar advanced trace features are all not supported
+  since they would require a kernel entry. However some basic filtering
+  is possible at the perf recording level, for example limit the scope
+  to a CPU or to a process. PT also supports address filter ranges
+  that allow filtering by IP.
+- More than one probe at the same site
+- Only 4 and 8 byte memory references are supported.
+- Fetch argument variety: classic probes fetch strings
+  (``:string``/``:ustring``), arrays, bitfields, nested derefs,
+  ``$retval``, ``$comm`` and ``$argN``. Ptwrite probes only take live
+  registers, ``\IMM`` constants, ``$stack``/``$stackN`` and
+  ``+disp(%reg)`` memory reads. Memory reads are 8-byte words for ``u64``
+  and 4-byte words for ``u32``/``s32``/``x32``.
+  (some of this could be relaxed, but it would require a writable stack)
+- Like normal uprobes one byte of the instruction stream is overwritten
+  (or 5 bytes for the nop case). If the program reads its own code
+  it might see different values.

@@ -30,12 +30,23 @@ enum {
 struct uprobe_xol_ops;
 
 /*
- * Stub block array size. Worst case = 250 B (8 MEM args, rsp bases, fault
- * table); 288 leaves 38 B slack. A file-scope static_assert in
- * arch/x86/kernel/uprobes.c re-derives the worst case; prepare() also
- * enforces it with -E2BIG at runtime.
+ * Stub block size. Conservative worst case is 298 bytes: 9-byte header,
+ * one lead fence, eight 21-byte memory forms with one 3-byte fence each,
+ * a 16-byte original-instruction copy, a 5-byte return jump, alignment,
+ * and 66 bytes of data/fault metadata. 384 leaves room. A static_assert in
+ * arch/x86/kernel/uprobes.c checks the bound; prepare() also checks it with
+ * -E2BIG.
  */
-#define UPROBE_PTWRITE_STUB_SIZE	288
+#define UPROBE_PTWRITE_STUB_SIZE	384
+
+
+/*
+ * Word pacing: insert this many LFENCEs between emitted ptwrite words and
+ * before the first word, unless UPROBE_PTWRITE_FL_NO_LEAD_PACE is requested.
+ */
+#define UPROBE_PTWRITE_SERIALIZE_LFENCES	1	/* LFENCEs per word gap */
+/* The encoded LFENCE instruction occupies three bytes. */
+#define UPROBE_PTWRITE_LFENCE_SIZE	3
 
 /*
  * ptwrite probe state. The stub template (code + data slots) is built

@@ -49,6 +60,9 @@ struct uprobe_ptwrite_arch {
 	u8	jmp_off;	/* offset of the final jmp's rel32 field */
 	u8	ndata;		/* number of u64 data slots */
 	u8	orig[MAX_UINSN_BYTES];	/* pristine file bytes, before generic analysis */
+	u16	ft_off;		/* fault table offset within the block (0 if none) */
+	u8	nft;		/* number of fault entries */
+	bool	allow_nop_run;	/* accept a five-byte run of 0x90 */
 };
 
 /* Per-mm page holding generated ptwrite stub blocks, like tramp_mapping. */

@@ -1323,6 +1323,24 @@ static int ptwrite_emit_riprel(u8 *p, s32 disp)
 	return 9;
 }
 
+static int ptwrite_emit_lfence(u8 *p)
+{
+	*p++ = 0x0f;
+	*p++ = 0xae;
+	*p++ = 0xe8;	/* LFENCE */
+	return UPROBE_PTWRITE_LFENCE_SIZE;
+}
+
+/* The default pacing: one or more fences per word gap. */
+static int ptwrite_emit_lfences(u8 *p)
+{
+	int i;
+
+	for (i = 0; i < UPROBE_PTWRITE_SERIALIZE_LFENCES; i++)
+		p += ptwrite_emit_lfence(p);
+	return UPROBE_PTWRITE_SERIALIZE_LFENCES *
+		UPROBE_PTWRITE_LFENCE_SIZE;
+}
 bool arch_uprobe_ptwrite_supported(void)
 {
 	u32 eax, ebx, ecx, edx;

@@ -1411,13 +1429,20 @@ int arch_uprobe_ptwrite_fetch(struct uprobe_ptwrite_arg *a,
 }
 
 /*
- * Worst-case stub block: header ptwriteq (9) + max memory args (10 bytes
- * each, including a SIB byte) + final jmp (5), rounded up; data adds one
- * header slot and one slot per immediate. Keep the bound below the stub size.
+ * Worst-case paced stub before instruction punning: a 9-byte header, one
+ * lead fence, one fence after the header, one fence between each argument,
+ * the largest memory form (10 bytes), and the return jump. Data adds one
+ * header slot and one slot per immediate.
  */
-static_assert((((9 + UPROBE_PTWRITE_MAX_ARGS * 10 + 5 + 7) & ~7) +
-	       8 * (1 + UPROBE_PTWRITE_MAX_ARGS)) <= UPROBE_PTWRITE_STUB_SIZE,
-	       "worst-case ptwrite stub block exceeds UPROBE_PTWRITE_STUB_SIZE");
+static_assert((((9 + UPROBE_PTWRITE_SERIALIZE_LFENCES *
+				  UPROBE_PTWRITE_LFENCE_SIZE +
+			  UPROBE_PTWRITE_MAX_ARGS *
+				  (10 + UPROBE_PTWRITE_SERIALIZE_LFENCES *
+				   UPROBE_PTWRITE_LFENCE_SIZE) +
+			  5 + 7) & ~7) +
+		       8 * (1 + UPROBE_PTWRITE_MAX_ARGS)) <=
+		      UPROBE_PTWRITE_STUB_SIZE,
+		      "worst-case ptwrite stub block exceeds UPROBE_PTWRITE_STUB_SIZE");
 
 static bool ptwrite_has_room(const u8 *base, const u8 *p, size_t len)
 {

@@ -1439,6 +1464,7 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
 	unsigned int data_off;
 	unsigned int hdr_off = 0;
 	unsigned int imm_idx = 0, n_imm = 0;
+	bool paced = false;
 	u64 hdr;
 	int i;
 

@@ -1446,7 +1472,9 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
 		return -EINVAL;
 	if (desc->nargs > UPROBE_PTWRITE_MAX_ARGS)
 		return -E2BIG;
-	if (desc->flags & ~UPROBE_PTWRITE_FL_ALLOW_MEM)
+	if (desc->flags & ~(UPROBE_PTWRITE_FL_ALLOW_MEM |
+			     UPROBE_PTWRITE_FL_NO_LEAD_PACE |
+			     UPROBE_PTWRITE_FL_ALLOW_NOP_RUN))
 		return -EINVAL;
 
 	/* The generic registration path copied these bytes before this hook. */

@@ -1477,9 +1505,22 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
 		}
 	}
 
+	paced = !(desc->flags & UPROBE_PTWRITE_FL_NO_LEAD_PACE);
+	if (paced) {
+		PTW_NEED(UPROBE_PTWRITE_SERIALIZE_LFENCES *
+			 UPROBE_PTWRITE_LFENCE_SIZE);
+		p += ptwrite_emit_lfences(p);
+	}
+
 	/* header word emission (disp32 patched below) */
 	PTW_NEED(9);
+	hdr_off = p - code;
 	p += ptwrite_emit_riprel(p, 0);
+	if (paced) {
+		PTW_NEED(UPROBE_PTWRITE_SERIALIZE_LFENCES *
+			 UPROBE_PTWRITE_LFENCE_SIZE);
+		p += ptwrite_emit_lfences(p);
+	}
 
 	for (i = 0; i < desc->nargs; i++) {
 		switch (desc->args[i].src) {

@@ -1515,9 +1556,14 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
 			break;
 		}
 		}
+		if (paced && i + 1 < desc->nargs) {
+			PTW_NEED(UPROBE_PTWRITE_SERIALIZE_LFENCES *
+				 UPROBE_PTWRITE_LFENCE_SIZE);
+			p += ptwrite_emit_lfences(p);
+		}
 	}
 
-	/* final jmp back to probe+5; rel32 patched per-mm at install */
+	/* final jmp back to probe+len; rel32 patched per-mm at install */
 	PTW_NEED(5);
 	*p++ = 0xe9;
 	if (p - code > U8_MAX)

@@ -1549,6 +1595,7 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
 
 	ptw->stub_len = data_off + 8 * (1 + n_imm);
 	ptw->ndata = 1 + n_imm;
+	ptw->allow_nop_run = desc->flags & UPROBE_PTWRITE_FL_ALLOW_NOP_RUN;
 	return 0;
 }
 #undef PTW_NEED

@@ -210,6 +210,8 @@ enum uprobe_ptwrite_src {
 
 /* uprobe_ptwrite_desc.flags */
 #define UPROBE_PTWRITE_FL_ALLOW_MEM	BIT(0) /* SRC_MEM args enabled */
+#define UPROBE_PTWRITE_FL_NO_LEAD_PACE	BIT(1) /* don't slow down probes */
+#define UPROBE_PTWRITE_FL_ALLOW_NOP_RUN	BIT(2) /* accept five 1-byte NOPs */
 
 struct uprobe_ptwrite_arg {
 	u8	src;		/* enum uprobe_ptwrite_src */

@@ -635,6 +635,8 @@ static int __trace_uprobe_create(int argc, const char **argv)
 	enum probe_print_type ptype;
 	bool is_return = false;
 	bool is_ptwrite = false;
+	bool is_nopace = false;
+	bool is_nop_run = false;
 	int i, ret, arg_start = 2;
 
 	ref_ctr_offset = 0;

@@ -660,13 +662,6 @@ static int __trace_uprobe_create(int argc, const char **argv)
 
 	trlog = trace_probe_log_init("trace_uprobe", argc, argv);
 
-	if (argc - 2 > MAX_TRACE_ARGS ||
-	    (is_ptwrite && argc - 2 > UPROBE_PTWRITE_MAX_ARGS)) {
-		trace_probe_log_set_index(2);
-		trace_probe_log_err(0, TOO_MANY_ARGS);
-		return -E2BIG;
-	}
-
 	if (is_ptwrite)
 		event = argv[0][3] == ':' && argv[0][4] ?
 			&argv[0][4] : NULL;

@@ -728,9 +723,19 @@ static int __trace_uprobe_create(int argc, const char **argv)
 
 	/* Check if there is %return suffix */
 	tmp = strchr(arg, '%');
+	if (tmp && is_ptwrite && !strcmp(tmp, "%nopace")) {
+		*tmp = '\0';
+		is_nopace = true;
+		tmp = NULL;
+	}
 	if (tmp && is_ptwrite) {
-		trace_probe_log_err(tmp - filename, BAD_ADDR_SUFFIX);
-		return -EINVAL;
+		if (!strcmp(tmp, "%multinop")) {
+			*tmp = '\0';
+			is_nop_run = true;
+		} else {
+			trace_probe_log_err(tmp - filename, BAD_ADDR_SUFFIX);
+			return -EINVAL;
+		}
 	} else if (tmp) {
 		if (!strcmp(tmp, "%return")) {
 			*tmp = '\0';

@@ -747,6 +752,28 @@ static int __trace_uprobe_create(int argc, const char **argv)
 		trace_probe_log_err(arg - filename, BAD_UPROBE_OFFS);
 		return ret;
 	}
+	if (is_ptwrite && arg_start < argc &&
+	    !strcmp(argv[arg_start], "%nopace")) {
+		is_nopace = true;
+		arg_start++;
+	}
+	if (is_ptwrite) {
+		while (arg_start < argc && !strcmp(argv[arg_start], "%multinop")) {
+			is_nop_run = true;
+			arg_start++;
+		}
+		if (arg_start < argc && !strcmp(argv[arg_start], "%nopace")) {
+			is_nopace = true;
+			arg_start++;
+		}
+	}
+
+	if (argc - arg_start > MAX_TRACE_ARGS ||
+	    (is_ptwrite && argc - arg_start > UPROBE_PTWRITE_MAX_ARGS)) {
+		trace_probe_log_set_index(arg_start);
+		trace_probe_log_err(0, TOO_MANY_ARGS);
+		return -E2BIG;
+	}
 
 	/* setup a probe */
 	trace_probe_log_set_index(0);

@@ -782,8 +809,8 @@ static int __trace_uprobe_create(int argc, const char **argv)
 		kfree(tail);
 	}
 
-	argc -= 2;
-	argv += 2;
+	argc -= arg_start;
+	argv += arg_start;
 
 	tu = alloc_trace_uprobe(group, event, argc, is_return);
 	if (IS_ERR(tu)) {

@@ -806,7 +833,7 @@ static int __trace_uprobe_create(int argc, const char **argv)
 
 	/* parse arguments */
 	for (i = 0; i < argc; i++) {
-		trace_probe_log_set_index(i + 2);
+		trace_probe_log_set_index(i + arg_start);
 		ret = traceprobe_parse_probe_arg(&tu->tp, i, argv[i], ctx);
 		if (ret)
 			return ret;

@@ -814,13 +841,16 @@ static int __trace_uprobe_create(int argc, const char **argv)
 
 	if (is_ptwrite) {
 		if (!argc) {
-			trace_probe_log_set_index(2);
+			trace_probe_log_set_index(arg_start);
 			trace_probe_log_err(0, NO_ARG_BODY);
 			return -EINVAL;	/* core rejects desc->nargs == 0 */
 		}
 		tu->is_ptwrite = true;
 		tu->ptwrite_desc.nargs = argc;
-		tu->ptwrite_desc.flags = 0;
+		tu->ptwrite_desc.flags = is_nop_run ?
+			UPROBE_PTWRITE_FL_ALLOW_NOP_RUN : 0;
+		if (is_nopace)
+			tu->ptwrite_desc.flags |= UPROBE_PTWRITE_FL_NO_LEAD_PACE;
 		for (i = 0; i < argc; i++) {
 			ret = ptwrite_compile_arg(tu, i);
 			if (ret) {

@@ -17,6 +17,7 @@
  *                        m<N>[:<disp>][:<size>] = memory arg [reg + disp32],
  *                        size 4 (u32 load) or 8 (u64 load, default)
  *   event_id=0x1234      identifier carried in the PTW header word
+ *   allow_nop_run=1      accept five one-byte NOPs at the site
  */
 #include <linux/module.h>
 #include <linux/uprobes.h>

@@ -36,6 +37,10 @@ static ushort event_id = 0x1234;
 module_param(event_id, ushort, 0444);
 MODULE_PARM_DESC(event_id, "event id carried in the PTW header word");
 
+static bool allow_nop_run;
+module_param(allow_nop_run, bool, 0444);
+MODULE_PARM_DESC(allow_nop_run, "accept five one-byte NOPs at the site");
+
 static char *args = "r0";
 module_param(args, charp, 0444);
 MODULE_PARM_DESC(args, "comma-separated args: r<N> GPR, i<hex> immediate, m<N>[:disp][:4|8] memory");

@@ -142,6 +147,7 @@ static int __init uprobe_ptwrite_test_init(void)
 	int ret;
 
 	desc.event_id = event_id;
+	desc.flags = allow_nop_run ? UPROBE_PTWRITE_FL_ALLOW_NOP_RUN : 0;
 	ret = parse_probe_args();
 	if (ret)
 		return ret;

@@ -39,6 +39,8 @@ struct uprobe_xol_ops;
  */
 #define UPROBE_PTWRITE_STUB_SIZE	384
 
+/* the out-of-line original-instruction copy slot (x86 max insn length) */
+#define UPROBE_PTWRITE_COPY_SIZE	MAX_UINSN_BYTES
 
 /*
  * Word pacing: insert this many LFENCEs between emitted ptwrite words and

@@ -50,14 +52,19 @@ struct uprobe_xol_ops;
 
 /*
  * ptwrite probe state. The stub template (code + data slots) is built
- * once at registration (mm-independent except the final jmp's rel32, patched
- * per-mm at install). Block layout:
- *   [ptwriteq hdr(%rip)] [arg emissions] [jmp probe+5] [u64 slots: header, imms]
+ * once at registration. Only the final jmp's rel32 and the copy's
+ * disp/rel fields are patched per-mm at install. Block layout:
+ *   [ptwriteq hdr(%rip)] [arg emissions] [orig-insn copy]
+ *   [jmp probe+len] [u64 slots: header, imms]
  */
 struct uprobe_ptwrite_arch {
 	u8	stub[UPROBE_PTWRITE_STUB_SIZE];
 	u16	stub_len;	/* code + data, whole block */
 	u8	jmp_off;	/* offset of the final jmp's rel32 field */
+	u8	copy_off;	/* offset of the out-of-line instruction copy */
+	u8	len;		/* copy length (0 = drop); back-jmp = vaddr+len */
+	u8	disp_off;	/* rip-relative disp32 offset in the copy (0 = none) */
+	s32	disp;		/* original disp32 (delta-patched per-mm) */
 	u8	ndata;		/* number of u64 data slots */
 	u8	orig[MAX_UINSN_BYTES];	/* pristine file bytes, before generic analysis */
 	u16	ft_off;		/* fault table offset within the block (0 if none) */

@@ -72,6 +79,17 @@ struct uprobe_ptwrite_page {
 	struct page		*page;		/* stub blocks written via kmap */
 	unsigned long		vaddr;		/* mapping base */
 	u16			cursor;		/* next free block offset */
+	u16			nblocks;
+	struct {
+		u16 off;	/* block offset in the page */
+		u16 len;	/* generated block length */
+		u8  orig0;	/* original site byte 0 (pun restore) */
+		u8  pun;	/* instruction-pun mechanism (single-byte poke) */
+		u8  site_len;	/* original instruction length (pun identity) */
+		s32 site_off;	/* probe site - page base (idempotent reinstall) */
+		u8  site_insn[MAX_UINSN_BYTES];	/* original bytes (pun identity) */
+	} index[PAGE_SIZE / 32];	/* exact: min block = 32 B (nargs >= 1), */
+					/* so <= 128 blocks fit a page */
 };
 
 struct arch_uprobe {

@@ -1019,8 +1019,17 @@ static int verify_insn(struct page *page, unsigned long vaddr, uprobe_opcode_t *
 {
 	struct write_opcode_ctx *ctx = data;
 	uprobe_opcode_t old_opcode[OPT_INSN_SIZE];
+	int len;
 
-	uprobe_copy_from_page(page, ctx->base, old_opcode, OPT_INSN_SIZE);
+	/*
+	 * Byte-state checks need only the first byte. Optimized-state checks
+	 * inspect the complete ten-byte instruction.
+	 */
+	len = ctx->expect == EXPECT_OPTIMIZED ||
+		ctx->expect == EXPECT_SWBP_OPTIMIZED ? OPT_INSN_SIZE : 1;
+	if (PAGE_SIZE - (ctx->base & ~PAGE_MASK) < len)
+		return -1;
+	uprobe_copy_from_page(page, ctx->base, old_opcode, len);
 
 	switch (ctx->expect) {
 	case EXPECT_SWBP:

@@ -1450,12 +1459,19 @@ static bool ptwrite_has_room(const u8 *base, const u8 *p, size_t len)
 		       sizeof(((struct uprobe_ptwrite_arch *)0)->stub) - len;
 }
 
+static bool pun_site_is_nop(const u8 *orig, bool allow_nop_run);
+static int pun_classify_insn(struct insn *insn, u8 *disp_off, s32 *disp);
+static int pun_decode_site(struct inode *inode, struct file *file,
+			       loff_t offset, u8 *copy,
+			       u8 *disp_off, s32 *disp, bool allow_nop_run);
 #define PTW_NEED(_len) do { \
 		if (!ptwrite_has_room(code, p, (_len))) \
 			return -E2BIG; \
 	} while (0)
 
 int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
+				struct inode *inode, struct file *file,
+				loff_t offset,
 				const struct uprobe_ptwrite_desc *desc)
 {
 	struct uprobe_ptwrite_arch *ptw = &auprobe->ptwrite;

@@ -1466,7 +1482,7 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
 	unsigned int imm_idx = 0, n_imm = 0;
 	bool paced = false;
 	u64 hdr;
-	int i;
+	int i, ret;
 
 	if (!desc || desc->nargs == 0)
 		return -EINVAL;

@@ -1563,6 +1579,13 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
 		}
 	}
 
+	/* the out-of-line original-instruction copy slot (patched per-mm) */
+	PTW_NEED(UPROBE_PTWRITE_COPY_SIZE);
+	if (p - code > U8_MAX)
+		return -E2BIG;
+	ptw->copy_off = p - code;
+	p += UPROBE_PTWRITE_COPY_SIZE;
+
 	/* final jmp back to probe+len; rel32 patched per-mm at install */
 	PTW_NEED(5);
 	*p++ = 0xe9;

@@ -1596,6 +1619,15 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
 	ptw->stub_len = data_off + 8 * (1 + n_imm);
 	ptw->ndata = 1 + n_imm;
 	ptw->allow_nop_run = desc->flags & UPROBE_PTWRITE_FL_ALLOW_NOP_RUN;
+
+	ret = pun_decode_site(inode, file, offset, code + ptw->copy_off,
+				  &ptw->disp_off, &ptw->disp,
+				  ptw->allow_nop_run);
+	if (ret < 0)
+		return ret;
+	ptw->len = ret;
+	memset(code + ptw->copy_off + ptw->len, 0x90,
+	       UPROBE_PTWRITE_COPY_SIZE - ptw->len);
 	return 0;
 }
 #undef PTW_NEED

@@ -1654,15 +1686,10 @@ static unsigned long find_ptwrite_page_area(struct mm_struct *mm,
 }
 
 static struct uprobe_ptwrite_page *
-create_uprobe_ptwrite_page(struct mm_struct *mm, unsigned long vaddr)
+create_uprobe_ptwrite_page_at(struct mm_struct *mm, unsigned long area)
 {
 	struct uprobe_ptwrite_page *ptw;
 	struct vm_area_struct *vma;
-	unsigned long area;
-
-	area = find_ptwrite_page_area(mm, vaddr);
-	if (IS_ERR_VALUE(area))
-		return NULL;
 
 	mmap_assert_write_locked(mm);
 

@@ -1685,6 +1712,17 @@ create_uprobe_ptwrite_page(struct mm_struct *mm, unsigned long vaddr)
 	}
 	return ptw;
 }
+
+static struct uprobe_ptwrite_page *
+create_uprobe_ptwrite_page(struct mm_struct *mm, unsigned long vaddr)
+{
+	unsigned long area = find_ptwrite_page_area(mm, vaddr);
+
+	if (IS_ERR_VALUE(area) || security_mmap_addr(area))
+		return NULL;
+	return create_uprobe_ptwrite_page_at(mm, area);
+}
+
 static struct uprobe_ptwrite_page *
 get_uprobe_ptwrite_page(struct mm_struct *mm, unsigned long vaddr,
 			unsigned int len)

@@ -1714,34 +1752,131 @@ get_uprobe_ptwrite_page(struct mm_struct *mm, unsigned long vaddr,
 	return ptw;
 }
 
-/* Probe site must be a 5-byte NOP that does not cross a page boundary. */
-static int ptwrite_validate_site(const u8 *orig, unsigned long vaddr)
+/*
+ * A run of short NOPs is accepted only when requested. This validation does
+ * not make the three-phase poke safe for threads that already passed byte 0.
+ */
+static bool ptwrite_is_nop_run(const u8 *orig)
+{
+	return !memchr_inv(orig, 0x90, 5);
+}
+
+static bool pun_site_is_nop(const u8 *orig, bool allow_nop_run)
 {
 	struct insn insn;
 	int ret;
-	int off = 0;
 
-	/*
-	 * The 5 displaced bytes must be NOPs: either one 5-byte NOP
-	 * (nopl 0x0(%rax,%rax,1)) or a run of shorter NOPs summing to
-	 * exactly 5 (gcc -fpatchable-function-entry=5 emits 5 x 0x90 on
-	 * modern toolchains). Any non-NOP byte, or a NOP crossing the
-	 * 5-byte window, is rejected.
-	 */
-	while (off < 5) {
-		ret = insn_decode(&insn, orig + off, 5 - off, INSN_MODE_64);
-		if (ret < 0)
-			return -EINVAL;
-		if (insn.length < 1 || insn.length > 5 - off ||
-		    !insn_is_nop(&insn))
-			return -EINVAL;
-		off += insn.length;
+	ret = insn_decode(&insn, orig, 5, INSN_MODE_64);
+	if (ret < 0)
+		return false;
+	if (insn.length == 5 && insn_is_nop(&insn))
+		return true;
+	if (!allow_nop_run)
+		return false;
+	return ptwrite_is_nop_run(orig);
+}
+
+/* Identify the explicitly opted-in run of five one-byte NOPs. */
+static bool ptwrite_site_is_multinop(const u8 *orig, bool allow_nop_run)
+{
+	return allow_nop_run && ptwrite_is_nop_run(orig);
+}
+
+/*
+ * Classify the site's single instruction for out-of-line execution.
+ * Returns the length, or a negative errno when it cannot run safely out of
+ * line.
+ */
+static int pun_classify_insn(struct insn *insn, u8 *disp_off, s32 *disp)
+{
+	u8 op = insn->opcode.bytes[0];
+
+	switch (op) {
+	case 0xcc:	/* int3 */
+	case 0xcd:	/* int imm8 */
+	case 0xce:	/* into */
+	case 0xcf:	/* iret */
+	case 0xf1:	/* int1 */
+	case 0xea:	/* jmp far */
+	case 0x9a:	/* call far */
+	/* Could be handled with special case code. */
+	case 0xe8:	/* call rel32 */
+	case 0xe0:	/* loopne rel8: cannot run out of line */
+	case 0xe1:	/* loope rel8 */
+	case 0xe2:	/* loop rel8 */
+	case 0xe3:	/* jecxz/jrcxz */
+	/* These two could be handled if the offsets fit */
+	case 0xe9:	/* jmp rel32 */
+	case 0xeb:	/* jmp rel8 */
+	case 0x70 ... 0x7f:	/* jcc rel8 */
+		return -EOPNOTSUPP;
+	}
+	/* XBEGIN's rel32 abort target is IP-relative, not RIP-relative. */
+	if (op == 0xc7 && insn->modrm.nbytes &&
+	    X86_MODRM_MOD(insn->modrm.value) == 3 &&
+	    X86_MODRM_REG(insn->modrm.value) == 7 &&
+	    X86_MODRM_RM(insn->modrm.value) == 0)
+		return -EOPNOTSUPP;
+	if (op == 0x0f) {
+		switch (insn->opcode.bytes[1]) {
+		case 0x05:	/* syscall */
+		case 0x34:	/* sysenter */
+		case 0x35:	/* sysexit */
+			return -EOPNOTSUPP;
+		}
+		/* jcc rel32: could be handled if offsets fit */
+		if (insn->opcode.bytes[1] >= 0x80 &&
+		    insn->opcode.bytes[1] <= 0x8f)
+			return -EOPNOTSUPP;
+		/* Allow endbranch because this is incompatible with CET anyways */
+	}
+	if (op == 0xff) {
+		u8 reg = X86_MODRM_REG(insn->modrm.value);
+
+		/* call/lcall/jmp-far indirect */
+		if (reg == 2 || reg == 3 || reg == 5)
+			return -EOPNOTSUPP;
+	}
+
+	if (insn_rip_relative(insn)) {
+		*disp_off = insn_offset_displacement(insn);
+		insn_get_displacement(insn);
+		*disp = insn->displacement.value;
 	}
-	if (off != 5)
+	return insn->length;
+}
+
+/*
+ * Read the site's instruction bytes from the file and classify them.
+ * The bytes are identical in every mm, so the copy is mm-independent.
+ */
+static int pun_decode_site(struct inode *inode, struct file *file,
+			       loff_t offset, u8 *copy,
+			       u8 *disp_off, s32 *disp, bool allow_nop_run)
+{
+	u8 buf[MAX_UINSN_BYTES] = {};
+	struct insn insn;
+	int ret;
+
+	ret = uprobe_copy_from_file(inode, file, offset, buf,
+				    MAX_UINSN_BYTES);
+	if (ret < 0)
+		return ret;
+	if (!ret)
+		return -EIO;
+
+	if (pun_site_is_nop(buf, allow_nop_run))
+		return 0;
+
+	/* Check single instruction */
+	if (insn_decode(&insn, buf, MAX_UINSN_BYTES, INSN_MODE_64))
 		return -EINVAL;
-	if (PAGE_SIZE - (vaddr & ~PAGE_MASK) < 5)
+	if (insn.length < 1 || insn.length > MAX_UINSN_BYTES)
 		return -EINVAL;
-	return 0;
+
+	/* the original bytes verbatim; classify only validates them */
+	memcpy(copy, buf, insn.length);
+	return pun_classify_insn(&insn, disp_off, disp);
 }
 
 static bool ptwrite_rel32(unsigned long from, unsigned long to, s32 *rel)

@@ -1805,16 +1940,184 @@ static int ptwrite_text_poke(struct arch_uprobe *auprobe,
 	return err;
 }
 
+static int pun_text_poke(struct arch_uprobe *auprobe,
+				 struct vm_area_struct *vma,
+				 unsigned long vaddr, u8 e9,
+				 struct write_opcode_ctx *ctx)
+{
+	int err;
+
+	err = uprobe_write(auprobe, vma, vaddr, &e9, 1, verify_insn,
+			   true, false, ctx);
+	if (err)
+		return err;
+	smp_text_poke_sync_each_cpu();
+	return 0;
+}
+
+static int pun_install(struct arch_uprobe *auprobe,
+			       struct vm_area_struct *vma, unsigned long vaddr,
+			       const u8 *orig)
+{
+	struct mm_struct *mm = vma->vm_mm;
+	struct uprobe_ptwrite_page *ptw;
+	struct uprobe_ptwrite_arch *ptw_a = &auprobe->ptwrite;
+	struct uprobes_state *state = &mm->uprobes_state;
+	struct write_opcode_ctx ctx = {
+		.base = vaddr,
+		.expect = EXPECT_BYTE,
+		.expect_byte = orig[0],
+	};
+	unsigned long t, page_base, block_off, stub_addr;
+	s64 site_delta, target;
+	s32 jump_rel, disp32, orig_rel;
+	u8 site_len;
+	bool found = false;
+	bool nop_fallback = ptwrite_site_is_multinop(orig,
+						     ptw_a->allow_nop_run) &&
+			    (vaddr & 7);
+	u8 *kaddr;
+	int b, ret;
+
+	mmap_assert_write_locked(mm);
+	if (nop_fallback) {
+		hlist_for_each_entry(ptw, &state->head_ptwrite, node) {
+			site_delta = (s64)vaddr - (s64)ptw->vaddr;
+			if (site_delta < INT_MIN || site_delta > INT_MAX)
+				continue;
+			for (b = 0; b < smp_load_acquire(&ptw->nblocks); b++)
+				if (!ptw->index[b].pun &&
+				    ptw->index[b].site_off == (s32)site_delta &&
+				    ptw->index[b].site_len == 5 &&
+				    !memcmp(ptw->index[b].site_insn, orig, 5))
+					break;
+			if (b >= smp_load_acquire(&ptw->nblocks))
+				continue;
+			if (!__in_uprobe_ptwrite(mm, ptw->vaddr))
+				continue;
+			return ptwrite_text_poke(auprobe, vma, vaddr,
+						 ptw->vaddr + ptw->index[b].off);
+		}
+		ptw = get_uprobe_ptwrite_page(mm, vaddr, ptw_a->stub_len);
+		if (!ptw)
+			return -ENOMEM;
+		block_off = ptw->cursor;
+	} else {
+		memcpy(&orig_rel, orig + 1, sizeof(orig_rel));
+		target = (s64)vaddr + 5 + (s64)orig_rel;
+		if (target < PAGE_SIZE || target >= TASK_SIZE_MAX)
+			return -EADDRNOTAVAIL;
+		t = (unsigned long)target;
+		page_base = t & PAGE_MASK;
+		block_off = t & (PAGE_SIZE - 1);
+		if (block_off + ptw_a->stub_len > PAGE_SIZE)
+			return -ENOSPC;
+
+		/* reuse an existing ptwrite page at the target, else map a new one */
+		hlist_for_each_entry(ptw, &state->head_ptwrite, node) {
+			if (ptw->vaddr == page_base) {
+				found = true;
+				break;
+			}
+		}
+		if (!found) {
+			if (vma_lookup(mm, page_base))
+				return -EADDRNOTAVAIL;	/* target page occupied */
+			ptw = create_uprobe_ptwrite_page_at(mm, page_base);
+			if (!ptw)
+				return -ENOMEM;
+			/* Order page initialization before publishing it to fault readers. */
+			smp_wmb();
+			hlist_add_head_rcu(&ptw->node, &state->head_ptwrite);
+		}
+	}
+
+	site_delta = (s64)vaddr - (s64)ptw->vaddr;
+	if (site_delta < INT_MIN || site_delta > INT_MAX)
+		return -ERANGE;
+
+	for (b = 0; b < ptw->nblocks; b++) {
+		u16 old_off = ptw->index[b].off;
+		u16 old_len = ptw->index[b].len;
+
+		if (block_off + ptw_a->stub_len <= old_off ||
+		    old_off + old_len <= block_off)
+			continue;
+		if (old_off == block_off && ptw->index[b].pun &&
+		    old_len == ptw_a->stub_len &&
+		    ptw->index[b].site_off == (s32)site_delta &&
+		    ptw->index[b].site_len == ptw_a->len &&
+		    !memcmp(ptw->index[b].site_insn, ptw_a->orig,
+			    ptw_a->len))
+			return pun_text_poke(auprobe, vma, vaddr, 0xe9, &ctx);
+		return -EADDRNOTAVAIL;
+	}
+	if (ptw->nblocks >= ARRAY_SIZE(ptw->index))
+		return -ENOMEM;
+
+	stub_addr = ptw->vaddr + block_off;
+	if (!ptwrite_rel32(stub_addr + ptw_a->jmp_off + 4,
+			   vaddr + (ptw_a->len ? ptw_a->len : 5), &jump_rel))
+		return -ERANGE;
+	if (ptw_a->len && ptw_a->disp_off) {
+		s64 d = (s64)ptw_a->disp + (s64)vaddr -
+			(s64)(stub_addr + ptw_a->copy_off);
+
+		if (d < INT_MIN || d > INT_MAX)
+			return -ERANGE;
+		disp32 = (s32)d;
+	}
+
+	/*
+	 * A NOP fallback needs a synthetic rel32 at the site, so it uses
+	 * the full five-byte poke and restore path rather than punning.
+	 */
+	site_len = nop_fallback ? 5 : ptw_a->len;
+	ptw->index[ptw->nblocks].off = block_off;
+	ptw->index[ptw->nblocks].len = ptw_a->stub_len;
+	ptw->index[ptw->nblocks].pun = !nop_fallback;
+	ptw->index[ptw->nblocks].orig0 = orig[0];
+	ptw->index[ptw->nblocks].site_len = site_len;
+	ptw->index[ptw->nblocks].site_off = (s32)site_delta;
+	memcpy(ptw->index[ptw->nblocks].site_insn, ptw_a->orig, site_len);
+	smp_store_release(&ptw->nblocks, ptw->nblocks + 1);
+
+	kaddr = kmap_local_page(ptw->page);
+	memcpy(kaddr + block_off, ptw_a->stub, ptw_a->stub_len);
+	memcpy(kaddr + block_off + ptw_a->jmp_off, &jump_rel, sizeof(jump_rel));
+	if (ptw_a->len && ptw_a->disp_off)
+		memcpy(kaddr + block_off + ptw_a->copy_off + ptw_a->disp_off,
+		       &disp32, sizeof(disp32));
+	kunmap_local(kaddr);
+
+	if (nop_fallback)
+		ret = ptwrite_text_poke(auprobe, vma, vaddr, stub_addr);
+	else
+		ret = pun_text_poke(auprobe, vma, vaddr, 0xe9, &ctx);
+	if (ret) {
+		/* Publish rollback before readers observe the reduced block count. */
+		smp_store_release(&ptw->nblocks, ptw->nblocks - 1);
+		return ret;
+	}
+
+	if (block_off + ptw_a->stub_len > ptw->cursor)
+		ptw->cursor = block_off + ptw_a->stub_len;
+	return 0;
+}
+
 int arch_uprobe_install_ptwrite(struct arch_uprobe *auprobe,
 		struct vm_area_struct *vma, unsigned long vaddr)
 {
 	struct mm_struct *mm = vma->vm_mm;
+	struct uprobes_state *state = &mm->uprobes_state;
 	struct uprobe_ptwrite_page *ptw;
 	struct uprobe_ptwrite_arch *ptw_a = &auprobe->ptwrite;
 	unsigned long block_off, stub_addr;
 	u8 *kaddr, orig[5];
+	s64 site_delta;
 	s32 rel;
 	int ret;
+	int b;
 
 	if (!is_64bit_mm(mm))
 		return -EOPNOTSUPP;