Re: [External] [PATCH v6 2/5] riscv: add support for SBI Supervisor Software Events extension

The SBI SSE extension allows the supervisor software to be notified by
the SBI of specific events that are not maskable. The context switch is
handled partially by the firmware which will save registers a6 and a7.
When entering kernel we can rely on these 2 registers to setup the stack
and save all the registers.

Since SSE events can be delivered at any time to the kernel (including
during exception handling, we need a way to locate the current_task for
context tracking. On RISC-V, it is sotred in scratch when in user space
or tp when in kernel space (in which case SSCRATCH is zero). But at a
at the beginning of exception handling, SSCRATCH is used to swap tp and
check the origin of the exception. If interrupted at that point, then,
there is no way to reliably know were is located the current
task_struct. Even checking the interruption location won't work as SSE
event can be nested on top of each other so the original interruption
site might be lost at some point. In order to retrieve it reliably,
store the current task in an additional __sse_entry_task per_cpu array.
This array is then used to retrieve the current task based on the
hart ID that is passed to the SSE event handler in a6.

That being said, the way the current task struct is stored should
probably be reworked to find a better reliable alternative.

Since each events (and each CPU for local events) have their own
context and can preempt each other, allocate a stack (and a shadow stack
if needed for each of them (and for each cpu for local events).

When completing the event, if we were coming from kernel with interrupts
disabled, simply return there. If coming from userspace or kernel with
interrupts enabled, simulate an interrupt exception by setting IE_SIE in
CSR_IP to allow delivery of signals to user task. For instance this can
happen, when a RAS event has been generated by a user application and a
SIGBUS has been sent to a task.

Signed-off-by: Clément Léger <redacted>
---
 arch/riscv/include/asm/asm.h         |  14 ++-
 arch/riscv/include/asm/scs.h         |   7 ++
 arch/riscv/include/asm/sse.h         |  47 +++++++
 arch/riscv/include/asm/switch_to.h   |  14 +++
 arch/riscv/include/asm/thread_info.h |   1 +
 arch/riscv/kernel/Makefile           |   1 +
 arch/riscv/kernel/asm-offsets.c      |  14 +++
 arch/riscv/kernel/sse.c              | 154 +++++++++++++++++++++++
 arch/riscv/kernel/sse_entry.S        | 180 +++++++++++++++++++++++++++
 9 files changed, 429 insertions(+), 3 deletions(-)
 create mode 100644 arch/riscv/include/asm/sse.h
 create mode 100644 arch/riscv/kernel/sse.c
 create mode 100644 arch/riscv/kernel/sse_entry.S

diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h
index a8a2af6dfe9d..982c4be9a9c3 100644
--- a/arch/riscv/include/asm/asm.h
+++ b/arch/riscv/include/asm/asm.h

@@ -90,16 +90,24 @@
 #define PER_CPU_OFFSET_SHIFT 3
 #endif

-.macro asm_per_cpu dst sym tmp
-       REG_L \tmp, TASK_TI_CPU_NUM(tp)
-       slli  \tmp, \tmp, PER_CPU_OFFSET_SHIFT
+.macro asm_per_cpu_with_cpu dst sym tmp cpu
+       slli  \tmp, \cpu, PER_CPU_OFFSET_SHIFT
        la    \dst, __per_cpu_offset
        add   \dst, \dst, \tmp
        REG_L \tmp, 0(\dst)
        la    \dst, \sym
        add   \dst, \dst, \tmp
 .endm
+
+.macro asm_per_cpu dst sym tmp
+       REG_L \tmp, TASK_TI_CPU_NUM(tp)
+       asm_per_cpu_with_cpu \dst \sym \tmp \tmp
+.endm
 #else /* CONFIG_SMP */
+.macro asm_per_cpu_with_cpu dst sym tmp cpu
+       la    \dst, \sym
+.endm
+
 .macro asm_per_cpu dst sym tmp
        la    \dst, \sym
 .endm

diff --git a/arch/riscv/include/asm/scs.h b/arch/riscv/include/asm/scs.h
index 0e45db78b24b..62344daad73d 100644
--- a/arch/riscv/include/asm/scs.h
+++ b/arch/riscv/include/asm/scs.h

@@ -18,6 +18,11 @@
        load_per_cpu gp, irq_shadow_call_stack_ptr, \tmp
 .endm

+/* Load the per-CPU IRQ shadow call stack to gp. */
+.macro scs_load_sse_stack reg_evt
+       REG_L gp, SSE_REG_EVT_SHADOW_STACK(\reg_evt)
+.endm
+
 /* Load task_scs_sp(current) to gp. */
 .macro scs_load_current
        REG_L   gp, TASK_TI_SCS_SP(tp)

@@ -41,6 +46,8 @@
 .endm
 .macro scs_load_irq_stack tmp
 .endm
+.macro scs_load_sse_stack reg_evt
+.endm
 .macro scs_load_current
 .endm
 .macro scs_load_current_if_task_changed prev

diff --git a/arch/riscv/include/asm/sse.h b/arch/riscv/include/asm/sse.h
new file mode 100644
index 000000000000..8929a268462c
--- /dev/null
+++ b/arch/riscv/include/asm/sse.h

@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2024 Rivos Inc.
+ */
+#ifndef __ASM_SSE_H
+#define __ASM_SSE_H
+
+#include <asm/sbi.h>
+
+#ifdef CONFIG_RISCV_SSE
+
+struct sse_event_interrupted_state {
+       unsigned long a6;
+       unsigned long a7;
+};
+
+struct sse_event_arch_data {
+       void *stack;
+       void *shadow_stack;
+       unsigned long tmp;
+       struct sse_event_interrupted_state interrupted;
+       unsigned long interrupted_phys;
+       u32 evt_id;
+       unsigned int hart_id;
+       unsigned int cpu_id;
+};
+
+static inline bool sse_event_is_global(u32 evt)
+{
+       return !!(evt & SBI_SSE_EVENT_GLOBAL);
+}
+
+void arch_sse_event_update_cpu(struct sse_event_arch_data *arch_evt, int cpu);
+int arch_sse_init_event(struct sse_event_arch_data *arch_evt, u32 evt_id,
+                       int cpu);
+void arch_sse_free_event(struct sse_event_arch_data *arch_evt);
+int arch_sse_register_event(struct sse_event_arch_data *arch_evt);
+
+void sse_handle_event(struct sse_event_arch_data *arch_evt,
+                     struct pt_regs *regs);
+asmlinkage void handle_sse(void);
+asmlinkage void do_sse(struct sse_event_arch_data *arch_evt,
+                      struct pt_regs *reg);
+
+#endif
+
+#endif

diff --git a/arch/riscv/include/asm/switch_to.h b/arch/riscv/include/asm/switch_to.h
index 0e71eb82f920..cd1cead0c682 100644
--- a/arch/riscv/include/asm/switch_to.h
+++ b/arch/riscv/include/asm/switch_to.h

@@ -88,6 +88,19 @@ static inline void __switch_to_envcfg(struct task_struct *next)
                        :: "r" (next->thread.envcfg) : "memory");
 }

+#ifdef CONFIG_RISCV_SSE
+DECLARE_PER_CPU(struct task_struct *, __sse_entry_task);
+
+static inline void __switch_sse_entry_task(struct task_struct *next)
+{
+       __this_cpu_write(__sse_entry_task, next);
+}
+#else
+static inline void __switch_sse_entry_task(struct task_struct *next)
+{
+}
+#endif
+
 extern struct task_struct *__switch_to(struct task_struct *,
                                       struct task_struct *);

@@ -122,6 +135,7 @@ do {                                                        \
        if (switch_to_should_flush_icache(__next))      \
                local_flush_icache_all();               \
        __switch_to_envcfg(__next);                     \
+       __switch_sse_entry_task(__next);                        \
        ((last) = __switch_to(__prev, __next));         \
 } while (0)

diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h
index f5916a70879a..28e9805e61fc 100644
--- a/arch/riscv/include/asm/thread_info.h
+++ b/arch/riscv/include/asm/thread_info.h

@@ -36,6 +36,7 @@
 #define OVERFLOW_STACK_SIZE     SZ_4K

 #define IRQ_STACK_SIZE         THREAD_SIZE
+#define SSE_STACK_SIZE         THREAD_SIZE

 #ifndef __ASSEMBLY__

diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile
index c7b542573407..62e4490b34ee 100644
--- a/arch/riscv/kernel/Makefile
+++ b/arch/riscv/kernel/Makefile

@@ -99,6 +99,7 @@ obj-$(CONFIG_DYNAMIC_FTRACE)  += mcount-dyn.o
 obj-$(CONFIG_PERF_EVENTS)      += perf_callchain.o
 obj-$(CONFIG_HAVE_PERF_REGS)   += perf_regs.o
 obj-$(CONFIG_RISCV_SBI)                += sbi.o sbi_ecall.o
+obj-$(CONFIG_RISCV_SSE)                += sse.o sse_entry.o
 ifeq ($(CONFIG_RISCV_SBI), y)
 obj-$(CONFIG_SMP)              += sbi-ipi.o
 obj-$(CONFIG_SMP) += cpu_ops_sbi.o

diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c
index 6e8c0d6feae9..315547c3a2ef 100644
--- a/arch/riscv/kernel/asm-offsets.c
+++ b/arch/riscv/kernel/asm-offsets.c

@@ -14,6 +14,8 @@
 #include <asm/ptrace.h>
 #include <asm/cpu_ops_sbi.h>
 #include <asm/stacktrace.h>
+#include <asm/sbi.h>
+#include <asm/sse.h>
 #include <asm/suspend.h>

 void asm_offsets(void);

@@ -528,4 +530,16 @@ void asm_offsets(void)
        DEFINE(FREGS_A6,            offsetof(struct __arch_ftrace_regs, a6));
        DEFINE(FREGS_A7,            offsetof(struct __arch_ftrace_regs, a7));
 #endif
+
+#ifdef CONFIG_RISCV_SSE
+       OFFSET(SSE_REG_EVT_STACK, sse_event_arch_data, stack);
+       OFFSET(SSE_REG_EVT_SHADOW_STACK, sse_event_arch_data, shadow_stack);
+       OFFSET(SSE_REG_EVT_TMP, sse_event_arch_data, tmp);
+       OFFSET(SSE_REG_HART_ID, sse_event_arch_data, hart_id);
+       OFFSET(SSE_REG_CPU_ID, sse_event_arch_data, cpu_id);
+
+       DEFINE(SBI_EXT_SSE, SBI_EXT_SSE);
+       DEFINE(SBI_SSE_EVENT_COMPLETE, SBI_SSE_EVENT_COMPLETE);
+       DEFINE(ASM_NR_CPUS, NR_CPUS);
+#endif
 }

diff --git a/arch/riscv/kernel/sse.c b/arch/riscv/kernel/sse.c
new file mode 100644
index 000000000000..d2da7e23a74a
--- /dev/null
+++ b/arch/riscv/kernel/sse.c

@@ -0,0 +1,154 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2024 Rivos Inc.
+ */
+#include <linux/nmi.h>
+#include <linux/scs.h>
+#include <linux/bitfield.h>
+#include <linux/riscv_sse.h>
+#include <linux/percpu-defs.h>
+
+#include <asm/asm-prototypes.h>
+#include <asm/switch_to.h>
+#include <asm/irq_stack.h>
+#include <asm/sbi.h>
+#include <asm/sse.h>
+
+DEFINE_PER_CPU(struct task_struct *, __sse_entry_task);
+
+void __weak sse_handle_event(struct sse_event_arch_data *arch_evt, struct pt_regs *regs)
+{
+}
+
+void do_sse(struct sse_event_arch_data *arch_evt, struct pt_regs *regs)
+{
+       nmi_enter();
+
+       /* Retrieve missing GPRs from SBI */
+       sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_READ, arch_evt->evt_id,
+                 SBI_SSE_ATTR_INTERRUPTED_A6,
+                 (SBI_SSE_ATTR_INTERRUPTED_A7 - SBI_SSE_ATTR_INTERRUPTED_A6) + 1,
+                 arch_evt->interrupted_phys, 0, 0);
+
+       memcpy(&regs->a6, &arch_evt->interrupted, sizeof(arch_evt->interrupted));
+
+       sse_handle_event(arch_evt, regs);
+
+       /*
+        * The SSE delivery path does not uses the "standard" exception path
+        * (see sse_entry.S) and does not process any pending signal/softirqs
+        * due to being similar to a NMI.
+        * Some drivers (PMU, RAS) enqueue pending work that needs to be handled
+        * as soon as possible by bottom halves. For that purpose, set the SIP
+        * software interrupt pending bit which will force a software interrupt
+        * to be serviced once interrupts are reenabled in the interrupted
+        * context if they were masked or directly if unmasked.
+        */
+       csr_set(CSR_IP, IE_SIE);