[RFC PATCH v2 34/45] arm64: nmi: Add handling of superpriority interrupts as NMIs

@@ -37,6 +37,13 @@ static inline bool arch_irqentry_exit_need_resched(void)
 	if (system_uses_irq_prio_masking() && read_sysreg(daif))
 		return false;
 
+	/*
+	 * If AllInt is set then we must have handled an NMI, so skip
+	 * preemption
+	 */
+	if (system_uses_nmi() && read_sysreg_s(SYS_ALLINT))
+		return false;
+
 	/*
 	 * Preempting a task from an IRQ means we leave copies of PSTATE
 	 * on the stack. cpufeature's enable calls may modify PSTATE, but

@@ -31,6 +31,14 @@
 #include <asm/sysreg.h>
 #include <asm/system_misc.h>
 
+static __always_inline bool is_nmi(void)
+{
+	if (!system_uses_nmi())
+		return false;
+
+	return !!(read_sysreg(isr_el1) & ISR_EL1_IS);
+}
+
 /*
  * Handle IRQ/context state management when entering from kernel mode.
  * Before this function is called it is not safe to call regular kernel code,

@@ -525,8 +533,8 @@ asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
 	arm64_debug_exc_context(CRITICAL_CONTEXT);
 }
 
-static __always_inline void __el1_pnmi(struct pt_regs *regs,
-				       void (*handler)(struct pt_regs *))
+static __always_inline void __el1_nmi(struct pt_regs *regs,
+				      void (*handler)(struct pt_regs *))
 {
 	arm64_exc_hwstate_t hwstate;
 	irqentry_state_t state;

@@ -545,7 +553,23 @@ static __always_inline void __el1_irq(struct pt_regs *regs,
 
 	state = arm64_enter_from_kernel_mode(regs);
 
-	arm64_unmask_exc_context(NONMI_CONTEXT);
+	/*
+	 * For non-NMI systems, we can switch to NOIRQ context since
+	 * NMIs cannot occur.
+	 *
+	 * For pseudo-NMI, we must keep NONMI context until we can
+	 * determine the interrupt priority, which is done by the IRQ
+	 * handler.
+	 *
+	 * For FEAT_NMI, we already know the interrupt is an IRQ, so we
+	 * can switch to NOIRQ context immediately. NMIs do not
+	 * interfere because they follow a separate handling path and do
+	 * not share the IRQ handling state.
+	 */
+	if (system_uses_irq_prio_masking())
+		arm64_unmask_exc_context(NONMI_CONTEXT);
+	else
+		arm64_unmask_exc_context(NOIRQ_CONTEXT);
 
 	irq_enter_rcu();
 	do_interrupt_handler(regs, handler);

@@ -565,8 +589,9 @@ static __always_inline void __el1_irq(struct pt_regs *regs,
 static void noinstr el1_interrupt(struct pt_regs *regs,
 				  void (*handler)(struct pt_regs *))
 {
-	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && regs_irqs_disabled(regs))
-		__el1_pnmi(regs, handler);
+	/* Is there a NMI to handle? */
+	if (regs_irqs_disabled(regs) || is_nmi())
+		__el1_nmi(regs, handler);
 	else
 		__el1_irq(regs, handler);
 

@@ -580,6 +605,8 @@ asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs)
 
 asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs)
 {
+	WARN_ON_ONCE(system_uses_nmi() && (read_sysreg(isr_el1) & ISR_EL1_FS));
+
 	el1_interrupt(regs, handle_arch_fiq);
 }
 

@@ -903,20 +930,36 @@ asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
 	arm64_debug_exc_context(CRITICAL_CONTEXT);
 }
 
-static void noinstr el0_interrupt(struct pt_regs *regs,
-				  void (*handler)(struct pt_regs *))
+static __always_inline void __el0_nmi(struct pt_regs *regs,
+				      void (*handler)(struct pt_regs *))
 {
-	arm64_enter_from_user_mode(regs);
+	irqentry_state_t state;
+	arm64_exc_hwstate_t hwstate;
 
-	arm64_unmask_exc_context(NONMI_CONTEXT);
+	state = irqentry_nmi_enter(regs);
+	hwstate = arm64_unmask_exc_context(NONMI_CONTEXT);
+	do_interrupt_handler(regs, handler);
+	arm64_mask_exc_context(hwstate);
+	irqentry_nmi_exit(regs, state);
+}
 
-	if (regs->pc & BIT(55))
-		arm64_apply_bp_hardening();
+static __always_inline void __el0_irq(struct pt_regs *regs,
+				      void (*handler)(struct pt_regs *))
+{
+	arm64_enter_from_user_mode(regs);
+	/*
+	 * For the same reason as in el1_irq() keep NONMI
+	 * context for pesudo-NMI, and switch to NOIRQ
+	 * otherwise
+	 */
+	if (system_uses_irq_prio_masking())
+		arm64_unmask_exc_context(NONMI_CONTEXT);
+	else
+		arm64_unmask_exc_context(NOIRQ_CONTEXT);
 
 	irq_enter_rcu();
 	do_interrupt_handler(regs, handler);
 	irq_exit_rcu();
-
 	/*
 	 * For the same reason as in el1_irq() we effectivly
 	 * have NOIRQ_CONTEXT on return from handler - keep

@@ -924,6 +967,20 @@ static void noinstr el0_interrupt(struct pt_regs *regs,
 	 */
 	arm64_debug_exc_context(NOIRQ_CONTEXT);
 	arm64_exit_to_user_mode(regs, arm64_exc_hwstate_of_context(NOIRQ_CONTEXT));
+}
+
+static void noinstr el0_interrupt(struct pt_regs *regs,
+				  void (*handler)(struct pt_regs *))
+{
+	if (regs->pc & BIT(55))
+		arm64_apply_bp_hardening();
+
+	/* Is there a NMI to handle? */
+	if (is_nmi())
+		__el0_nmi(regs, handler);
+	else
+		__el0_irq(regs, handler);
+
 	arm64_debug_exc_context(CRITICAL_CONTEXT);
 }
 

@@ -939,6 +996,8 @@ asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs)
 
 static void noinstr __el0_fiq_handler_common(struct pt_regs *regs)
 {
+	WARN_ON_ONCE(system_uses_nmi() && (read_sysreg(isr_el1) & ISR_EL1_FS));
+
 	el0_interrupt(regs, handle_arch_fiq);
 }
 

@@ -1035,7 +1035,7 @@ static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
 
 static bool ipi_should_be_nmi(enum ipi_msg_type ipi)
 {
-	if (!system_uses_irq_prio_masking())
+	if (!system_uses_nmi() && !system_uses_irq_prio_masking())
 		return false;
 
 	switch (ipi) {