[RFC PATCH v2 11/45] arm64: irqflags: save and use both DAIF and PMR
From: Vladimir Murzin <hidden>
Date: 2026-07-27 16:36:09
Subsystem:
arm64 port (aarch64 architecture), ptrace support, the rest · Maintainers:
Catalin Marinas, Will Deacon, Oleg Nesterov, Linus Torvalds
From: Ada Couprie Diaz <redacted>
With pseudo-NMIs enabled, both DAIF and the PMR affect interrupt masking.
Now that we have a type which can track both of them at the same time,
update our irqflags implementation to use it.
Save DAIF flags in all cases, as they can be manipulated directly by other
code, and the PMR if it is in use.
When checking if IRQs are disabled, now that we always save DAIF we can
check that the I flag is set and bypass checking the PMR if it is.
We can also properly check if PMR masks interrupts (PMR < GIC_PRIO_IRQON),
now that we don't need to rely on the GIC_PRIO_PSR_I_SET bit being set in
the PMR to know if DAIF is already masking interrupts.
Update `irqs_priority_unmasked()` to align with this change.
This allows us to remove the `__daif_...` and `__pmr_...` versions
of the save and check functions, as they are now unified.
We can reasonably merge the two `__{daif,pmr}_irq_restore()` functions
in the main one, as the DAIF and PMR values are properly split now.
Signed-off-by: Ada Couprie Diaz <redacted>
Signed-off-by: Vladimir Murzin <redacted>
---
arch/arm64/include/asm/irqflags.h | 118 +++++++-----------------------
arch/arm64/include/asm/ptrace.h | 18 +++--
2 files changed, 38 insertions(+), 98 deletions(-)
diff --git a/arch/arm64/include/asm/irqflags.h b/arch/arm64/include/asm/irqflags.h
index 1863a955d119..50720a01bde4 100644
--- a/arch/arm64/include/asm/irqflags.h
+++ b/arch/arm64/include/asm/irqflags.h@@ -96,135 +96,69 @@ static __always_inline void arch_local_irq_disable(void) } } -static __always_inline unsigned long __daif_local_save_flags(void) -{ - arm64_exc_hwstate_t hwstate = { .daif = read_sysreg(daif) }; - - return hwstate.flags; -} - -static __always_inline unsigned long __pmr_local_save_flags(void) -{ - arm64_exc_hwstate_t hwstate = { .pmr = read_sysreg_s(SYS_ICC_PMR_EL1) }; - - return hwstate.flags; -} - /* * Save the current interrupt enable state. */ static __always_inline unsigned long arch_local_save_flags(void) { - if (system_uses_irq_prio_masking()) { - return __pmr_local_save_flags(); - } else { - return __daif_local_save_flags(); - } -} + arm64_exc_hwstate_t hwstate = { .daif = read_sysreg(daif) }; -static __always_inline bool __daif_irqs_disabled_flags(unsigned long flags) -{ - arm64_exc_hwstate_t hwstate = { .flags = flags }; + if (system_uses_irq_prio_masking()) + hwstate.pmr = read_sysreg_s(SYS_ICC_PMR_EL1); - return hwstate.daif & PSR_I_BIT; + return hwstate.flags; } -static __always_inline bool __pmr_irqs_disabled_flags(unsigned long flags) +static __always_inline bool arch_irqs_disabled_flags(unsigned long flags) { arm64_exc_hwstate_t hwstate = { .flags = flags }; - return hwstate.pmr != GIC_PRIO_IRQON; -} + if (hwstate.daif & PSR_I_BIT) + return true; -static __always_inline bool arch_irqs_disabled_flags(unsigned long flags) -{ if (system_uses_irq_prio_masking()) { - return __pmr_irqs_disabled_flags(flags); - } else { - return __daif_irqs_disabled_flags(flags); - } -} + WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_IRQFLAGS) && + hwstate.pmr != GIC_PRIO_IRQOFF && + hwstate.pmr != GIC_PRIO_IRQON); -static __always_inline bool __daif_irqs_disabled(void) -{ - return __daif_irqs_disabled_flags(__daif_local_save_flags()); -} + return hwstate.pmr != GIC_PRIO_IRQON; + } -static __always_inline bool __pmr_irqs_disabled(void) -{ - return __pmr_irqs_disabled_flags(__pmr_local_save_flags()); + return false; } static __always_inline bool arch_irqs_disabled(void) { - if (system_uses_irq_prio_masking()) { - return __pmr_irqs_disabled(); - } else { - return __daif_irqs_disabled(); - } + return arch_irqs_disabled_flags(arch_local_save_flags()); } -static __always_inline unsigned long __daif_local_irq_save(void) -{ - unsigned long flags = __daif_local_save_flags(); - - __daif_local_irq_disable(); - - return flags; -} - -static __always_inline unsigned long __pmr_local_irq_save(void) +static __always_inline unsigned long arch_local_irq_save(void) { - unsigned long flags = __pmr_local_save_flags(); + unsigned long flags = arch_local_save_flags(); - /* - * There are too many states with IRQs disabled, just keep the current - * state if interrupts are already disabled/masked. - */ - if (!__pmr_irqs_disabled_flags(flags)) + if (system_uses_irq_prio_masking()) __pmr_local_irq_disable(); + else + __daif_local_irq_disable(); return flags; } -static __always_inline unsigned long arch_local_irq_save(void) -{ - if (system_uses_irq_prio_masking()) { - return __pmr_local_irq_save(); - } else { - return __daif_local_irq_save(); - } -} - -static __always_inline void __daif_local_irq_restore(unsigned long flags) -{ - arm64_exc_hwstate_t hwstate = { .flags = flags }; - - barrier(); - write_sysreg(hwstate.daif, daif); - barrier(); -} - -static __always_inline void __pmr_local_irq_restore(unsigned long flags) -{ - arm64_exc_hwstate_t hwstate = { .flags = flags }; - - barrier(); - write_sysreg_s(hwstate.pmr, SYS_ICC_PMR_EL1); - pmr_sync(); - barrier(); -} - /* * restore saved IRQ state */ static __always_inline void arch_local_irq_restore(unsigned long flags) { + arm64_exc_hwstate_t hwstate = { .flags = flags }; + + barrier(); if (system_uses_irq_prio_masking()) { - __pmr_local_irq_restore(flags); + write_sysreg_s(hwstate.pmr, SYS_ICC_PMR_EL1); + pmr_sync(); } else { - __daif_local_irq_restore(flags); + write_sysreg(hwstate.daif, daif); } + barrier(); } #endif /* __ASM_IRQFLAGS_H */
diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h
index f7dc5fb9427d..e4c7e6bafa3d 100644
--- a/arch/arm64/include/asm/ptrace.h
+++ b/arch/arm64/include/asm/ptrace.h@@ -203,14 +203,20 @@ static inline void forget_syscall(struct pt_regs *regs) #define processor_mode(regs) \ ((regs)->pstate & PSR_MODE_MASK) -#define irqs_priority_unmasked(regs) \ - (system_uses_irq_prio_masking() ? \ - (regs)->pmr == GIC_PRIO_IRQON : \ - true) - static __always_inline bool regs_irqs_disabled(const struct pt_regs *regs) { - return (regs->pstate & PSR_I_BIT) || !irqs_priority_unmasked(regs); + if (regs->pstate & PSR_I_BIT) + return true; + + if (system_uses_irq_prio_masking()) { + WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_IRQFLAGS) && + regs->pmr != GIC_PRIO_IRQOFF && + regs->pmr != GIC_PRIO_IRQON); + + return regs->pmr != GIC_PRIO_IRQON; + } + + return false; } #define interrupts_enabled(regs) (!regs_irqs_disabled(regs))
--
2.34.1