prep_irq_for_user_exit() is a superset of
prep_irq_for_kernel_enabled_exit(). In order to allow refactoring in
following patch, interchange the two as prep_irq_for_user_exit() will
call prep_irq_for_kernel_enabled_exit().
Signed-off-by: Christophe Leroy <redacted>
---
This series applies on top of Nic's series to speed up interrupt return on 64s
arch/powerpc/kernel/interrupt.c | 32 ++++++++++++++++----------------
1 file changed, 16 insertions(+), 16 deletions(-)
@@ -40,33 +40,27 @@ static inline bool exit_must_hard_disable(void)#endif/*-*localirqsmustbedisabled.Returnsfalseifthecallermustre-enable-*them,checkfornewwork,andtryagain.-*-*Thisshouldbecalledwithlocalirqsdisabled,butiftheywerepreviously-*enabledwhentheinterrupthandlerreturns(indicatingaprocess-context/-*synchronousinterrupt)thenirqs_enabledshouldbetrue.+*restartableistruethenEE/RIcanbeleftonbecauseinterruptsarehandled+*witharestartsequence.*/-staticnotrace__always_inlineboolprep_irq_for_user_exit(void)+staticnotrace__always_inlineboolprep_irq_for_kernel_enabled_exit(boolrestartable){-user_enter_irqoff();/* This must be done with RI=1 because tracing may touch vmaps */trace_hardirqs_on();#ifdef CONFIG_PPC32__hard_EE_RI_disable();#else-if(exit_must_hard_disable())+if(exit_must_hard_disable()||!restartable)__hard_EE_RI_disable();/* This pattern matches prep_irq_for_idle */if(unlikely(lazy_irq_pending_nocheck())){-if(exit_must_hard_disable()){+if(exit_must_hard_disable()||!restartable){local_paca->irq_happened|=PACA_IRQ_HARD_DIS;__hard_RI_enable();}trace_hardirqs_off();-user_exit_irqoff();returnfalse;}
@@ -75,27 +69,33 @@ static notrace __always_inline bool prep_irq_for_user_exit(void)}/*-*restartableistruethenEE/RIcanbeleftonbecauseinterruptsarehandled-*witharestartsequence.+*localirqsmustbedisabled.Returnsfalseifthecallermustre-enable+*them,checkfornewwork,andtryagain.+*+*Thisshouldbecalledwithlocalirqsdisabled,butiftheywerepreviously+*enabledwhentheinterrupthandlerreturns(indicatingaprocess-context/+*synchronousinterrupt)thenirqs_enabledshouldbetrue.*/-staticnotrace__always_inlineboolprep_irq_for_kernel_enabled_exit(boolrestartable)+staticnotrace__always_inlineboolprep_irq_for_user_exit(void){+user_enter_irqoff();/* This must be done with RI=1 because tracing may touch vmaps */trace_hardirqs_on();#ifdef CONFIG_PPC32__hard_EE_RI_disable();#else-if(exit_must_hard_disable()||!restartable)+if(exit_must_hard_disable())__hard_EE_RI_disable();/* This pattern matches prep_irq_for_idle */if(unlikely(lazy_irq_pending_nocheck())){-if(exit_must_hard_disable()||!restartable){+if(exit_must_hard_disable()){local_paca->irq_happened|=PACA_IRQ_HARD_DIS;__hard_RI_enable();}trace_hardirqs_off();+user_exit_irqoff();returnfalse;}
@@ -78,29 +78,14 @@ static notrace __always_inline bool prep_irq_for_kernel_enabled_exit(bool restar*/staticnotrace__always_inlineboolprep_irq_for_user_exit(void){-user_enter_irqoff();-/* This must be done with RI=1 because tracing may touch vmaps */-trace_hardirqs_on();--#ifdef CONFIG_PPC32-__hard_EE_RI_disable();-#else-if(exit_must_hard_disable())-__hard_EE_RI_disable();+boolret;-/* This pattern matches prep_irq_for_idle */-if(unlikely(lazy_irq_pending_nocheck())){-if(exit_must_hard_disable()){-local_paca->irq_happened|=PACA_IRQ_HARD_DIS;-__hard_RI_enable();-}-trace_hardirqs_off();+user_enter_irqoff();+ret=prep_irq_for_kernel_enabled_exit(true);+if(!ret)user_exit_irqoff();-returnfalse;-}-#endif-returntrue;+returnret;}/* Has to run notrace because it is entered not completely "reconciled" */
Rename syscall_exit_prepare_main() into interrupt_exit_prepare_main()
Make it static as it is not used anywhere else.
Pass it the 'ret' so that it can 'or' it directly instead of
oring twice, once inside the function and once outside.
And remove 'r3' parameter which is not used.
Also fix a typo where CONFIG_PPC_BOOK3S should be CONFIG_PPC_BOOK3S_64.
Signed-off-by: Christophe Leroy <redacted>
---
arch/powerpc/kernel/interrupt.c | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
@@ -254,7 +253,7 @@ notrace unsigned long syscall_exit_prepare_main(unsigned long r3,ti_flags=READ_ONCE(current_thread_info()->flags);}-if(IS_ENABLED(CONFIG_PPC_BOOK3S)&&IS_ENABLED(CONFIG_PPC_FPU)){+if(IS_ENABLED(CONFIG_PPC_BOOK3S_64)&&IS_ENABLED(CONFIG_PPC_FPU)){if(IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM)&&unlikely((ti_flags&_TIF_RESTORE_TM))){restore_tm_state(regs);
@@ -350,7 +349,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3,}local_irq_disable();-ret|=syscall_exit_prepare_main(r3,regs);+ret=interrupt_exit_user_prepare_main(regs,ret);#ifdef CONFIG_PPC64regs->exit_result=ret;
@@ -378,7 +377,7 @@ notrace unsigned long syscall_exit_restart(unsigned long r3, struct pt_regs *regBUG_ON(!user_mode(regs));-regs->exit_result|=syscall_exit_prepare_main(r3,regs);+regs->exit_result=interrupt_exit_user_prepare_main(regs,regs->exit_result);returnregs->exit_result;}
@@ -385,9 +385,7 @@ notrace unsigned long syscall_exit_restart(unsigned long r3, struct pt_regs *regnotraceunsignedlonginterrupt_exit_user_prepare(structpt_regs*regs){-unsignedlongti_flags;-unsignedlongflags;-unsignedlongret=0;+unsignedlongret;if(!IS_ENABLED(CONFIG_BOOKE)&&!IS_ENABLED(CONFIG_40x))BUG_ON(!(regs->msr&MSR_RI));
@@ -401,63 +399,14 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs)*/kuap_assert_locked();-local_irq_save(flags);--again:-ti_flags=READ_ONCE(current_thread_info()->flags);-while(unlikely(ti_flags&(_TIF_USER_WORK_MASK&~_TIF_RESTORE_TM))){-local_irq_enable();/* returning to user: may enable */-if(ti_flags&_TIF_NEED_RESCHED){-schedule();-}else{-if(ti_flags&_TIF_SIGPENDING)-ret|=_TIF_RESTOREALL;-do_notify_resume(regs,ti_flags);-}-local_irq_disable();-ti_flags=READ_ONCE(current_thread_info()->flags);-}--if(IS_ENABLED(CONFIG_PPC_BOOK3S_64)&&IS_ENABLED(CONFIG_PPC_FPU)){-if(IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM)&&-unlikely((ti_flags&_TIF_RESTORE_TM))){-restore_tm_state(regs);-}else{-unsignedlongmathflags=MSR_FP;--if(cpu_has_feature(CPU_FTR_VSX))-mathflags|=MSR_VEC|MSR_VSX;-elseif(cpu_has_feature(CPU_FTR_ALTIVEC))-mathflags|=MSR_VEC;--/* See above restore_math comment */-if((regs->msr&mathflags)!=mathflags)-restore_math(regs);-}-}--if(!prep_irq_for_user_exit()){-local_irq_enable();-local_irq_disable();-gotoagain;-}--booke_load_dbcr0();--#ifdef CONFIG_PPC_TRANSACTIONAL_MEM-local_paca->tm_scratch=regs->msr;-#endif+local_irq_disable();-account_cpu_user_exit();+ret=interrupt_exit_user_prepare_main(regs,0);#ifdef CONFIG_PPC64regs->exit_result=ret;#endif-/* Restore user access locks last */-kuap_user_restore(regs);-kuep_unlock();-returnret;}
From: Nicholas Piggin <npiggin@gmail.com> Date: 2021-06-11 02:26:53
Excerpts from Christophe Leroy's message of June 5, 2021 12:56 am:
prep_irq_for_user_exit() is a superset of
prep_irq_for_kernel_enabled_exit(). In order to allow refactoring in
following patch, interchange the two as prep_irq_for_user_exit() will
call prep_irq_for_kernel_enabled_exit().
Signed-off-by: Christophe Leroy <redacted>
---
This series applies on top of Nic's series to speed up interrupt return on 64s
Thanks for rebasing it.
Reviewed-by: Nicholas Piggin <npiggin@gmail.com>
@@ -40,33 +40,27 @@ static inline bool exit_must_hard_disable(void)#endif/*-*localirqsmustbedisabled.Returnsfalseifthecallermustre-enable-*them,checkfornewwork,andtryagain.-*-*Thisshouldbecalledwithlocalirqsdisabled,butiftheywerepreviously-*enabledwhentheinterrupthandlerreturns(indicatingaprocess-context/-*synchronousinterrupt)thenirqs_enabledshouldbetrue.+*restartableistruethenEE/RIcanbeleftonbecauseinterruptsarehandled+*witharestartsequence.*/-staticnotrace__always_inlineboolprep_irq_for_user_exit(void)+staticnotrace__always_inlineboolprep_irq_for_kernel_enabled_exit(boolrestartable){-user_enter_irqoff();/* This must be done with RI=1 because tracing may touch vmaps */trace_hardirqs_on();#ifdef CONFIG_PPC32__hard_EE_RI_disable();#else-if(exit_must_hard_disable())+if(exit_must_hard_disable()||!restartable)__hard_EE_RI_disable();/* This pattern matches prep_irq_for_idle */if(unlikely(lazy_irq_pending_nocheck())){-if(exit_must_hard_disable()){+if(exit_must_hard_disable()||!restartable){local_paca->irq_happened|=PACA_IRQ_HARD_DIS;__hard_RI_enable();}trace_hardirqs_off();-user_exit_irqoff();returnfalse;}
@@ -75,27 +69,33 @@ static notrace __always_inline bool prep_irq_for_user_exit(void)}/*-*restartableistruethenEE/RIcanbeleftonbecauseinterruptsarehandled-*witharestartsequence.+*localirqsmustbedisabled.Returnsfalseifthecallermustre-enable+*them,checkfornewwork,andtryagain.+*+*Thisshouldbecalledwithlocalirqsdisabled,butiftheywerepreviously+*enabledwhentheinterrupthandlerreturns(indicatingaprocess-context/+*synchronousinterrupt)thenirqs_enabledshouldbetrue.*/-staticnotrace__always_inlineboolprep_irq_for_kernel_enabled_exit(boolrestartable)+staticnotrace__always_inlineboolprep_irq_for_user_exit(void){+user_enter_irqoff();/* This must be done with RI=1 because tracing may touch vmaps */trace_hardirqs_on();#ifdef CONFIG_PPC32__hard_EE_RI_disable();#else-if(exit_must_hard_disable()||!restartable)+if(exit_must_hard_disable())__hard_EE_RI_disable();/* This pattern matches prep_irq_for_idle */if(unlikely(lazy_irq_pending_nocheck())){-if(exit_must_hard_disable()||!restartable){+if(exit_must_hard_disable()){local_paca->irq_happened|=PACA_IRQ_HARD_DIS;__hard_RI_enable();}trace_hardirqs_off();+user_exit_irqoff();returnfalse;}
From: Nicholas Piggin <npiggin@gmail.com> Date: 2021-06-11 02:31:51
Excerpts from Christophe Leroy's message of June 5, 2021 12:56 am:
prep_irq_for_user_exit() is a superset of
prep_irq_for_kernel_enabled_exit().
Refactor it.
I like the refactoring, but now prep_irq_for_user_exit() is calling
prep_irq_for_kernel_enabled_exit(), which seems like the wrong naming.
You could re-name prep_irq_for_kernel_enabled_exit() to
prep_irq_for_enabled_exit() maybe? Or it could be
__prep_irq_for_enabled_exit() then prep_irq_for_kernel_enabled_exit()
and prep_irq_for_user_exit() would both call it.
Otherwise
Reviewed-by: Nicholas Piggin <npiggin@gmail.com>
@@ -78,29 +78,14 @@ static notrace __always_inline bool prep_irq_for_kernel_enabled_exit(bool restar*/staticnotrace__always_inlineboolprep_irq_for_user_exit(void){-user_enter_irqoff();-/* This must be done with RI=1 because tracing may touch vmaps */-trace_hardirqs_on();--#ifdef CONFIG_PPC32-__hard_EE_RI_disable();-#else-if(exit_must_hard_disable())-__hard_EE_RI_disable();+boolret;-/* This pattern matches prep_irq_for_idle */-if(unlikely(lazy_irq_pending_nocheck())){-if(exit_must_hard_disable()){-local_paca->irq_happened|=PACA_IRQ_HARD_DIS;-__hard_RI_enable();-}-trace_hardirqs_off();+user_enter_irqoff();+ret=prep_irq_for_kernel_enabled_exit(true);+if(!ret)user_exit_irqoff();-returnfalse;-}-#endif-returntrue;+returnret;}/* Has to run notrace because it is entered not completely "reconciled" */
From: Nicholas Piggin <npiggin@gmail.com> Date: 2021-06-11 02:32:56
Excerpts from Christophe Leroy's message of June 5, 2021 12:56 am:
Rename syscall_exit_prepare_main() into interrupt_exit_prepare_main()
Make it static as it is not used anywhere else.
Pass it the 'ret' so that it can 'or' it directly instead of
oring twice, once inside the function and once outside.
And remove 'r3' parameter which is not used.
Also fix a typo where CONFIG_PPC_BOOK3S should be CONFIG_PPC_BOOK3S_64.
This all looks good I think. I need to grab this fix from your series.
Reviewed-by: Nicholas Piggin <npiggin@gmail.com>
@@ -254,7 +253,7 @@ notrace unsigned long syscall_exit_prepare_main(unsigned long r3,ti_flags=READ_ONCE(current_thread_info()->flags);}-if(IS_ENABLED(CONFIG_PPC_BOOK3S)&&IS_ENABLED(CONFIG_PPC_FPU)){+if(IS_ENABLED(CONFIG_PPC_BOOK3S_64)&&IS_ENABLED(CONFIG_PPC_FPU)){if(IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM)&&unlikely((ti_flags&_TIF_RESTORE_TM))){restore_tm_state(regs);
@@ -350,7 +349,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3,}local_irq_disable();-ret|=syscall_exit_prepare_main(r3,regs);+ret=interrupt_exit_user_prepare_main(regs,ret);#ifdef CONFIG_PPC64regs->exit_result=ret;
@@ -378,7 +377,7 @@ notrace unsigned long syscall_exit_restart(unsigned long r3, struct pt_regs *regBUG_ON(!user_mode(regs));-regs->exit_result|=syscall_exit_prepare_main(r3,regs);+regs->exit_result=interrupt_exit_user_prepare_main(regs,regs->exit_result);returnregs->exit_result;}
Excerpts from Christophe Leroy's message of June 5, 2021 12:56 am:
quoted
prep_irq_for_user_exit() is a superset of
prep_irq_for_kernel_enabled_exit().
Refactor it.
I like the refactoring, but now prep_irq_for_user_exit() is calling
prep_irq_for_kernel_enabled_exit(), which seems like the wrong naming.
You could re-name prep_irq_for_kernel_enabled_exit() to
prep_irq_for_enabled_exit() maybe? Or it could be
__prep_irq_for_enabled_exit() then prep_irq_for_kernel_enabled_exit()
and prep_irq_for_user_exit() would both call it.
I renamed it prep_irq_for_enabled_exit().
And I realised that after patch 4, prep_irq_for_enabled_exit() has become a trivial function used only once.
So I swapped patches 1/2 with patches 3/4 and added a 5th one to squash prep_irq_for_enabled_exit() into its caller.
You didn't have any comment on patch 4 (that is now patch 2) ?
Thanks for the review
Christophe
From: Nicholas Piggin <npiggin@gmail.com> Date: 2021-06-17 03:34:03
Excerpts from Christophe Leroy's message of June 15, 2021 6:37 pm:
Le 11/06/2021 à 04:30, Nicholas Piggin a écrit :
quoted
Excerpts from Christophe Leroy's message of June 5, 2021 12:56 am:
quoted
prep_irq_for_user_exit() is a superset of
prep_irq_for_kernel_enabled_exit().
Refactor it.
I like the refactoring, but now prep_irq_for_user_exit() is calling
prep_irq_for_kernel_enabled_exit(), which seems like the wrong naming.
You could re-name prep_irq_for_kernel_enabled_exit() to
prep_irq_for_enabled_exit() maybe? Or it could be
__prep_irq_for_enabled_exit() then prep_irq_for_kernel_enabled_exit()
and prep_irq_for_user_exit() would both call it.
I renamed it prep_irq_for_enabled_exit().
And I realised that after patch 4, prep_irq_for_enabled_exit() has become a trivial function used
only once.
So I swapped patches 1/2 with patches 3/4 and added a 5th one to squash prep_irq_for_enabled_exit()
into its caller.
You didn't have any comment on patch 4 (that is now patch 2) ?
I think it's okay, just trying to hunt down some apparent big-endian bug
with my series. I can't see any problems with yours though, thanks for
rebasing them, I'll take a better look when I can.
Thanks,
Nick