[PATCH v8 6/6] powerpc/pv-qspinlock: Optimise native unlock path
From: Pan Xinhui <hidden>
Date: 2016-12-05 10:25:40
Also in:
lkml, virtualization
Subsystem:
linux for powerpc (32-bit and 64-bit), paravirt_ops interface, the rest · Maintainers:
Madhavan Srinivasan, Michael Ellerman, Juergen Gross, Linus Torvalds
Avoid a function call under native version of qspinlock. On powerNV, bafore applying this patch, every unlock is expensive. This small optimizes enhance the performance. We use static_key with jump_label which removes unnecessary loads of lppaca and its stuff. Signed-off-by: Pan Xinhui <redacted> --- arch/powerpc/include/asm/qspinlock_paravirt.h | 18 +++++++++++++++++- arch/powerpc/kernel/paravirt.c | 4 ++++ 2 files changed, 21 insertions(+), 1 deletion(-)
diff --git a/arch/powerpc/include/asm/qspinlock_paravirt.h b/arch/powerpc/include/asm/qspinlock_paravirt.h
index d87cda0..8d39446 100644
--- a/arch/powerpc/include/asm/qspinlock_paravirt.h
+++ b/arch/powerpc/include/asm/qspinlock_paravirt.h@@ -6,12 +6,14 @@ #define _ASM_QSPINLOCK_PARAVIRT_H #include <asm/qspinlock_paravirt_types.h> +#include <linux/jump_label.h> extern void pv_lock_init(void); extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); extern void __pv_init_lock_hash(void); extern void __pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); extern void __pv_queued_spin_unlock(struct qspinlock *lock); +extern struct static_key_true sharedprocessor_key; static inline void pv_queued_spin_lock(struct qspinlock *lock, u32 val) {
@@ -20,7 +22,21 @@ static inline void pv_queued_spin_lock(struct qspinlock *lock, u32 val) static inline void pv_queued_spin_unlock(struct qspinlock *lock) { - pv_lock_op.unlock(lock); + /* + * on powerNV and pSeries with jump_label, code will be + * PowerNV: pSeries: + * nop; b 2f; + * native unlock 2: + * pv unlock; + * In this way, we can do unlock quick in native case. + * + * IF jump_label is not enabled, we fall back into + * if condition, IOW, ld && cmp && bne. + */ + if (static_branch_likely(&sharedprocessor_key)) + native_queued_spin_unlock(lock); + else + pv_lock_op.unlock(lock); } static inline void pv_wait(u8 *ptr, u8 val)
diff --git a/arch/powerpc/kernel/paravirt.c b/arch/powerpc/kernel/paravirt.c
index e697b17..a0a000e 100644
--- a/arch/powerpc/kernel/paravirt.c
+++ b/arch/powerpc/kernel/paravirt.c@@ -140,6 +140,9 @@ struct pv_lock_ops pv_lock_op = { }; EXPORT_SYMBOL(pv_lock_op); +struct static_key_true sharedprocessor_key = STATIC_KEY_TRUE_INIT; +EXPORT_SYMBOL(sharedprocessor_key); + void __init pv_lock_init(void) { if (SHARED_PROCESSOR) {
@@ -149,5 +152,6 @@ void __init pv_lock_init(void) pv_lock_op.unlock = __pv_queued_spin_unlock; pv_lock_op.wait = __pv_wait; pv_lock_op.kick = __pv_kick; + static_branch_disable(&sharedprocessor_key); } }
--
2.4.11