From: Lai Jiangshan <jiangshan.ljs@xxxxxxxxxxxx>
Add rcu_read_unlock_special_thunk(), so that the inlined rcu_read_unlock()
doesn't need any code to save the caller-saved registers.
Make rcu_read_unlock() only two instructions in the slow path at the
caller site.
Cc: "Paul E. McKenney" <paulmck@xxxxxxxxxx>
Cc: Peter Zijlstra <peterz@xxxxxxxxxxxxx>
Cc: Frederic Weisbecker <frederic@xxxxxxxxxx>
Signed-off-by: Lai Jiangshan <jiangshan.ljs@xxxxxxxxxxxx>
---
arch/x86/entry/thunk_64.S | 5 +++++
arch/x86/include/asm/rcu_preempt.h | 4 +++-
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/arch/x86/entry/thunk_64.S b/arch/x86/entry/thunk_64.S
index 119ebdc3d362..10c60369a67c 100644
--- a/arch/x86/entry/thunk_64.S
+++ b/arch/x86/entry/thunk_64.S
@@ -13,3 +13,8 @@ THUNK preempt_schedule_thunk, preempt_schedule
THUNK preempt_schedule_notrace_thunk, preempt_schedule_notrace
EXPORT_SYMBOL(preempt_schedule_thunk)
EXPORT_SYMBOL(preempt_schedule_notrace_thunk)
+
+#ifdef CONFIG_PCPU_RCU_PREEMPT_COUNT
+THUNK rcu_read_unlock_special_thunk, rcu_read_unlock_special
+EXPORT_SYMBOL_GPL(rcu_read_unlock_special_thunk)
+#endif /* #ifdef CONFIG_PCPU_RCU_PREEMPT_COUNT */
diff --git a/arch/x86/include/asm/rcu_preempt.h b/arch/x86/include/asm/rcu_preempt.h
index cb25ebe038a5..acdd73b74c05 100644
--- a/arch/x86/include/asm/rcu_preempt.h
+++ b/arch/x86/include/asm/rcu_preempt.h
@@ -97,9 +97,11 @@ static __always_inline bool pcpu_rcu_preempt_count_dec_and_test(void)
__percpu_arg([var]));
}
+extern asmlinkage void rcu_read_unlock_special_thunk(void);
+
#define pcpu_rcu_read_unlock_special() \
do { \
- rcu_read_unlock_special(); \
+ asm volatile ("call rcu_read_unlock_special_thunk" : ASM_CALL_CONSTRAINT);\
} while (0)
#endif // #ifdef CONFIG_PCPU_RCU_PREEMPT_COUNT
--
2.19.1.6.gb485710b
