Currently, the start_poll_synchronize_rcu_expedited() can be invoked
very early. before rcu_init(), the rcu_data structure's->mynode is not
initialized, if invoke start_poll_synchronize_rcu_expedited() before
rcu_init(), will trigger a null rcu_node structure's->exp_seq_poll access.
This commit add boot_exp_seq_poll_rq member to rcu_state structure to
store seq number return by invoke start_poll_synchronize_rcu_expedited()
very early.
Fixes: d96c52fe4907 ("rcu: Add polled expedited grace-period primitives")
Signed-off-by: Zqiang <qiang1.zhang@xxxxxxxxx>
---
kernel/rcu/tree.c | 3 ++-
kernel/rcu/tree.h | 1 +
kernel/rcu/tree_exp.h | 6 ++++--
3 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 63545d79da51..12f0891ce7f4 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -92,6 +92,7 @@ static struct rcu_state rcu_state = {
.exp_mutex = __MUTEX_INITIALIZER(rcu_state.exp_mutex),
.exp_wake_mutex = __MUTEX_INITIALIZER(rcu_state.exp_wake_mutex),
.ofl_lock = __ARCH_SPIN_LOCK_UNLOCKED,
+ .boot_exp_seq_poll_rq = RCU_GET_STATE_COMPLETED,
};
/* Dump rcu_node combining tree at boot to verify correct setup. */
@@ -4938,7 +4939,7 @@ void __init rcu_init(void)
qovld_calc = qovld;
// Kick-start any polled grace periods that started early.
- if (!(per_cpu_ptr(&rcu_data, cpu)->mynode->exp_seq_poll_rq & 0x1))
+ if (!(rcu_state.boot_exp_seq_poll_rq & 0x1))
(void)start_poll_synchronize_rcu_expedited();
rcu_test_sync_prims();
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index 192536916f9a..ae50ca6853ad 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -397,6 +397,7 @@ struct rcu_state {
/* Synchronize offline with */
/* GP pre-initialization. */
int nocb_is_setup; /* nocb is setup from boot */
+ unsigned long boot_exp_seq_poll_rq;
};
/* Values for rcu_state structure's gp_flags field. */
diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h
index 956cd459ba7f..1b35a1e233d9 100644
--- a/kernel/rcu/tree_exp.h
+++ b/kernel/rcu/tree_exp.h
@@ -1068,9 +1068,11 @@ unsigned long start_poll_synchronize_rcu_expedited(void)
if (rcu_init_invoked())
raw_spin_lock_irqsave(&rnp->exp_poll_lock, flags);
if (!poll_state_synchronize_rcu(s)) {
- rnp->exp_seq_poll_rq = s;
- if (rcu_init_invoked())
+ if (rcu_init_invoked()) {
+ rnp->exp_seq_poll_rq = s;
queue_work(rcu_gp_wq, &rnp->exp_poll_wq);
+ } else
+ rcu_state.boot_exp_seq_poll_rq = s;
}
if (rcu_init_invoked())
raw_spin_unlock_irqrestore(&rnp->exp_poll_lock, flags);
--
2.25.1
