Move synchronize_rcu() main control data under the rcu_state
structure. An access is done via "rcu_state" global variable.
Signed-off-by: Uladzislau Rezki (Sony) <urezki@xxxxxxxxx>
---
kernel/rcu/tree.c | 50 ++++++++++++++---------------------------------
kernel/rcu/tree.h | 19 ++++++++++++++++++
2 files changed, 34 insertions(+), 35 deletions(-)
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 69663a6d5050..c0d3e46730e8 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -1384,19 +1384,6 @@ static void rcu_poll_gp_seq_end_unlocked(unsigned long *snap)
raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
}
-/*
- * A max threshold for synchronize_rcu() users which are
- * awaken directly by the rcu_gp_kthread(). Left part is
- * deferred to the main worker.
- */
-#define SR_MAX_USERS_WAKE_FROM_GP 5
-#define SR_NORMAL_GP_WAIT_HEAD_MAX 5
-
-struct sr_wait_node {
- atomic_t inuse;
- struct llist_node node;
-};
-
/*
* There is a single llist, which is used for handling
* synchronize_rcu() users' enqueued rcu_synchronize nodes.
@@ -1523,17 +1510,10 @@ struct sr_wait_node {
* +----------+ +--------+
*
*/
-static struct sr_normal_state {
- struct llist_head srs_next; /* request a GP users. */
- struct llist_node *srs_wait_tail; /* wait for GP users. */
- struct llist_node *srs_done_tail; /* ready for GP users. */
- struct sr_wait_node srs_wait_nodes[SR_NORMAL_GP_WAIT_HEAD_MAX];
-} sr;
-
static bool rcu_sr_is_wait_head(struct llist_node *node)
{
- return &(sr.srs_wait_nodes)[0].node <= node &&
- node <= &(sr.srs_wait_nodes)[SR_NORMAL_GP_WAIT_HEAD_MAX - 1].node;
+ return &(rcu_state.srs_wait_nodes)[0].node <= node &&
+ node <= &(rcu_state.srs_wait_nodes)[SR_NORMAL_GP_WAIT_HEAD_MAX - 1].node;
}
static struct llist_node *rcu_sr_get_wait_head(void)
@@ -1542,7 +1522,7 @@ static struct llist_node *rcu_sr_get_wait_head(void)
int i;
for (i = 0; i < SR_NORMAL_GP_WAIT_HEAD_MAX; i++) {
- sr_wn = &(sr.srs_wait_nodes)[i];
+ sr_wn = &(rcu_state.srs_wait_nodes)[i];
if (!atomic_cmpxchg_acquire(&sr_wn->inuse, 0, 1))
return &sr_wn->node;
@@ -1590,7 +1570,7 @@ static void rcu_sr_normal_gp_cleanup_work(struct work_struct *work)
* cannot execute concurrently by multiple kworkers,
* the done tail list manipulations are protected here.
*/
- done = smp_load_acquire(&sr.srs_done_tail);
+ done = smp_load_acquire(&rcu_state.srs_done_tail);
if (!done)
return;
@@ -1626,12 +1606,12 @@ static void rcu_sr_normal_gp_cleanup(void)
struct llist_node *wait_tail, *head, *rcu;
int done = 0;
- wait_tail = sr.srs_wait_tail;
+ wait_tail = rcu_state.srs_wait_tail;
if (wait_tail == NULL)
return;
- sr.srs_wait_tail = NULL;
- ASSERT_EXCLUSIVE_WRITER(sr.srs_wait_tail);
+ rcu_state.srs_wait_tail = NULL;
+ ASSERT_EXCLUSIVE_WRITER(rcu_state.srs_wait_tail);
WARN_ON_ONCE(!rcu_sr_is_wait_head(wait_tail));
head = wait_tail->next;
@@ -1662,8 +1642,8 @@ static void rcu_sr_normal_gp_cleanup(void)
}
// concurrent sr_normal_gp_cleanup work might observe this update.
- smp_store_release(&sr.srs_done_tail, wait_tail);
- ASSERT_EXCLUSIVE_WRITER(sr.srs_done_tail);
+ smp_store_release(&rcu_state.srs_done_tail, wait_tail);
+ ASSERT_EXCLUSIVE_WRITER(rcu_state.srs_done_tail);
if (wait_tail->next)
queue_work(system_highpri_wq, &sr_normal_gp_cleanup);
@@ -1678,7 +1658,7 @@ static bool rcu_sr_normal_gp_init(void)
struct llist_node *wait_head;
bool start_new_poll = false;
- first = READ_ONCE(sr.srs_next.first);
+ first = READ_ONCE(rcu_state.srs_next.first);
if (!first || rcu_sr_is_wait_head(first))
return start_new_poll;
@@ -1690,23 +1670,23 @@ static bool rcu_sr_normal_gp_init(void)
}
/* Inject a wait-dummy-node. */
- llist_add(wait_head, &sr.srs_next);
+ llist_add(wait_head, &rcu_state.srs_next);
/*
* A waiting list of rcu_synchronize nodes should be empty on
* this step, since a GP-kthread, rcu_gp_init() -> gp_cleanup(),
* rolls it over. If not, it is a BUG, warn a user.
*/
- WARN_ON_ONCE(sr.srs_wait_tail != NULL);
- sr.srs_wait_tail = wait_head;
- ASSERT_EXCLUSIVE_WRITER(sr.srs_wait_tail);
+ WARN_ON_ONCE(rcu_state.srs_wait_tail != NULL);
+ rcu_state.srs_wait_tail = wait_head;
+ ASSERT_EXCLUSIVE_WRITER(rcu_state.srs_wait_tail);
return start_new_poll;
}
static void rcu_sr_normal_add_req(struct rcu_synchronize *rs)
{
- llist_add((struct llist_node *) &rs->head, &sr.srs_next);
+ llist_add((struct llist_node *) &rs->head, &rcu_state.srs_next);
}
/*
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index 192536916f9a..f72166b5067a 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -316,6 +316,19 @@ do { \
__set_current_state(TASK_RUNNING); \
} while (0)
+/*
+ * A max threshold for synchronize_rcu() users which are
+ * awaken directly by the rcu_gp_kthread(). Left part is
+ * deferred to the main worker.
+ */
+#define SR_MAX_USERS_WAKE_FROM_GP 5
+#define SR_NORMAL_GP_WAIT_HEAD_MAX 5
+
+struct sr_wait_node {
+ atomic_t inuse;
+ struct llist_node node;
+};
+
/*
* RCU global state, including node hierarchy. This hierarchy is
* represented in "heap" form in a dense array. The root (first level)
@@ -397,6 +410,12 @@ struct rcu_state {
/* Synchronize offline with */
/* GP pre-initialization. */
int nocb_is_setup; /* nocb is setup from boot */
+
+ /* synchronize_rcu() part. */
+ struct llist_head srs_next; /* request a GP users. */
+ struct llist_node *srs_wait_tail; /* wait for GP users. */
+ struct llist_node *srs_done_tail; /* ready for GP users. */
+ struct sr_wait_node srs_wait_nodes[SR_NORMAL_GP_WAIT_HEAD_MAX];
};
/* Values for rcu_state structure's gp_flags field. */
--
2.39.2
