On Tue, 2013-01-08 at 17:19 -0500, Chris Perl wrote:
+AD4- On Tue, Jan 08, 2013 at 05:16:51PM -0500, Chris Perl wrote:
+AD4- +AD4- +AD4- The lock is associated with the rpc+AF8-task. Threads can normally only
+AD4- +AD4- +AD4- access an rpc+AF8-task when it is on a wait queue (while holding the wait
+AD4- +AD4- +AD4- queue lock) unless they are given ownership of the rpc+AF8-task.
+AD4- +AD4- +AD4-
+AD4- +AD4- +AD4- IOW: the scenario you describe should not be possible, since it relies
+AD4- +AD4- +AD4- on thread 1 assigning the lock to the rpc+AF8-task after it has been removed
+AD4- +AD4- +AD4- from the wait queue.
+AD4- +AD4-
+AD4- +AD4- Hrm. I guess I'm in over my head here. Apologoies if I'm just asking
+AD4- +AD4- silly bumbling questions. You can start ignoring me at any time. :)
+AD4- +AD4-
+AD4- +AD4- I was talking about setting (or leaving set) the XPRT+AF8-LOCKED bit in
+AD4- +AD4- rpc+AF8-xprt-+AD4-state. By +ACI-assigning the lock+ACI- I really just mean that thread
+AD4- +AD4- 1 leaves XPRT+AF8-LOCKED set in rpc+AF8-xprt-+AD4-state and sets rpc+AF8-xprt-+AD4-snd+AF8-task
+AD4- +AD4- to thread 2.
+AD4- +AD4-
+AD4- +AD4- +AD4- If you are recompiling the kernel, perhaps you can also add in a patch
+AD4- +AD4- +AD4- to rpc+AF8-show+AF8-tasks() to display the current value of
+AD4- +AD4- +AD4- clnt-+AD4-cl+AF8-xprt-+AD4-snd+AF8-task?
+AD4- +AD4-
+AD4- +AD4- Sure. This is what 'echo 0 +AD4- /proc/sys/sunrpc/rpc+AF8-debug' shows after
+AD4- +AD4- the hang (with my extra prints):
+AD4- +AD4-
+AD4- +AD4- +ACM- cat /proc/kmsg
+AD4- +AD4- ...
+AD4- +AD4- +ADw-6+AD4-client: ffff88082b6c9c00, xprt: ffff880824aef800, snd+AF8-task: ffff881029c63ec0
+AD4- +AD4- +ADw-6+AD4-client: ffff88082b6c9e00, xprt: ffff880824aef800, snd+AF8-task: ffff881029c63ec0
+AD4- +AD4- +ADw-6+AD4--pid- flgs status -client- --rqstp- -timeout ---ops--
+AD4- +AD4- +ADw-6+AD4-18091 0080 -11 ffff88082b6c9e00 (null) ffff0770ns3 ACCESS a:call+AF8-reserveresult q:xprt+AF8-sending
+AD4- +AD4- +ADw-6+AD4-client: ffff88082a244600, xprt: ffff88082a343000, snd+AF8-task: (null)
+AD4- +AD4- +ADw-6+AD4-client: ffff880829181600, xprt: ffff88082a343000, snd+AF8-task: (null)
+AD4- +AD4- +ADw-6+AD4-client: ffff880828170200, xprt: ffff880824aef800, snd+AF8-task: ffff881029c63ec0
+AD4-
+AD4- Sorry, that output was a little messed up. Here it is again:
+AD4-
+AD4- +ADw-6+AD4-client: ffff88082b6c9c00, xprt: ffff880824aef800, snd+AF8-task: ffff881029c63ec0
+AD4- +ADw-6+AD4-client: ffff88082b6c9e00, xprt: ffff880824aef800, snd+AF8-task: ffff881029c63ec0
+AD4- +ADw-6+AD4--pid- flgs status -client- --rqstp- -timeout ---ops--
+AD4- +ADw-6+AD4-18091 0080 -11 ffff88082b6c9e00 (null) 0 ffffffffa027b7e0 nfsv3 ACCESS a:call+AF8-reserveresult q:xprt+AF8-sending
+AD4- +ADw-6+AD4-client: ffff88082a244600, xprt: ffff88082a343000, snd+AF8-task: (null)
+AD4- +ADw-6+AD4-client: ffff880829181600, xprt: ffff88082a343000, snd+AF8-task: (null)
+AD4- +ADw-6+AD4-client: ffff880828170200, xprt: ffff880824aef800, snd+AF8-task: ffff881029c63ec0
Hi Chris,
It looks as if the problem here is that the rpc+AF8-task in question is not
being woken up. I'm aware of at least one problem with priority queues
in RHEL-6.3/CentOS-6.3, and that has been fixed in the upstream kernel.
See attachment.
Cheers
Trond
--
Trond Myklebust
Linux NFS client maintainer
NetApp
Trond.Myklebust+AEA-netapp.com
www.netapp.com
From c05eecf636101dd4347b2d8fa457626bf0088e0a Mon Sep 17 00:00:00 2001
From: Trond Myklebust <Trond.Myklebust@xxxxxxxxxx>
Date: Fri, 30 Nov 2012 23:59:29 -0500
Subject: [PATCH] SUNRPC: Don't allow low priority tasks to pre-empt higher
priority ones
Currently, the priority queues attempt to be 'fair' to lower priority
tasks by scheduling them after a certain number of higher priority tasks
have run. The problem is that both the transport send queue and
the NFSv4.1 session slot queue have strong ordering requirements.
This patch therefore removes the fairness code in favour of strong
ordering of task priorities.
Signed-off-by: Trond Myklebust <Trond.Myklebust@xxxxxxxxxx>
---
include/linux/sunrpc/sched.h | 1 -
net/sunrpc/sched.c | 44 ++++++++++++++++++++++----------------------
2 files changed, 22 insertions(+), 23 deletions(-)
diff --git a/include/linux/sunrpc/sched.h b/include/linux/sunrpc/sched.h
index dc0c3cc..b64f8eb 100644
--- a/include/linux/sunrpc/sched.h
+++ b/include/linux/sunrpc/sched.h
@@ -192,7 +192,6 @@ struct rpc_wait_queue {
pid_t owner; /* process id of last task serviced */
unsigned char maxpriority; /* maximum priority (0 if queue is not a priority queue) */
unsigned char priority; /* current priority */
- unsigned char count; /* # task groups remaining serviced so far */
unsigned char nr; /* # tasks remaining for cookie */
unsigned short qlen; /* total # tasks waiting in queue */
struct rpc_timer timer_list;
diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c
index 1aefc9f..d17a704 100644
--- a/net/sunrpc/sched.c
+++ b/net/sunrpc/sched.c
@@ -98,6 +98,23 @@ __rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task)
list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list);
}
+static void rpc_set_waitqueue_priority(struct rpc_wait_queue *queue, int priority)
+{
+ queue->priority = priority;
+}
+
+static void rpc_set_waitqueue_owner(struct rpc_wait_queue *queue, pid_t pid)
+{
+ queue->owner = pid;
+ queue->nr = RPC_BATCH_COUNT;
+}
+
+static void rpc_reset_waitqueue_priority(struct rpc_wait_queue *queue)
+{
+ rpc_set_waitqueue_priority(queue, queue->maxpriority);
+ rpc_set_waitqueue_owner(queue, 0);
+}
+
/*
* Add new request to a priority queue.
*/
@@ -109,9 +126,11 @@ static void __rpc_add_wait_queue_priority(struct rpc_wait_queue *queue,
struct rpc_task *t;
INIT_LIST_HEAD(&task->u.tk_wait.links);
- q = &queue->tasks[queue_priority];
if (unlikely(queue_priority > queue->maxpriority))
- q = &queue->tasks[queue->maxpriority];
+ queue_priority = queue->maxpriority;
+ if (queue_priority > queue->priority)
+ rpc_set_waitqueue_priority(queue, queue_priority);
+ q = &queue->tasks[queue_priority];
list_for_each_entry(t, q, u.tk_wait.list) {
if (t->tk_owner == task->tk_owner) {
list_add_tail(&task->u.tk_wait.list, &t->u.tk_wait.links);
@@ -180,24 +199,6 @@ static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_tas
task->tk_pid, queue, rpc_qname(queue));
}
-static inline void rpc_set_waitqueue_priority(struct rpc_wait_queue *queue, int priority)
-{
- queue->priority = priority;
- queue->count = 1 << (priority * 2);
-}
-
-static inline void rpc_set_waitqueue_owner(struct rpc_wait_queue *queue, pid_t pid)
-{
- queue->owner = pid;
- queue->nr = RPC_BATCH_COUNT;
-}
-
-static inline void rpc_reset_waitqueue_priority(struct rpc_wait_queue *queue)
-{
- rpc_set_waitqueue_priority(queue, queue->maxpriority);
- rpc_set_waitqueue_owner(queue, 0);
-}
-
static void __rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname, unsigned char nr_queues)
{
int i;
@@ -464,8 +465,7 @@ static struct rpc_task *__rpc_find_next_queued_priority(struct rpc_wait_queue *q
/*
* Check if we need to switch queues.
*/
- if (--queue->count)
- goto new_owner;
+ goto new_owner;
}
/*
--
1.7.11.7
