On Dec 12, 2010, at 3:59 PM, Fred Isaman wrote:
> On Mon, Dec 6, 2010 at 10:19 PM, <andros@xxxxxxxxxx> wrote:
>> From: Andy Adamson <andros@xxxxxxxxxx>
>>
>> Currently session draining only drains the fore channel.
>> The back channel processing must also be drained.
>>
>> Use the back channel highest_slot_used to indicate that a callback is being
>> processed by the callback thread. Move the session complete to be per channel.
>>
>> When the session is draininig, wait for any current back channel processing
>> to complete and stop all new back channel processing by returning NFS4ERR_DELAY
>> to the back channel client.
>>
>> Drain the back channel, then the fore channel.
>>
>> Signed-off-by: Andy Adamson <andros@xxxxxxxxxx>
>> ---
>> fs/nfs/callback.h | 1 +
>> fs/nfs/callback_proc.c | 6 ++++++
>> fs/nfs/callback_xdr.c | 17 +++++++++++++++++
>> fs/nfs/nfs4proc.c | 26 +++++++++++++++++++-------
>> fs/nfs/nfs4state.c | 29 ++++++++++++++++++++++-------
>> include/linux/nfs_fs_sb.h | 2 +-
>> 6 files changed, 66 insertions(+), 15 deletions(-)
>>
>> diff --git a/fs/nfs/callback.h b/fs/nfs/callback.h
>> index 85a7cfd..434233e 100644
>> --- a/fs/nfs/callback.h
>> +++ b/fs/nfs/callback.h
>> @@ -131,6 +131,7 @@ extern unsigned nfs4_callback_recallslot(struct cb_recallslotargs *args,
>>
>> extern __be32 nfs4_callback_getattr(struct cb_getattrargs *args, struct cb_getattrres *res);
>> extern __be32 nfs4_callback_recall(struct cb_recallargs *args, void *dummy);
>> +extern void nfs4_check_drain_bc_complete(struct nfs4_session *ses);
>>
>> #ifdef CONFIG_NFS_V4
>> extern int nfs_callback_up(u32 minorversion, struct rpc_xprt *xprt);
>> diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c
>> index 2950fca..7a976ef 100644
>> --- a/fs/nfs/callback_proc.c
>> +++ b/fs/nfs/callback_proc.c
>> @@ -287,6 +287,12 @@ __be32 nfs4_callback_sequence(struct cb_sequenceargs *args,
>> if (clp == NULL)
>> goto out;
>>
>> + /* state manager is resetting the session */
>> + if (test_bit(NFS4_SESSION_DRAINING, &clp->cl_session->session_state)) {
>> + status = NFS4ERR_DELAY;
>> + goto out_putclient;
>> + }
>> +
>> status = validate_seqid(&clp->cl_session->bc_slot_table, args);
>> if (status)
>> goto out_putclient;
>> diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c
>> index 05af212..0a5e32d 100644
>> --- a/fs/nfs/callback_xdr.c
>> +++ b/fs/nfs/callback_xdr.c
>> @@ -593,6 +593,20 @@ preprocess_nfs41_op(int nop, unsigned int op_nr, struct callback_op **op)
>> return htonl(NFS_OK);
>> }
>>
>> +static void nfs4_callback_free_slot(struct nfs4_session *session)
>
>
> Am I missing something, or was the call to this function omitted?
You're right, I omitted it - should be in nfs4_callback_compound. I'll check all these patches again, and resend to Trond who I suspect lost them in the flurry of patches...
-->Andy
>
> Fred
>
>> +{
>> + struct nfs4_slot_table *tbl = &session->bc_slot_table;
>> +
>> + spin_lock(&tbl->slot_tbl_lock);
>> + /*
>> + * Let the state manager know callback processing done.
>> + * A single slot, so highest used slotid is either 0 or -1
>> + */
>> + tbl->highest_used_slotid--;
>> + nfs4_check_drain_bc_complete(session);
>> + spin_unlock(&tbl->slot_tbl_lock);
>> +}
>> +
>> #else /* CONFIG_NFS_V4_1 */
>>
>> static __be32
>> @@ -601,6 +615,9 @@ preprocess_nfs41_op(int nop, unsigned int op_nr, struct callback_op **op)
>> return htonl(NFS4ERR_MINOR_VERS_MISMATCH);
>> }
>>
>> +static void nfs4_callback_free_slot(struct nfs4_session *session)
>> +{
>> +}
>> #endif /* CONFIG_NFS_V4_1 */
>>
>> static __be32
>> diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
>> index 6a653ff..b48f607 100644
>> --- a/fs/nfs/nfs4proc.c
>> +++ b/fs/nfs/nfs4proc.c
>> @@ -355,9 +355,9 @@ nfs4_free_slot(struct nfs4_slot_table *tbl, struct nfs4_slot *free_slot)
>> }
>>
>> /*
>> - * Signal state manager thread if session is drained
>> + * Signal state manager thread if session fore channel is drained
>> */
>> -static void nfs41_check_drain_session_complete(struct nfs4_session *ses)
>> +static void nfs4_check_drain_fc_complete(struct nfs4_session *ses)
>> {
>> struct rpc_task *task;
>>
>> @@ -371,8 +371,20 @@ static void nfs41_check_drain_session_complete(struct nfs4_session *ses)
>> if (ses->fc_slot_table.highest_used_slotid != -1)
>> return;
>>
>> - dprintk("%s COMPLETE: Session Drained\n", __func__);
>> - complete(&ses->complete);
>> + dprintk("%s COMPLETE: Session Fore Channel Drained\n", __func__);
>> + complete(&ses->fc_slot_table.complete);
>> +}
>> +
>> +/*
>> + * Signal state manager thread if session back channel is drained
>> + */
>> +void nfs4_check_drain_bc_complete(struct nfs4_session *ses)
>> +{
>> + if (!test_bit(NFS4_SESSION_DRAINING, &ses->session_state) ||
>> + ses->bc_slot_table.highest_used_slotid != -1)
>> + return;
>> + dprintk("%s COMPLETE: Session Back Channel Drained\n", __func__);
>> + complete(&ses->bc_slot_table.complete);
>> }
>>
>> static void nfs41_sequence_free_slot(struct nfs4_sequence_res *res)
>> @@ -389,7 +401,7 @@ static void nfs41_sequence_free_slot(struct nfs4_sequence_res *res)
>>
>> spin_lock(&tbl->slot_tbl_lock);
>> nfs4_free_slot(tbl, res->sr_slot);
>> - nfs41_check_drain_session_complete(res->sr_session);
>> + nfs4_check_drain_fc_complete(res->sr_session);
>> spin_unlock(&tbl->slot_tbl_lock);
>> res->sr_slot = NULL;
>> }
>> @@ -4767,17 +4779,17 @@ struct nfs4_session *nfs4_alloc_session(struct nfs_client *clp)
>> if (!session)
>> return NULL;
>>
>> - init_completion(&session->complete);
>> -
>> tbl = &session->fc_slot_table;
>> tbl->highest_used_slotid = -1;
>> spin_lock_init(&tbl->slot_tbl_lock);
>> rpc_init_priority_wait_queue(&tbl->slot_tbl_waitq, "ForeChannel Slot table");
>> + init_completion(&tbl->complete);
>>
>> tbl = &session->bc_slot_table;
>> tbl->highest_used_slotid = -1;
>> spin_lock_init(&tbl->slot_tbl_lock);
>> rpc_init_wait_queue(&tbl->slot_tbl_waitq, "BackChannel Slot table");
>> + init_completion(&tbl->complete);
>>
>> session->session_state = 1<<NFS4_SESSION_INITING;
>>
>> diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c
>> index f575a31..0082483 100644
>> --- a/fs/nfs/nfs4state.c
>> +++ b/fs/nfs/nfs4state.c
>> @@ -142,6 +142,11 @@ static int nfs41_setup_state_renewal(struct nfs_client *clp)
>> return status;
>> }
>>
>> +/*
>> + * Back channel returns NFS4ERR_DELAY for new requests when
>> + * NFS4_SESSION_DRAINING is set so there is no work to be done when draining
>> + * is ended.
>> + */
>> static void nfs4_end_drain_session(struct nfs_client *clp)
>> {
>> struct nfs4_session *ses = clp->cl_session;
>> @@ -165,21 +170,31 @@ static void nfs4_end_drain_session(struct nfs_client *clp)
>> }
>> }
>>
>> -static int nfs4_begin_drain_session(struct nfs_client *clp)
>> +static int nfs4_wait_on_slot_tbl(struct nfs4_slot_table *tbl)
>> {
>> - struct nfs4_session *ses = clp->cl_session;
>> - struct nfs4_slot_table *tbl = &ses->fc_slot_table;
>> -
>> spin_lock(&tbl->slot_tbl_lock);
>> - set_bit(NFS4_SESSION_DRAINING, &ses->session_state);
>> if (tbl->highest_used_slotid != -1) {
>> - INIT_COMPLETION(ses->complete);
>> + INIT_COMPLETION(tbl->complete);
>> spin_unlock(&tbl->slot_tbl_lock);
>> - return wait_for_completion_interruptible(&ses->complete);
>> + return wait_for_completion_interruptible(&tbl->complete);
>> }
>> spin_unlock(&tbl->slot_tbl_lock);
>> return 0;
>> }
>> +
>> +static int nfs4_begin_drain_session(struct nfs_client *clp)
>> +{
>> + struct nfs4_session *ses = clp->cl_session;
>> + int ret = 0;
>> +
>> + set_bit(NFS4_SESSION_DRAINING, &ses->session_state);
>> + /* back channel */
>> + ret = nfs4_wait_on_slot_tbl(&ses->bc_slot_table);
>> + if (ret)
>> + return ret;
>> + /* fore channel */
>> + return nfs4_wait_on_slot_tbl(&ses->fc_slot_table);
>> +}
>>
>> int nfs41_init_clientid(struct nfs_client *clp, struct rpc_cred *cred)
>> {
>> diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h
>> index 452d964..8ded472 100644
>> --- a/include/linux/nfs_fs_sb.h
>> +++ b/include/linux/nfs_fs_sb.h
>> @@ -196,6 +196,7 @@ struct nfs4_slot_table {
>> * op for dynamic resizing */
>> int target_max_slots; /* Set by CB_RECALL_SLOT as
>> * the new max_slots */
>> + struct completion complete;
>> };
>>
>> static inline int slot_idx(struct nfs4_slot_table *tbl, struct nfs4_slot *sp)
>> @@ -212,7 +213,6 @@ struct nfs4_session {
>> unsigned long session_state;
>> u32 hash_alg;
>> u32 ssv_len;
>> - struct completion complete;
>>
>> /* The fore and back channel */
>> struct nfs4_channel_attrs fc_attrs;
>> --
>> 1.6.6
>>
>> --
>> To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
>> the body of a message to majordomo@xxxxxxxxxxxxxxx
>> More majordomo info at http://vger.kernel.org/majordomo-info.html
>>
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
