On 06/27/2011 08:42 AM, Bryan Schumaker wrote: > On 06/23/2011 03:55 PM, J. Bruce Fields wrote: >> On Fri, May 20, 2011 at 02:13:22PM -0400, Bryan Schumaker wrote: >>> I have been looking at adding fault injection to the NFS server in order >>> to test the client's state manager and recovery threads. Simulating >>> errors on the server seems like the easiest way to guarentee that >>> specific errors happen, and happen when we tell them to. >>> >>> This patch uses debugfs to add a simple framework for fault injection to >>> the server. This framework is a config option, and can be enabled >>> through CONFIG_NFSD_FAULT_INJECTION. Assuming you have debugfs mounted >>> to /sys/debug, a set of files will be created in /sys/debug/nfsd/. >>> Writing to any of these files will cause the corresponding action and >>> write a log entry to dmesg. >>> >>> Changes in v2: >>> - Replaced "forget all state owners" with "forget all open owners" >>> - Include fs/nfsd/fault_inject.c in the patch >>> >>> Suggestions? Comments? >> >> If this is still the most recent version of the patch, and if you can >> confirm that this has been tried and found useful for testing the >> client, then I'll go ahead and apply for 3.1. > > Yes, this is the most recent version of the patch. It is part of my Jenkins setup, but I haven't been running it on a regular basis recently. Trond and I just talked about more things to add in for fault injection, so I'll hopefully have a new version sometime in the near future. > > - Bryan > >> >> --b. >> >>> >>> - Bryan >>> --- >>> diff --git a/fs/nfsd/Kconfig b/fs/nfsd/Kconfig >>> index 18b3e89..8c02eab 100644 >>> --- a/fs/nfsd/Kconfig >>> +++ b/fs/nfsd/Kconfig >>> @@ -91,3 +91,16 @@ config NFSD_V4 >>> available from http://linux-nfs.org/. >>> >>> If unsure, say N. >>> + >>> +config NFSD_FAULT_INJECTION >>> + bool "NFS server manual fault injection" >>> + depends on NFSD_V4 && DEBUG_KERNEL >>> + help >>> + This option enables support for manually injecting faults >>> + into the NFS server. This is intended to be used for >>> + testing error recovery on the NFS client. >>> + >>> + If unsure, say N. >>> + >>> + >>> + >>> diff --git a/fs/nfsd/Makefile b/fs/nfsd/Makefile >>> index 9b118ee..69eae75 100644 >>> --- a/fs/nfsd/Makefile >>> +++ b/fs/nfsd/Makefile >>> @@ -5,7 +5,8 @@ >>> obj-$(CONFIG_NFSD) += nfsd.o >>> >>> nfsd-y := nfssvc.o nfsctl.o nfsproc.o nfsfh.o vfs.o \ >>> - export.o auth.o lockd.o nfscache.o nfsxdr.o stats.o >>> + export.o auth.o lockd.o nfscache.o nfsxdr.o stats.o \ >>> + fault_inject.o >>> nfsd-$(CONFIG_NFSD_V2_ACL) += nfs2acl.o >>> nfsd-$(CONFIG_NFSD_V3) += nfs3proc.o nfs3xdr.o >>> nfsd-$(CONFIG_NFSD_V3_ACL) += nfs3acl.o >>> diff --git a/fs/nfsd/fault_inject.c b/fs/nfsd/fault_inject.c >>> new file mode 100644 >>> index 0000000..3dd1c32 >>> --- /dev/null >>> +++ b/fs/nfsd/fault_inject.c >>> @@ -0,0 +1,106 @@ >>> + >>> +#ifdef CONFIG_NFSD_FAULT_INJECTION >>> + >>> +#include <linux/types.h> >>> +#include <linux/fs.h> >>> +#include <linux/debugfs.h> >>> +#include <linux/module.h> >>> + >>> +#include "state.h" >>> +#include "nfsd.h" >>> + >>> +static void nfsd_forget_everything(void); >>> + >>> +struct nfsd_fault_inject_op { >>> + char *name; >>> + char *text; >>> + int file_data; >>> + void (*func)(void); >>> +}; >>> + >>> +#define INJECTION_OP(op_name, op_text, op_func) \ >>> +{ \ >>> + .name = op_name, \ >>> + .text = op_text, \ >>> + .func = op_func, \ >>> +} >>> + >>> +static struct nfsd_fault_inject_op inject_ops[] = { >>> + INJECTION_OP("forget_clients", "forget all clients", nfsd_forget_all_clients), >>> + INJECTION_OP("forget_locks", "forget all locks", nfsd_forget_all_locks), >>> + INJECTION_OP("forget_open_owners", "forget all open owners", nfsd_forget_all_open_owners), >>> + INJECTION_OP("forget_delegations", "forget all delegations", nfsd_forget_all_delegations), >>> + INJECTION_OP("forget_everything", "forget everything", nfsd_forget_everything), >>> + INJECTION_OP("recall_delegations", "recall all delegations", nfsd_recall_all_delegations), >>> +}; >>> + >>> +static long int NUM_INJECT_OPS = sizeof(inject_ops) / sizeof(struct nfsd_fault_inject_op); >>> +static struct dentry *debug_dir; >>> + >>> +static void nfsd_forget_everything(void) >>> +{ >>> + nfsd_forget_all_clients(); >>> + nfsd_forget_all_locks(); >>> + nfsd_forget_all_open_owners(); >>> + nfsd_forget_all_delegations(); >>> +} >>> + >>> +static int nfsd_inject_set(void *data, u64 val) >>> +{ >>> + int i; >>> + for (i = 0; i < NUM_INJECT_OPS; i++) { >>> + if (&inject_ops[i].file_data == data) { >>> + printk(KERN_INFO "%s %s Server will %s", __FILE__, __func__, inject_ops[i].text); >>> + inject_ops[i].func(); >>> + } >>> + } >>> + return 0; >>> +} >>> + >>> +static int nfsd_inject_get(void *data, u64 *val) >>> +{ >>> + return 0; >>> +} >>> + >>> +DEFINE_SIMPLE_ATTRIBUTE(fops_nfsd, nfsd_inject_get, nfsd_inject_set, "%llu\n"); >>> + >>> +void >>> +nfsd_cleanup_fault_inject(void) >>> +{ >>> + debugfs_remove_recursive(debug_dir); >>> +} >>> + >>> +int >>> +nfsd_fault_inject_init(void) >>> +{ >>> + unsigned int i; >>> + struct nfsd_fault_inject_op *op; >>> + mode_t mode = S_IFREG | S_IRUSR | S_IWUSR; >>> + >>> + debug_dir = debugfs_create_dir("nfsd", NULL); >>> + if (!debug_dir) >>> + goto fail; >>> + >>> + for (i = 0; i < NUM_INJECT_OPS; i++) { >>> + op = &inject_ops[i]; >>> + debugfs_create_file(op->name, mode, debug_dir, &op->file_data, &fops_nfsd); >>> + } >>> + return 0; >>> +fail: >>> + nfsd_cleanup_fault_inject(); >>> + return -ENOMEM; >>> +} >>> + >>> +#else /* CONFIG_NFSD_FAULT_INJECTION */ >>> + >>> +inline void >>> +nfsd_cleanup_fault_inject(void) >>> +{} >>> + >>> +inline int >>> +nfsd_fault_inject_init(void) >>> +{ >>> + return 0; >>> +} >>> + >>> +#endif /* CONFIG_NFSD_FAULT_INJECTION */ >>> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c >>> index 4cf04e1..f555e02 100644 >>> --- a/fs/nfsd/nfs4state.c >>> +++ b/fs/nfsd/nfs4state.c >>> @@ -4274,6 +4274,144 @@ nfs4_check_open_reclaim(clientid_t *clid) >>> return nfs4_find_reclaim_client(clid) ? nfs_ok : nfserr_reclaim_bad; >>> } >>> >>> +#ifdef CONFIG_NFSD_FAULT_INJECTION >>> + >>> +void nfsd_forget_all_clients(void) >>> +{ >>> + struct nfs4_client *clp; >>> + struct list_head *pos, *next, reaplist; >>> + int count = 0; >>> + >>> + nfs4_lock_state(); >>> + INIT_LIST_HEAD(&reaplist); >>> + >>> + spin_lock(&client_lock); >>> + list_for_each_safe(pos, next, &client_lru) { >>> + clp = list_entry(pos, struct nfs4_client, cl_lru); >>> + unhash_client_locked(clp); >>> + list_add(&clp->cl_lru, &reaplist); >>> + count ++; >>> + } >>> + spin_unlock(&client_lock); >>> + list_for_each_safe(pos, next, &reaplist) { >>> + clp = list_entry(pos, struct nfs4_client, cl_lru); >>> + nfsd4_remove_clid_dir(clp); >>> + expire_client(clp); >>> + } >>> + >>> + nfs4_unlock_state(); >>> + printk(KERN_INFO "%s %s Forgot %d clients", __FILE__, __func__, count); >>> +} >>> + >>> +void nfsd_forget_all_locks(void) >>> +{ >>> + struct nfs4_stateid *stp; >>> + struct nfs4_stateowner *sop; >>> + struct list_head reaplist; >>> + int count = 0; >>> + int i; >>> + >>> + INIT_LIST_HEAD(&reaplist); >>> + nfs4_lock_state(); >>> + for (i = 0; i < LOCK_HASH_SIZE; i++) { >>> + list_for_each_entry(sop, &lock_ownerid_hashtbl[i], so_idhash) { >>> + list_for_each_entry(stp, &sop->so_stateids, st_perstateowner) { >>> + list_add(&sop->so_perclient, &reaplist); >>> + } >>> + } >>> + } >>> + >>> + while (!list_empty(&reaplist)) { >>> + sop = list_entry(reaplist.next, struct nfs4_stateowner, so_perclient); >>> + list_del(&sop->so_perclient); >>> + release_lockowner(sop); >>> + count ++; >>> + } >>> + nfs4_unlock_state(); >>> + >>> + printk(KERN_INFO "%s %s Forgot %d locks", __FILE__, __func__, count); >>> +} >>> + >>> +void nfsd_forget_all_open_owners(void) >>> +{ >>> + struct nfs4_stateid *stp, *next; >>> + unsigned int i; >>> + int count = 0; >>> + >>> + nfs4_lock_state(); >>> + for (i = 0; i < STATEID_HASH_SIZE; i++) { >>> + list_for_each_entry_safe(stp, next, &stateid_hashtbl[i], st_hash) { >>> + if (stp->st_openstp == NULL) { >>> + release_openowner(stp->st_stateowner); >>> + count++; >>> + } >>> + } >>> + } >>> + nfs4_unlock_state(); >>> + printk(KERN_INFO "%s %s Forgot %d open owners", __FILE__, __func__, count); >>> +} >>> + >>> +void nfsd_forget_all_delegations(void) >>> +{ >>> + struct nfs4_file *fp; >>> + struct nfs4_delegation *dp; >>> + struct list_head *pos, *next, reaplist; >>> + unsigned int i; >>> + unsigned int count = 0; >>> + >>> + INIT_LIST_HEAD(&reaplist); >>> + >>> + nfs4_lock_state(); >>> + spin_lock(&recall_lock); >>> + for (i = 0; i < FILE_HASH_SIZE; i++) { >>> + list_for_each_entry(fp, &file_hashtbl[i], fi_hash) { >>> + list_for_each(pos, &fp->fi_delegations) { >>> + dp = list_entry(pos, struct nfs4_delegation, dl_perfile); >>> + list_del_init(&dp->dl_perclnt); >>> + list_move(&dp->dl_recall_lru, &reaplist); >>> + } >>> + } >>> + } >>> + >>> + spin_unlock(&recall_lock); >>> + list_for_each_safe(pos, next, &reaplist) { >>> + dp = list_entry(pos, struct nfs4_delegation, dl_recall_lru); >>> + list_del_init(&dp->dl_recall_lru); >>> + unhash_delegation(dp); >>> + count++; >>> + } >>> + nfs4_unlock_state(); >>> + >>> + printk(KERN_INFO "%s %s Forgot %d delegations", __FILE__, __func__, count); >>> +} >>> + >>> +void nfsd_recall_all_delegations(void) >>> +{ >>> + struct nfs4_file *fp; >>> + struct nfs4_delegation *dp; >>> + struct list_head *pos; >>> + unsigned int i; >>> + unsigned int count = 0; >>> + >>> + nfs4_lock_state(); >>> + spin_lock(&recall_lock); >>> + for (i = 0; i < FILE_HASH_SIZE; i++) { >>> + list_for_each_entry(fp, &file_hashtbl[i], fi_hash) { >>> + list_for_each(pos, &fp->fi_delegations) { >>> + dp = list_entry(pos, struct nfs4_delegation, dl_perfile); >>> + nfsd_break_one_deleg(dp); >>> + count++; >>> + } >>> + } >>> + } >>> + spin_unlock(&recall_lock); >>> + nfs4_unlock_state(); >>> + printk(KERN_INFO "%s %s Recalled %d delegations", __FILE__, __func__, count); >>> +} >>> + >>> +#endif /* CONFIG_NFSD_FAULT_INJECTION */ >>> + >>> + >>> /* initialization to perform at module load time: */ >>> >>> int >>> diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c >>> index 1f5eae4..1eb3e92 100644 >>> --- a/fs/nfsd/nfsctl.c >>> +++ b/fs/nfsd/nfsctl.c >>> @@ -1481,6 +1481,9 @@ static int __init init_nfsd(void) >>> retval = nfs4_state_init(); /* nfs4 locking state */ >>> if (retval) >>> return retval; >>> + retval = nfsd_fault_inject_init(); /* nfsd fault injection controls */ >>> + if (retval) >>> + goto out_cleanup_fault_injection; >>> nfsd_stat_init(); /* Statistics */ >>> retval = nfsd_reply_cache_init(); >>> if (retval) >>> @@ -1512,6 +1515,8 @@ out_free_cache: >>> out_free_stat: >>> nfsd_stat_shutdown(); >>> nfsd4_free_slabs(); >>> +out_cleanup_fault_injection: >>> + nfsd_cleanup_fault_inject(); >>> return retval; >>> } >>> >>> @@ -1525,6 +1530,7 @@ static void __exit exit_nfsd(void) >>> nfsd_lockd_shutdown(); >>> nfsd_idmap_shutdown(); >>> nfsd4_free_slabs(); >>> + nfsd_cleanup_fault_inject(); >>> unregister_filesystem(&nfsd_fs_type); >>> } >>> >>> diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h >>> index 7ecfa24..4a6cded 100644 >>> --- a/fs/nfsd/nfsd.h >>> +++ b/fs/nfsd/nfsd.h >>> @@ -79,11 +79,18 @@ static inline int nfsd_v4client(struct svc_rqst *rq) >>> #ifdef CONFIG_NFSD_V4 >>> extern unsigned int max_delegations; >>> int nfs4_state_init(void); >>> +int nfsd_fault_inject_init(void); >>> +void nfsd_cleanup_fault_inject(void); >>> void nfsd4_free_slabs(void); >>> int nfs4_state_start(void); >>> void nfs4_state_shutdown(void); >>> void nfs4_reset_lease(time_t leasetime); >>> int nfs4_reset_recoverydir(char *recdir); >>> +void nfsd_forget_all_clients(void); >>> +void nfsd_forget_all_locks(void); >>> +void nfsd_forget_all_open_owners(void); >>> +void nfsd_forget_all_delegations(void); >>> +void nfsd_recall_all_delegations(void); >>> #else >>> static inline int nfs4_state_init(void) { return 0; } >>> static inline void nfsd4_free_slabs(void) { } >>> @@ -91,6 +98,11 @@ static inline int nfs4_state_start(void) { return 0; } >>> static inline void nfs4_state_shutdown(void) { } >>> static inline void nfs4_reset_lease(time_t leasetime) { } >>> static inline int nfs4_reset_recoverydir(char *recdir) { return 0; } >>> +static inline void nfsd_forget_all_clients(void) {}; >>> +static inline void nfsd_forget_all_locks(void) {}; >>> +static inline void nfsd_forget_all_open_owners(void) {}; >>> +static inline void nfsd_forget_all_delegations(void) {}; >>> +static inline void nfsd_recall_all_delegations(void) {}; >>> #endif >>> >>> /* > > -- > To unsubscribe from this list: send the line "unsubscribe linux-nfs" in > the body of a message to majordomo@xxxxxxxxxxxxxxx > More majordomo info at http://vger.kernel.org/majordomo-info.html -- To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to majordomo@xxxxxxxxxxxxxxx More majordomo info at http://vger.kernel.org/majordomo-info.html
