> From: Sujit Reddy Thumma <sthumma@xxxxxxxxxxxxxx>
>
> Use fault-injection framework to simulate error conditions
> in the controller and verify error handling mechanisms
> implemented in UFS host controller driver.
>
> This is used only during development and hence
> guarded by CONFIG_UFS_FAULT_INJECTION debug config option.
>
> Signed-off-by: Sujit Reddy Thumma <sthumma@xxxxxxxxxxxxxx>
> ---
> drivers/scsi/ufs/ufs-debugfs.c | 140
> +++++++++++++++++++++++++++++++++++++++++
> drivers/scsi/ufs/ufs-debugfs.h | 4 ++
> drivers/scsi/ufs/ufshcd.c | 2 +
> drivers/scsi/ufs/ufshcd.h | 5 ++
> lib/Kconfig.debug | 14 +++++
> 5 files changed, 165 insertions(+)
>
> diff --git a/drivers/scsi/ufs/ufs-debugfs.c
> b/drivers/scsi/ufs/ufs-debugfs.c
> index d1eb4f8..53dcb00 100644
> --- a/drivers/scsi/ufs/ufs-debugfs.c
> +++ b/drivers/scsi/ufs/ufs-debugfs.c
> @@ -17,6 +17,7 @@
> *
> */
>
> +#include <linux/random.h>
> #include "ufs-debugfs.h"
> #include "unipro.h"
>
> @@ -41,6 +42,143 @@ struct desc_field_offset {
> } while (0)
> #define DOORBELL_CLR_TOUT_US (1000 * 1000) /* 1 sec */
>
> +#ifdef CONFIG_UFS_FAULT_INJECTION
> +
> +#define INJECT_COMMAND_HANG (0x0)
> +
> +static DECLARE_FAULT_ATTR(fail_default_attr);
> +static char *fail_request;
> +module_param(fail_request, charp, 0);
> +
> +static bool inject_fatal_err_tr(struct ufs_hba *hba, u8 ocs_err)
> +{
> + int tag;
> +
> + tag = find_first_bit(&hba->outstanding_reqs, hba->nutrs);
> + if (tag == hba->nutrs)
> + return 0;
> +
> + ufshcd_writel(hba, ~(1 << tag), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
> + (&hba->lrb[tag])->utr_descriptor_ptr->header.dword_2 =
> +
> cpu_to_be32(ocs_err);
> +
> + /* fatal error injected */
> + return 1;
> +}
> +
> +static bool inject_fatal_err_tm(struct ufs_hba *hba, u8 ocs_err)
> +{
> + int tag;
> +
> + tag = find_first_bit(&hba->outstanding_tasks, hba->nutmrs);
> + if (tag == hba->nutmrs)
> + return 0;
> +
> + ufshcd_writel(hba, ~(1 << tag), REG_UTP_TASK_REQ_LIST_CLEAR);
> + (&hba->utmrdl_base_addr[tag])->header.dword_2 =
> + cpu_to_be32(ocs_err);
> +
> + /* fatal error injected */
> + return 1;
> +}
> +
> +static bool inject_cmd_hang_tr(struct ufs_hba *hba)
> +{
> + int tag;
> +
> + tag = find_first_bit(&hba->outstanding_reqs, hba->nutrs);
> + if (tag == hba->nutrs)
> + return 0;
> +
> + __clear_bit(tag, &hba->outstanding_reqs);
> + hba->lrb[tag].cmd = NULL;
> + __clear_bit(tag, &hba->lrb_in_use);
> +
> + /* command hang injected */
> + return 1;
> +}
> +
> +static int inject_cmd_hang_tm(struct ufs_hba *hba)
> +{
> + int tag;
> +
> + tag = find_first_bit(&hba->outstanding_tasks, hba->nutmrs);
> + if (tag == hba->nutmrs)
> + return 0;
> +
> + __clear_bit(tag, &hba->outstanding_tasks);
> + __clear_bit(tag, &hba->tm_slots_in_use);
> +
> + /* command hang injected */
> + return 1;
> +}
> +
> +void ufsdbg_fail_request(struct ufs_hba *hba, u32 *intr_status)
> +{
> + u8 ocs_err;
> + static const u32 errors[] = {
> + CONTROLLER_FATAL_ERROR,
> + SYSTEM_BUS_FATAL_ERROR,
> + INJECT_COMMAND_HANG,
> + };
> +
> + if (!should_fail(&hba->debugfs_files.fail_attr, 1))
> + goto out;
> +
> + *intr_status = errors[prandom_u32() % ARRAY_SIZE(errors)];
> + dev_info(hba->dev, "%s: fault-inject error: 0x%x\n",
> + __func__, *intr_status);
> +
> + switch (*intr_status) {
> + case CONTROLLER_FATAL_ERROR: /* fall through */
> + ocs_err = OCS_FATAL_ERROR;
> + goto set_ocs;
> + case SYSTEM_BUS_FATAL_ERROR:
> + ocs_err = OCS_INVALID_CMD_TABLE_ATTR;
> +set_ocs:
> + if (!inject_fatal_err_tr(hba, ocs_err))
> + if (!inject_fatal_err_tm(hba, ocs_err))
> + *intr_status = 0;
> + break;
> + case INJECT_COMMAND_HANG:
> + if (!inject_cmd_hang_tr(hba))
> + inject_cmd_hang_tm(hba);
> + break;
> + default:
> + BUG();
> + /* some configurations ignore panics caused by BUG() */
> + break;
> + }
> +out:
> + return;
> +}
> +
> +static void ufsdbg_setup_fault_injection(struct ufs_hba *hba)
> +{
> + hba->debugfs_files.fail_attr = fail_default_attr;
> +
> + if (fail_request)
> + setup_fault_attr(&hba->debugfs_files.fail_attr,
> fail_request);
> +
> + /* suppress dump stack everytime failure is injected */
> + hba->debugfs_files.fail_attr.verbose = 0;
> +
> + if (IS_ERR(fault_create_debugfs_attr("inject_fault",
> + hba->debugfs_files.debugfs_root,
> + &hba->debugfs_files.fail_attr)))
> + dev_err(hba->dev, "%s: failed to create debugfs entry\n",
> + __func__);
> +}
> +#else
> +void ufsdbg_fail_request(struct ufs_hba *hba, u32 *intr_status)
> +{
> +}
> +
> +static void ufsdbg_setup_fault_injection(struct ufs_hba *hba)
> +{
> +}
> +#endif /* CONFIG_UFS_FAULT_INJECTION */
> +
> #define BUFF_LINE_CAPACITY 16
> #define TAB_CHARS 8
>
> @@ -885,6 +1023,8 @@ void ufsdbg_add_debugfs(struct ufs_hba *hba)
> goto err;
> }
>
> + ufsdbg_setup_fault_injection(hba);
> +
> return;
>
> err:
> diff --git a/drivers/scsi/ufs/ufs-debugfs.h
> b/drivers/scsi/ufs/ufs-debugfs.h
> index 7ed308d..54c68f4 100644
> --- a/drivers/scsi/ufs/ufs-debugfs.h
> +++ b/drivers/scsi/ufs/ufs-debugfs.h
> @@ -26,6 +26,7 @@
> #ifdef CONFIG_DEBUG_FS
> void ufsdbg_add_debugfs(struct ufs_hba *hba);
> void ufsdbg_remove_debugfs(struct ufs_hba *hba);
> +void ufsdbg_fail_request(struct ufs_hba *hba, u32 *intr_status);
> #else
> static inline void ufsdbg_add_debugfs(struct ufs_hba *hba)
> {
> @@ -33,6 +34,9 @@ static inline void ufsdbg_add_debugfs(struct ufs_hba
> *hba)
> static inline void ufsdbg_remove_debugfs(struct ufs_hba *hba)
> {
> }
> +static inline void ufsdbg_fail_request(struct ufs_hba *hba, u32
> *intr_status)
> +{
> +}
> #endif
>
> #endif /* End of Header */
> diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c
> index ae934f2..99c1a81 100644
> --- a/drivers/scsi/ufs/ufshcd.c
> +++ b/drivers/scsi/ufs/ufshcd.c
> @@ -4003,6 +4003,8 @@ static void ufshcd_tmc_handler(struct ufs_hba *hba)
> */
> static void ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
> {
> + ufsdbg_fail_request(hba, &intr_status);
> +
> hba->errors = UFSHCD_ERROR_MASK & intr_status;
> if (hba->errors)
> ufshcd_check_errors(hba);
> diff --git a/drivers/scsi/ufs/ufshcd.h b/drivers/scsi/ufs/ufshcd.h
> index d9eb2ca..b065295 100644
> --- a/drivers/scsi/ufs/ufshcd.h
> +++ b/drivers/scsi/ufs/ufshcd.h
> @@ -64,6 +64,8 @@
> #include <scsi/scsi_dbg.h>
> #include <scsi/scsi_eh.h>
>
> +#include <linux/fault-inject.h>
> +
> #include "ufs.h"
> #include "ufshci.h"
>
> @@ -283,6 +285,9 @@ struct debugfs_files {
> struct dentry *dme_peer_read;
> u32 dme_local_attr_id;
> u32 dme_peer_attr_id;
> +#ifdef CONFIG_UFS_FAULT_INJECTION
> + struct fault_attr fail_attr;
> +#endif
> };
>
> /* tag stats statistics types */
> diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
> index 5f2ce61..3fc79e7 100644
> --- a/lib/Kconfig.debug
> +++ b/lib/Kconfig.debug
> @@ -1432,6 +1432,20 @@ config FAIL_MMC_REQUEST
> and to test how the mmc host driver handles retries from
> the block device.
>
> +config UFS_FAULT_INJECTION
> + bool "Fault-injection capability for UFS IO"
> + select DEBUG_FS
> + depends on FAULT_INJECTION && SCSI_UFSHCD
> + help
> + Provide fault-injection capability for UFS IO.
> + This will make the UFS host controller driver to randomly
> + abort ongoing commands in the host controller, update OCS
> + field according to the injected fatal error and can also
> + forcefully hang the command indefinitely till upper layer
> + timeout occurs. This is useful to test error handling in
> + the UFS contoller driver and test how the driver handles
> + the retries from block/SCSI mid layer.
> +
> config FAULT_INJECTION_DEBUG_FS
> bool "Debugfs entries for fault-injection capabilities"
> depends on FAULT_INJECTION && SYSFS && DEBUG_FS
> --
> Qualcomm Israel, on behalf of Qualcomm Innovation Center, Inc.
> The Qualcomm Innovation Center, Inc. is a member of the Code Aurora Forum,
> a Linux Foundation Collaborative Project
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-scsi" in
> the body of a message to majordomo@xxxxxxxxxxxxxxx
> More majordomo info at http://vger.kernel.org/majordomo-info.html
>
Reviewed-by: Dov Levenglick <dovl@xxxxxxxxxxxxxx>
QUALCOMM ISRAEL, on behalf of Qualcomm Innovation Center, Inc.
The Qualcomm Innovation Center, Inc. is a member of the Code Aurora Forum,
a Linux Foundation Collaborative Project
--
To unsubscribe from this list: send the line "unsubscribe linux-scsi" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
