Linux-scsi, et al.
The follow patch address two major issues found under extensive testing.
While pounding data io down the card and performing large scale queries to
the controller about device state and function parameters, the following
were discovered.
Random (hard to reproduce, without a noise injection into the SATA
connector or cable) hardware error states which locks the card and in the
majority of the cases caused the array to be lost. If the array was not
lost then a drive was failed but one could not remove/replace w/ a new
drive. Thus adding in a pci_master_abort test and clear function proved
to allow recovery in all cases where the card shutdown communication to
the host. This may not address all cases; however, clearly this is a
missing part of the driver base when entry to eh_scsi_* begins.
The compond issue in the failed recovery resulted in a deref NULL pointer
in the various list_head calls. After change the individual list_add to
list_move and such, the NULL point issue has never shown up in the past 6
weeks of heavy testing.
In all cases in the past, the baseline for error was 6:1. Meaning either
one system in six failed and/or one in six test/stress runs failed. With
the attached changes, there have been zero failures in the past three
weeks. This sound great, but I wish it would fail to allow some
statistics of improved error handling.
Please note the changes to SAS are minor and not tested, but seem correct
for the entire directory code base. SAS shares the CMM core with MBOX,
thus the rational for changes to SAS.
Please comment and provide suggestions.
Cheers,
Andre Hedrick
LAD Storage Consulting Group
diff -ur linux-2.6.17-rc4/drivers/scsi/megaraid/megaraid_mbox.c linux-2.6.17-rc4.new/drivers/scsi/megaraid/megaraid_mbox.c
--- linux-2.6.17-rc4/drivers/scsi/megaraid/megaraid_mbox.c 2006-05-11 16:31:53.000000000 -0700
+++ linux-2.6.17-rc4.new/drivers/scsi/megaraid/megaraid_mbox.c 2006-05-16 10:15:11.282454536 -0700
@@ -94,6 +94,8 @@
static int megaraid_sysfs_alloc_resources(adapter_t *);
static void megaraid_sysfs_free_resources(adapter_t *);
+static int megaraid_pci_master_abort(struct pci_dev *);
+
static int megaraid_abort_handler(struct scsi_cmnd *);
static int megaraid_reset_handler(struct scsi_cmnd *);
@@ -1276,7 +1278,7 @@
// detach scb from free pool
spin_lock_irqsave(SCSI_FREE_LIST_LOCK(adapter), flags);
- if (list_empty(head)) {
+ if (list_empty_careful(head)) {
spin_unlock_irqrestore(SCSI_FREE_LIST_LOCK(adapter), flags);
return NULL;
}
@@ -1314,7 +1316,7 @@
scb->scp = NULL;
spin_lock_irqsave(SCSI_FREE_LIST_LOCK(adapter), flags);
- list_add(&scb->list, &adapter->kscb_pool);
+ list_move(&scb->list, &adapter->kscb_pool);
spin_unlock_irqrestore(SCSI_FREE_LIST_LOCK(adapter), flags);
@@ -1911,7 +1913,7 @@
if (scb_q) {
scb_q->state = SCB_PENDQ;
- list_add_tail(&scb_q->list, &adapter->pend_list);
+ list_move_tail(&scb_q->list, &adapter->pend_list);
}
// if the adapter in not in quiescent mode, post the commands to FW
@@ -1920,7 +1922,7 @@
return;
}
- while (!list_empty(&adapter->pend_list)) {
+ while (!list_empty_careful(&adapter->pend_list)) {
assert_spin_locked(PENDING_LIST_LOCK(adapter));
@@ -1946,7 +1948,7 @@
scb->state = SCB_PENDQ;
- list_add(&scb->list, &adapter->pend_list);
+ list_move(&scb->list, &adapter->pend_list);
spin_unlock_irqrestore(PENDING_LIST_LOCK(adapter),
flags);
@@ -2148,7 +2150,7 @@
}
scb->status = mbox->status;
- list_add_tail(&scb->list, &clist);
+ list_move_tail(&scb->list, &clist);
}
// Acknowledge interrupt
@@ -2477,6 +2479,27 @@
/**
+ * megaraid_pci_master_abort
+ * @dev : pci device structure
+ *
+ * Tests for PCI Master Abort on the host adapter and clears state
+ * Returns state of error with inverted logic test to give proper
+ * state of the pci statuts bit describing master_abort.
+ */
+static int megaraid_pci_master_abort(struct pci_dev* dev)
+{
+ u16 status, error_bits;
+
+ pci_read_config_word(dev, PCI_STATUS, &status);
+ if (error_bits)
+ pci_write_config_word(dev, PCI_STATUS, error_bits);
+ pci_read_config_word(dev, PCI_STATUS, &status);
+ error_bits = (status & PCI_STATUS_REC_MASTER_ABORT);
+ return (!error_bits) ? 0 : 1;
+}
+
+
+/**
* megaraid_abort_handler - abort the scsi command
* @scp : command to be aborted
*
@@ -2505,9 +2528,13 @@
// If FW has stopped responding, simply return failure
if (raid_dev->hw_error) {
- con_log(CL_ANN, (KERN_NOTICE
- "megaraid: hw error, not aborting\n"));
- return FAILED;
+ // test if adapter is locked up because of pci master abort
+ if (megaraid_pci_master_abort(adapter->pdev)) {
+ con_log(CL_ANN, (KERN_NOTICE
+ "megaraid: hw error, cannot reset\n"));
+ return FAILED;
+ }
+ raid_dev->hw_error = 0;
}
// There might a race here, where the command was completed by the
@@ -2519,14 +2546,13 @@
list_for_each_entry_safe(scb, tmp, &adapter->completed_list, list) {
if (scb->scp == scp) { // Found command
-
- list_del_init(&scb->list); // from completed list
-
con_log(CL_ANN, (KERN_WARNING
"megaraid: %ld:%d[%d:%d], abort from completed list\n",
scp->serial_number, scb->sno,
scb->dev_channel, scb->dev_target));
+ list_del_init(&scb->list); // from completed list
+
scp->result = (DID_ABORT << 16);
scp->scsi_done(scp);
@@ -2549,8 +2575,6 @@
if (scb->scp == scp) { // Found command
- list_del_init(&scb->list); // from pending list
-
ASSERT(!(scb->state & SCB_ISSUED));
con_log(CL_ANN, (KERN_WARNING
@@ -2558,6 +2582,8 @@
scp->serial_number, scb->dev_channel,
scb->dev_target));
+ list_del_init(&scb->list); // from pending list
+
scp->result = (DID_ABORT << 16);
scp->scsi_done(scp);
@@ -3606,7 +3632,7 @@
// detach one scb from free pool
spin_lock_irqsave(USER_FREE_LIST_LOCK(adapter), flags);
- if (list_empty(head)) { // should never happen because of CMM
+ if (list_empty_careful(head)) { // should never happen because of CMM
con_log(CL_ANN, (KERN_WARNING
"megaraid mbox: bug in cmm handler, lost resources\n"));
@@ -3737,7 +3763,7 @@
spin_lock_irqsave(USER_FREE_LIST_LOCK(adapter), flags);
- list_add(&scb->list, &adapter->uscb_pool);
+ list_move(&scb->list, &adapter->uscb_pool);
spin_unlock_irqrestore(USER_FREE_LIST_LOCK(adapter), flags);
diff -ur linux-2.6.17-rc4/drivers/scsi/megaraid/megaraid_mm.c linux-2.6.17-rc4.new/drivers/scsi/megaraid/megaraid_mm.c
--- linux-2.6.17-rc4/drivers/scsi/megaraid/megaraid_mm.c 2006-05-11 16:31:53.000000000 -0700
+++ linux-2.6.17-rc4.new/drivers/scsi/megaraid/megaraid_mm.c 2006-05-16 10:16:19.461954269 -0700
@@ -577,7 +577,7 @@
head = &adp->kioc_pool;
- if (list_empty(head)) {
+ if (list_empty_careful(head)) {
up(&adp->kioc_semaphore);
spin_unlock_irqrestore(&adp->kioc_pool_lock, flags);
@@ -641,7 +641,7 @@
/* Return the kioc to the free pool */
spin_lock_irqsave(&adp->kioc_pool_lock, flags);
- list_add(&kioc->list, &adp->kioc_pool);
+ list_move(&kioc->list, &adp->kioc_pool);
spin_unlock_irqrestore(&adp->kioc_pool_lock, flags);
/* increment the free kioc count */
diff -ur linux-2.6.17-rc4/drivers/scsi/megaraid/megaraid_sas.c linux-2.6.17-rc4.new/drivers/scsi/megaraid/megaraid_sas.c
--- linux-2.6.17-rc4/drivers/scsi/megaraid/megaraid_sas.c 2006-05-11 16:31:53.000000000 -0700
+++ linux-2.6.17-rc4.new/drivers/scsi/megaraid/megaraid_sas.c 2006-05-16 10:25:54.562822089 -0700
@@ -95,7 +95,7 @@
spin_lock_irqsave(&instance->cmd_pool_lock, flags);
- if (!list_empty(&instance->cmd_pool)) {
+ if (!list_empty_careful(&instance->cmd_pool)) {
cmd = list_entry((&instance->cmd_pool)->next,
struct megasas_cmd, list);
list_del_init(&cmd->list);
@@ -120,7 +120,7 @@
spin_lock_irqsave(&instance->cmd_pool_lock, flags);
cmd->scmd = NULL;
- list_add_tail(&cmd->list, &instance->cmd_pool);
+ list_move_tail(&cmd->list, &instance->cmd_pool);
spin_unlock_irqrestore(&instance->cmd_pool_lock, flags);
}
