When trying to expand a ring, if enqueue pointer is behind dequeue pointer
and they're in the same segment, we can not expand the ring until the
dequeue pointer move pass the link TRB. This is because if the hardware
has not cached the link TRB, it will jump to the new segments, but the
data there is not right and the hardware may stop if the cycle_bit
indicates it's software owned.
Wait for the dequeue pointer move pass the link TRB, and then it's safe
to expand the ring. Pending all the urbs submitted during the waiting period
and queue them after the ring successfully expanded. When an endpoint is
stopped, clear all the pending urbs.
Signed-off-by: Andiry Xu <andiry.xu@xxxxxxx>
---
drivers/usb/host/xhci-mem.c | 2 +
drivers/usb/host/xhci-ring.c | 182 +++++++++++++++++++++++++++++++++++++++--
drivers/usb/host/xhci.c | 57 +++++++++----
drivers/usb/host/xhci.h | 8 ++
4 files changed, 223 insertions(+), 26 deletions(-)
diff --git a/drivers/usb/host/xhci-mem.c b/drivers/usb/host/xhci-mem.c
index 7f90bb4..27788d5 100644
--- a/drivers/usb/host/xhci-mem.c
+++ b/drivers/usb/host/xhci-mem.c
@@ -183,6 +183,7 @@ static void xhci_initialize_ring_info(struct xhci_ring *ring,
* handling ring expansion, set the cycle state equal to the old ring.
*/
ring->cycle_state = cycle_state;
+ ring->pending_num_trbs = 0;
/* Not necessary for new rings, but needed for re-initialized rings */
ring->enq_updates = 0;
ring->deq_updates = 0;
@@ -248,6 +249,7 @@ static struct xhci_ring *xhci_ring_alloc(struct xhci_hcd *xhci,
ring->num_segs = num_segs;
INIT_LIST_HEAD(&ring->td_list);
+ INIT_LIST_HEAD(&ring->pending_urb_list);
ring->type = type;
if (num_segs == 0)
return ring;
diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c
index 3e4195d..494024e 100644
--- a/drivers/usb/host/xhci-ring.c
+++ b/drivers/usb/host/xhci-ring.c
@@ -140,6 +140,89 @@ static void next_trb(struct xhci_hcd *xhci,
}
/*
+ * Expand the ring, and queue the URBs. If failed, giveback the URB.
+ */
+static int xhci_queue_pending_urbs(struct xhci_hcd *xhci,
+ struct xhci_ring *ep_ring)
+{
+ struct xhci_pending_urb *curr, *next;
+ struct urb *urb;
+ unsigned int slot_id, ep_index;
+ int ret = 0;
+ bool expansion_failed = false;
+
+ if (ep_ring->type == TYPE_COMMAND || ep_ring->type == TYPE_EVENT)
+ return -EINVAL;
+
+ /* It's safe to expand the ring now */
+ ret = xhci_ring_expansion(xhci, ep_ring, ep_ring->pending_num_trbs,
+ GFP_ATOMIC);
+ if (ret) {
+ xhci_err(xhci, "Ring expansion failed\n");
+ goto clear;
+ }
+
+ /* Clear the pending urbs and queue them */
+ ep_ring->pending_num_trbs = 0;
+
+ list_for_each_entry_safe(curr, next, &ep_ring->pending_urb_list, list) {
+ urb = curr->urb;
+ if (!expansion_failed) {
+ slot_id = urb->dev->slot_id;
+ ep_index = xhci_get_endpoint_index(&urb->ep->desc);
+ switch (ep_ring->type) {
+ case TYPE_CTRL:
+ ret = xhci_queue_ctrl_tx(xhci, GFP_ATOMIC, urb,
+ slot_id, ep_index);
+ break;
+ case TYPE_ISOC:
+ ret = xhci_queue_isoc_tx_prepare(xhci,
+ GFP_ATOMIC, urb, slot_id, ep_index);
+ break;
+ case TYPE_BULK:
+ case TYPE_STREAM:
+ ret = xhci_queue_bulk_tx(xhci, GFP_ATOMIC, urb,
+ slot_id, ep_index);
+ break;
+ case TYPE_INTR:
+ ret = xhci_queue_intr_tx(xhci, GFP_ATOMIC, urb,
+ slot_id, ep_index);
+ break;
+ default:
+ ret = -EINVAL;
+ break;
+ }
+ }
+
+ if (!ret)
+ goto clear;
+
+ list_del(&curr->list);
+ kfree(curr);
+ }
+
+ return ret;
+
+clear:
+ /*
+ * Ring expansion or urb submission failed. Giveback all the urbs on
+ * the pending list...
+ */
+ list_for_each_entry_safe(curr, next, &ep_ring->pending_urb_list, list) {
+ urb = curr->urb;
+ xhci_urb_free_priv(xhci, urb->hcpriv);
+ spin_unlock(&xhci->lock);
+ usb_hcd_giveback_urb(bus_to_hcd(urb->dev->bus), urb, -EPIPE);
+ spin_lock(&xhci->lock);
+ list_del(&curr->list);
+ kfree(curr);
+ }
+ ep_ring->pending_num_trbs = 0;
+
+ return ret;
+}
+
+/*
* See Cycle bit rules. SW is the consumer for the event ring only.
* Don't make a ring full of link TRBs. That would be dumb and this would loop.
*/
@@ -171,6 +254,12 @@ static void inc_deq(struct xhci_hcd *xhci, struct xhci_ring *ring)
ring->deq_seg = ring->deq_seg->next;
ring->dequeue = ring->deq_seg->trbs;
next = ring->dequeue;
+ /*
+ * If there are pending URBs, first expand the ring, and
+ * queue the URBs.
+ */
+ if (ring->pending_num_trbs > 0)
+ xhci_queue_pending_urbs(xhci, ring);
}
addr = (unsigned long long) xhci_trb_virt_to_dma(ring->deq_seg, ring->dequeue);
}
@@ -618,6 +707,46 @@ static void xhci_giveback_urb_in_irq(struct xhci_hcd *xhci,
}
}
+static void xhci_giveback_pending_urbs(struct xhci_hcd *xhci,
+ struct xhci_ring *ep_ring)
+{
+ struct xhci_pending_urb *curr, *next;
+
+ list_for_each_entry_safe(curr, next,
+ &ep_ring->pending_urb_list, list) {
+ xhci_urb_free_priv(xhci, curr->urb->hcpriv);
+ spin_unlock(&xhci->lock);
+ usb_hcd_giveback_urb(bus_to_hcd(curr->urb->dev->bus), curr->urb,
+ -ESHUTDOWN);
+ spin_lock(&xhci->lock);
+ list_del(&curr->list);
+ kfree(curr);
+ }
+ ep_ring->pending_num_trbs = 0;
+}
+
+/* Scan all the rings of the endpoint, giveback pending URBs */
+static int xhci_clear_ep_pending_urbs(struct xhci_hcd *xhci,
+ struct xhci_virt_ep *ep)
+{
+ struct xhci_ring *ep_ring;
+ int i;
+
+ if (!(ep->ep_state & EP_HAS_STREAMS)) {
+ ep_ring = ep->ring;
+ if (ep_ring && ep_ring->pending_num_trbs > 0)
+ xhci_giveback_pending_urbs(xhci, ep_ring);
+ } else {
+ for (i = 0; i < ep->stream_info->num_streams; ++i) {
+ ep_ring = ep->stream_info->stream_rings[i];
+ if (ep_ring && ep_ring->pending_num_trbs > 0)
+ xhci_giveback_pending_urbs(xhci, ep_ring);
+ }
+ }
+
+ return 0;
+}
+
/*
* When we get a command completion for a Stop Endpoint Command, we need to
* unlink any cancelled TDs from the ring. There are two ways to do that:
@@ -843,6 +972,7 @@ void xhci_stop_endpoint_command_watchdog(unsigned long arg)
continue;
for (j = 0; j < 31; j++) {
temp_ep = &xhci->devs[i]->eps[j];
+ xhci_clear_ep_pending_urbs(xhci, temp_ep);
ring = temp_ep->ring;
if (!ring)
continue;
@@ -2401,10 +2531,10 @@ static void queue_trb(struct xhci_hcd *xhci, struct xhci_ring *ring,
/*
* Does various checks on the endpoint ring, and makes it ready to queue num_trbs.
- * FIXME allocate segments if the ring is full.
*/
static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
- u32 ep_state, unsigned int num_trbs, gfp_t mem_flags)
+ struct urb *urb, u32 ep_state, unsigned int num_trbs,
+ gfp_t mem_flags)
{
unsigned int num_trbs_needed;
@@ -2445,11 +2575,28 @@ static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
return -ENOMEM;
}
+ /*
+ * Waiting for dequeue pointer to walk pass the link TRB;
+ * until then, add the URBs to the ring's URB list,
+ * queue them later.
+ */
if (ep_ring->enq_seg == ep_ring->deq_seg &&
ep_ring->dequeue > ep_ring->enqueue) {
- xhci_err(xhci, "Can not expand the ring while dequeue "
- "pointer has not passed the link TRB\n");
- return -ENOMEM;
+ struct xhci_pending_urb *pending_urb;
+
+ xhci_dbg(xhci, "Waiting for dequeue "
+ "pointer to pass the link TRB\n");
+ pending_urb = kzalloc(sizeof(struct xhci_pending_urb),
+ mem_flags);
+ if (!pending_urb)
+ return -ENOMEM;
+ ep_ring->pending_num_trbs += num_trbs;
+ pending_urb->urb = urb;
+ pending_urb->num_trbs = num_trbs;
+ INIT_LIST_HEAD(&pending_urb->list);
+ list_add_tail(&pending_urb->list,
+ &ep_ring->pending_urb_list);
+ return -EBUSY;
}
xhci_dbg(xhci, "ERROR no room on ep ring, "
@@ -2522,7 +2669,23 @@ static int prepare_transfer(struct xhci_hcd *xhci,
return -EINVAL;
}
- ret = prepare_ring(xhci, ep_ring,
+ if (ep_ring->pending_num_trbs > 0) {
+ struct xhci_pending_urb *pending_urb;
+
+ pending_urb = kzalloc(sizeof(struct xhci_pending_urb),
+ mem_flags);
+ if (!pending_urb)
+ return -ENOMEM;
+ xhci_dbg(xhci, "Adding urb %p to ring's pending list\n", urb);
+ ep_ring->pending_num_trbs += num_trbs;
+ pending_urb->urb = urb;
+ pending_urb->num_trbs = num_trbs;
+ INIT_LIST_HEAD(&pending_urb->list);
+ list_add_tail(&pending_urb->list, &ep_ring->pending_urb_list);
+ return -EBUSY;
+ }
+
+ ret = prepare_ring(xhci, ep_ring, urb,
le32_to_cpu(ep_ctx->ep_info) & EP_STATE_MASK,
num_trbs, mem_flags);
if (ret)
@@ -3435,8 +3598,9 @@ int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags,
/* Check the ring to guarantee there is enough room for the whole urb.
* Do not insert any td of the urb to the ring if the check failed.
*/
- ret = prepare_ring(xhci, ep_ring, le32_to_cpu(ep_ctx->ep_info) & EP_STATE_MASK,
- num_trbs, mem_flags);
+ ret = prepare_ring(xhci, ep_ring, urb,
+ le32_to_cpu(ep_ctx->ep_info) & EP_STATE_MASK,
+ num_trbs, mem_flags);
if (ret)
return ret;
@@ -3496,7 +3660,7 @@ static int queue_command(struct xhci_hcd *xhci, u32 field1, u32 field2,
if (!command_must_succeed)
reserved_trbs++;
- ret = prepare_ring(xhci, xhci->cmd_ring, EP_STATE_RUNNING,
+ ret = prepare_ring(xhci, xhci->cmd_ring, NULL, EP_STATE_RUNNING,
reserved_trbs, GFP_ATOMIC);
if (ret < 0) {
xhci_err(xhci, "ERR: No room for command on command ring\n");
diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c
index ec35256..3112de1 100644
--- a/drivers/usb/host/xhci.c
+++ b/drivers/usb/host/xhci.c
@@ -1155,9 +1155,6 @@ int xhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flags)
goto dying;
ret = xhci_queue_ctrl_tx(xhci, GFP_ATOMIC, urb,
slot_id, ep_index);
- if (ret)
- goto free_priv;
- spin_unlock_irqrestore(&xhci->lock, flags);
} else if (usb_endpoint_xfer_bulk(&urb->ep->desc)) {
spin_lock_irqsave(&xhci->lock, flags);
if (xhci->xhc_state & XHCI_STATE_DYING)
@@ -1177,28 +1174,30 @@ int xhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flags)
ret = xhci_queue_bulk_tx(xhci, GFP_ATOMIC, urb,
slot_id, ep_index);
}
- if (ret)
- goto free_priv;
- spin_unlock_irqrestore(&xhci->lock, flags);
} else if (usb_endpoint_xfer_int(&urb->ep->desc)) {
spin_lock_irqsave(&xhci->lock, flags);
if (xhci->xhc_state & XHCI_STATE_DYING)
goto dying;
ret = xhci_queue_intr_tx(xhci, GFP_ATOMIC, urb,
slot_id, ep_index);
- if (ret)
- goto free_priv;
- spin_unlock_irqrestore(&xhci->lock, flags);
} else {
spin_lock_irqsave(&xhci->lock, flags);
if (xhci->xhc_state & XHCI_STATE_DYING)
goto dying;
ret = xhci_queue_isoc_tx_prepare(xhci, GFP_ATOMIC, urb,
slot_id, ep_index);
- if (ret)
- goto free_priv;
- spin_unlock_irqrestore(&xhci->lock, flags);
}
+
+ if (ret == -EBUSY)
+ /*
+ * We're waiting for dequeue pointer walk pass link TRB and
+ * will queue this URB later. Tell usbcore it's queued
+ * successfully...
+ */
+ ret = 0;
+ if (ret)
+ goto free_priv;
+ spin_unlock_irqrestore(&xhci->lock, flags);
exit:
return ret;
dying:
@@ -1296,13 +1295,39 @@ int xhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
unsigned int ep_index;
struct xhci_ring *ep_ring;
struct xhci_virt_ep *ep;
+ struct xhci_pending_urb *curr, *next;
xhci = hcd_to_xhci(hcd);
spin_lock_irqsave(&xhci->lock, flags);
+
+ ep_index = xhci_get_endpoint_index(&urb->ep->desc);
+ ep = &xhci->devs[urb->dev->slot_id]->eps[ep_index];
+ ep_ring = xhci_urb_to_transfer_ring(xhci, urb);
+
/* Make sure the URB hasn't completed or been unlinked already */
ret = usb_hcd_check_unlink_urb(hcd, urb, status);
- if (ret || !urb->hcpriv)
+ if (ret) {
+ if (!ep_ring || ep_ring->pending_num_trbs == 0 || !urb->hcpriv)
+ goto done;
+ /* Check the pending urb list */
+ list_for_each_entry_safe(curr, next,
+ &ep_ring->pending_urb_list, list) {
+ if (curr->urb == urb) {
+ xhci_urb_free_priv(xhci, curr->urb->hcpriv);
+ list_del(&curr->list);
+ spin_unlock_irqrestore(&xhci->lock, flags);
+ usb_hcd_giveback_urb(
+ bus_to_hcd(curr->urb->dev->bus),
+ curr->urb, -ESHUTDOWN);
+ list_del(&curr->list);
+ ep_ring->pending_num_trbs -= curr->num_trbs;
+ kfree(curr);
+ goto stop_ep;
+ }
+ }
goto done;
+ }
+
temp = xhci_readl(xhci, &xhci->op_regs->status);
if (temp == 0xffffffff || (xhci->xhc_state & XHCI_STATE_HALTED)) {
xhci_dbg(xhci, "HW died, freeing TD.\n");
@@ -1321,6 +1346,7 @@ int xhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
xhci_urb_free_priv(xhci, urb_priv);
return ret;
}
+
if ((xhci->xhc_state & XHCI_STATE_DYING) ||
(xhci->xhc_state & XHCI_STATE_HALTED)) {
xhci_dbg(xhci, "Ep 0x%x: URB %p to be canceled on "
@@ -1337,14 +1363,10 @@ int xhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
xhci_dbg(xhci, "Cancel URB %p\n", urb);
xhci_dbg(xhci, "Event ring:\n");
xhci_debug_ring(xhci, xhci->event_ring);
- ep_index = xhci_get_endpoint_index(&urb->ep->desc);
- ep = &xhci->devs[urb->dev->slot_id]->eps[ep_index];
- ep_ring = xhci_urb_to_transfer_ring(xhci, urb);
if (!ep_ring) {
ret = -EINVAL;
goto done;
}
-
xhci_dbg(xhci, "Endpoint ring:\n");
xhci_debug_ring(xhci, ep_ring);
@@ -1355,6 +1377,7 @@ int xhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
list_add_tail(&td->cancelled_td_list, &ep->cancelled_td_list);
}
+stop_ep:
/* Queue a stop endpoint command, but only if this is
* the first cancellation to be handled.
*/
diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h
index 063405a..6271cbe 100644
--- a/drivers/usb/host/xhci.h
+++ b/drivers/usb/host/xhci.h
@@ -1254,6 +1254,12 @@ struct xhci_dequeue_state {
int new_cycle_state;
};
+struct xhci_pending_urb {
+ struct urb *urb;
+ struct list_head list;
+ unsigned int num_trbs;
+};
+
enum xhci_ring_type {
TYPE_CTRL = 0,
TYPE_ISOC,
@@ -1274,6 +1280,7 @@ struct xhci_ring {
struct xhci_segment *deq_seg;
unsigned int deq_updates;
struct list_head td_list;
+ struct list_head pending_urb_list;
/*
* Write the cycle state into the TRB cycle field to give ownership of
* the TRB to the host controller (if we are the producer), or to check
@@ -1284,6 +1291,7 @@ struct xhci_ring {
unsigned int num_segs;
unsigned int num_trbs_free;
unsigned int num_trbs_free_temp;
+ unsigned int pending_num_trbs;
enum xhci_ring_type type;
bool last_td_was_short;
};
--
1.7.4.1
--
To unsubscribe from this list: send the line "unsubscribe linux-usb" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
