Implement the XHCI operations for allocating and requesting for a secondary
interrupter. The secondary interrupter can allow for events for a
particular endpoint to be routed to a separate event ring. The event
routing is defined when submitting a transfer descriptor to the USB HW.
There is a specific field which denotes which interrupter ring to route the
event to when the transfer is completed.
An example use case, such as audio packet offloading can utilize a separate
event ring, so that these events can be routed to a different processor
within the system. The processor would be able to independently submit
transfers and handle its completions without intervention from the main
processor.
Signed-off-by: Wesley Cheng <quic_wcheng@xxxxxxxxxxx>
---
drivers/usb/host/xhci-mem.c | 219 ++++++++++++++++++++++++++++-------
drivers/usb/host/xhci-plat.c | 2 +
drivers/usb/host/xhci.c | 169 ++++++++++++++++++++++++++-
drivers/usb/host/xhci.h | 15 +++
4 files changed, 363 insertions(+), 42 deletions(-)
diff --git a/drivers/usb/host/xhci-mem.c b/drivers/usb/host/xhci-mem.c
index 81ca2bc1f0be..d5cb4b82ad3d 100644
--- a/drivers/usb/host/xhci-mem.c
+++ b/drivers/usb/host/xhci-mem.c
@@ -1835,6 +1835,7 @@ void xhci_free_erst(struct xhci_hcd *xhci, struct xhci_erst *erst)
void xhci_mem_cleanup(struct xhci_hcd *xhci)
{
struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
+ struct xhci_sec *sec, *tmp;
int i, j, num_ports;
cancel_delayed_work_sync(&xhci->cmd_timer);
@@ -1846,6 +1847,16 @@ void xhci_mem_cleanup(struct xhci_hcd *xhci)
xhci->event_ring = NULL;
xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed event ring");
+ list_for_each_entry_safe(sec, tmp, &xhci->xhci_sec_list, list) {
+ list_del(&sec->list);
+ if (sec->event_ring) {
+ xhci_ring_free(xhci, sec->event_ring);
+ xhci_dbg_trace(xhci, trace_xhci_dbg_init,
+ "Freed secondary ring %d", sec->intr_num);
+ }
+ kfree(sec);
+ }
+
if (xhci->cmd_ring)
xhci_ring_free(xhci, xhci->cmd_ring);
xhci->cmd_ring = NULL;
@@ -2087,18 +2098,18 @@ static int xhci_check_trb_in_td_math(struct xhci_hcd *xhci)
return 0;
}
-static void xhci_set_hc_event_deq(struct xhci_hcd *xhci)
+static void xhci_set_hc_event_deq(struct xhci_hcd *xhci, struct xhci_ring *er,
+ struct xhci_intr_reg __iomem *ir_set)
{
u64 temp;
dma_addr_t deq;
- deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg,
- xhci->event_ring->dequeue);
+ deq = xhci_trb_virt_to_dma(er->deq_seg, er->dequeue);
if (!deq)
xhci_warn(xhci, "WARN something wrong with SW event ring "
"dequeue ptr.\n");
/* Update HC event ring dequeue pointer */
- temp = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
+ temp = xhci_read_64(xhci, &ir_set->erst_dequeue);
temp &= ERST_PTR_MASK;
/* Don't clear the EHB bit (which is RW1C) because
* there might be more events to service.
@@ -2108,7 +2119,7 @@ static void xhci_set_hc_event_deq(struct xhci_hcd *xhci)
"// Write event ring dequeue pointer, "
"preserving EHB bit");
xhci_write_64(xhci, ((u64) deq & (u64) ~ERST_PTR_MASK) | temp,
- &xhci->ir_set->erst_dequeue);
+ &ir_set->erst_dequeue);
}
static void xhci_add_in_port(struct xhci_hcd *xhci, unsigned int num_ports,
@@ -2375,10 +2386,159 @@ static int xhci_setup_port_arrays(struct xhci_hcd *xhci, gfp_t flags)
return 0;
}
+static void xhci_reset_evt_ring_base(struct xhci_hcd *xhci, struct xhci_ring *er,
+ struct xhci_erst *erst, struct xhci_intr_reg __iomem *ir_set)
+{
+ u64 val_64;
+
+ xhci_dbg_trace(xhci, trace_xhci_dbg_init,
+ "// Set ERST entries to point to event ring.");
+ /* set the segment table base address */
+ xhci_dbg_trace(xhci, trace_xhci_dbg_init,
+ "// Set ERST base address for ir_set 0 = 0x%llx",
+ (unsigned long long)erst->erst_dma_addr);
+ val_64 = xhci_read_64(xhci, &ir_set->erst_base);
+ val_64 &= ERST_PTR_MASK;
+ pr_err("after clearing ptr = 0x%llx\n", val_64);
+ val_64 |= (erst->erst_dma_addr & (u64) ~ERST_PTR_MASK);
+ pr_err("after clearing ptr = 0x%llx\n", val_64);
+
+ xhci_write_64(xhci, val_64, &ir_set->erst_base);
+ xhci_dbg_trace(xhci, trace_xhci_dbg_init,
+ "done.");
+ /* Set the event ring dequeue address */
+ xhci_set_hc_event_deq(xhci, er, ir_set);
+ xhci_dbg_trace(xhci, trace_xhci_dbg_init,
+ "Wrote ERST address to ir_set 0.");
+}
+
+void xhci_handle_sec_intr_events(struct xhci_hcd *xhci, struct xhci_ring *ring,
+ struct xhci_intr_reg __iomem *ir_set, struct xhci_sec *sec)
+{
+ union xhci_trb *erdp_trb, *current_trb;
+ struct xhci_segment *seg;
+ u64 erdp_reg;
+ u32 iman_reg;
+ dma_addr_t deq;
+ unsigned long segment_offset;
+ struct xhci_erst *erst = &sec->erst;
+
+ /* disable irq, ack pending interrupt and ack all pending events */
+
+ iman_reg = readl_relaxed(&ir_set->irq_pending);
+ iman_reg &= ~IMAN_IE;
+ writel_relaxed(iman_reg, &ir_set->irq_pending);
+ iman_reg = readl_relaxed(&ir_set->irq_pending);
+ if (iman_reg & IMAN_IP)
+ writel_relaxed(iman_reg, &ir_set->irq_pending);
+
+ /* last acked event trb is in erdp reg */
+ erdp_reg = xhci_read_64(xhci, &ir_set->erst_dequeue);
+ deq = (dma_addr_t)(erdp_reg & ~ERST_PTR_MASK);
+ if (!deq) {
+ pr_debug("%s: event ring handling not required\n", __func__);
+ return;
+ }
+
+ seg = ring->first_seg;
+ segment_offset = deq - seg->dma;
+
+ /* find out virtual address of the last acked event trb */
+ erdp_trb = current_trb = &seg->trbs[0] +
+ (segment_offset/sizeof(*current_trb));
+
+ /* read cycle state of the last acked trb to find out CCS */
+ ring->cycle_state = le32_to_cpu(current_trb->event_cmd.flags) & TRB_CYCLE;
+
+ while (1) {
+ /* last trb of the event ring: toggle cycle state */
+ if (current_trb == &seg->trbs[TRBS_PER_SEGMENT - 1]) {
+ ring->cycle_state ^= 1;
+ current_trb = &seg->trbs[0];
+ } else {
+ current_trb++;
+ }
+
+ /* cycle state transition */
+ if ((le32_to_cpu(current_trb->event_cmd.flags) & TRB_CYCLE) !=
+ ring->cycle_state)
+ break;
+ }
+
+ if (erdp_trb != current_trb) {
+ deq = xhci_trb_virt_to_dma(ring->deq_seg, current_trb);
+ if (deq == 0)
+ xhci_warn(xhci,
+ "WARN invalid SW event ring dequeue ptr.\n");
+ /* Update HC event ring dequeue pointer */
+ erdp_reg &= ERST_PTR_MASK;
+ erdp_reg |= ((u64) deq & (u64) ~ERST_PTR_MASK);
+ }
+
+ /* Clear the event handler busy flag (RW1C); event ring is empty. */
+ erdp_reg |= ERST_EHB;
+ xhci_write_64(xhci, erdp_reg, &ir_set->erst_dequeue);
+
+ xhci_reset_evt_ring_base(xhci, ring, erst, ir_set);
+}
+
+static int xhci_event_ring_setup(struct xhci_hcd *xhci, struct xhci_ring **er,
+ struct xhci_erst *erst, struct xhci_intr_reg __iomem *ir_set,
+ unsigned int intr_num, gfp_t flags)
+{
+ int ret;
+ unsigned int val;
+
+ /*
+ * Event ring setup: Allocate a normal ring, but also setup
+ * the event ring segment table (ERST). Section 4.9.3.
+ */
+ xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Allocating event ring");
+ *er = xhci_ring_alloc(xhci, ERST_NUM_SEGS, 1, TYPE_EVENT,
+ 0, flags);
+ if (!*er)
+ return -ENOMEM;
+
+ ret = xhci_alloc_erst(xhci, *er, erst, flags);
+ if (ret)
+ return ret;
+
+ /* set ERST count with the number of entries in the segment table */
+ val = readl(&ir_set->erst_size);
+ val &= ERST_SIZE_MASK;
+ val |= ERST_NUM_SEGS;
+ xhci_dbg_trace(xhci, trace_xhci_dbg_init,
+ "// Write ERST size = %i to ir_set 0 (some bits preserved)",
+ val);
+ writel(val, &ir_set->erst_size);
+
+ xhci_reset_evt_ring_base(xhci, *er, erst, ir_set);
+
+ return 0;
+}
+
+int xhci_mem_sec_init(struct xhci_hcd *xhci, struct xhci_sec *sec)
+{
+ int ret;
+
+ sec->ir_set = &xhci->run_regs->ir_set[sec->intr_num];
+ ret = xhci_event_ring_setup(xhci, &sec->event_ring,
+ &sec->erst, sec->ir_set, sec->intr_num, GFP_KERNEL);
+ if (ret) {
+ xhci_err(xhci, "sec event ring setup failed inter#%d\n",
+ sec->intr_num);
+ return ret;
+ }
+ sec->xhci = 0;
+
+ return 0;
+}
+
int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags)
{
dma_addr_t dma;
struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
+ struct xhci_sec *sec;
unsigned int val, val2;
u64 val_64;
u32 page_size, temp;
@@ -2497,46 +2657,23 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags)
/* Set ir_set to interrupt register set 0 */
xhci->ir_set = &xhci->run_regs->ir_set[0];
- /*
- * Event ring setup: Allocate a normal ring, but also setup
- * the event ring segment table (ERST). Section 4.9.3.
- */
- xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Allocating event ring");
- xhci->event_ring = xhci_ring_alloc(xhci, ERST_NUM_SEGS, 1, TYPE_EVENT,
- 0, flags);
- if (!xhci->event_ring)
- goto fail;
- if (xhci_check_trb_in_td_math(xhci) < 0)
+ ret = xhci_event_ring_setup(xhci, &xhci->event_ring, &xhci->erst, xhci->ir_set, 0, flags);
+ if (ret < 0)
goto fail;
- ret = xhci_alloc_erst(xhci, xhci->event_ring, &xhci->erst, flags);
- if (ret)
+ if (xhci_check_trb_in_td_math(xhci) < 0)
goto fail;
- /* set ERST count with the number of entries in the segment table */
- val = readl(&xhci->ir_set->erst_size);
- val &= ERST_SIZE_MASK;
- val |= ERST_NUM_SEGS;
- xhci_dbg_trace(xhci, trace_xhci_dbg_init,
- "// Write ERST size = %i to ir_set 0 (some bits preserved)",
- val);
- writel(val, &xhci->ir_set->erst_size);
-
- xhci_dbg_trace(xhci, trace_xhci_dbg_init,
- "// Set ERST entries to point to event ring.");
- /* set the segment table base address */
- xhci_dbg_trace(xhci, trace_xhci_dbg_init,
- "// Set ERST base address for ir_set 0 = 0x%llx",
- (unsigned long long)xhci->erst.erst_dma_addr);
- val_64 = xhci_read_64(xhci, &xhci->ir_set->erst_base);
- val_64 &= ERST_PTR_MASK;
- val_64 |= (xhci->erst.erst_dma_addr & (u64) ~ERST_PTR_MASK);
- xhci_write_64(xhci, val_64, &xhci->ir_set->erst_base);
-
- /* Set the event ring dequeue address */
- xhci_set_hc_event_deq(xhci);
- xhci_dbg_trace(xhci, trace_xhci_dbg_init,
- "Wrote ERST address to ir_set 0.");
+ /* Setup secondary interrupters (if any) */
+ INIT_LIST_HEAD(&xhci->xhci_sec_list);
+ for (i = 0; i < xhci->max_interrupters; i++) {
+ sec = kzalloc(sizeof(struct xhci_sec), GFP_KERNEL);
+ if (sec) {
+ sec->intr_num = i + 1;
+ xhci_mem_sec_init(xhci, sec);
+ list_add_tail(&sec->list, &xhci->xhci_sec_list);
+ }
+ }
xhci->isoc_bei_interval = AVOID_BEI_INTERVAL_MAX;
diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c
index 5fb55bf19493..a1b6c17ecf74 100644
--- a/drivers/usb/host/xhci-plat.c
+++ b/drivers/usb/host/xhci-plat.c
@@ -303,6 +303,8 @@ static int xhci_plat_probe(struct platform_device *pdev)
device_property_read_u32(tmpdev, "imod-interval-ns",
&xhci->imod_interval);
+ device_property_read_u8(tmpdev, "num-hc-interrupters",
+ &xhci->max_interrupters);
}
hcd->usb_phy = devm_usb_get_phy_by_phandle(sysdev, "usb-phy", 0);
diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c
index 79d7931c048a..353b06df2000 100644
--- a/drivers/usb/host/xhci.c
+++ b/drivers/usb/host/xhci.c
@@ -2101,6 +2101,64 @@ int xhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
}
EXPORT_SYMBOL_GPL(xhci_add_endpoint);
+static int xhci_stop_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
+ struct usb_host_endpoint *ep)
+{
+ struct xhci_hcd *xhci = hcd_to_xhci(hcd);
+ unsigned int ep_index;
+ struct xhci_virt_device *virt_dev;
+ struct xhci_command *cmd;
+ unsigned long flags;
+ int ret = 0;
+
+ ret = xhci_check_args(hcd, udev, ep, 1, true, __func__);
+ if (ret <= 0)
+ return ret;
+
+ cmd = xhci_alloc_command(xhci, true, GFP_NOIO);
+ if (!cmd)
+ return -ENOMEM;
+
+ spin_lock_irqsave(&xhci->lock, flags);
+ virt_dev = xhci->devs[udev->slot_id];
+ if (!virt_dev) {
+ ret = -ENODEV;
+ goto err;
+ }
+
+ ep_index = xhci_get_endpoint_index(&ep->desc);
+ if (virt_dev->eps[ep_index].ring &&
+ virt_dev->eps[ep_index].ring->dequeue) {
+ ret = xhci_queue_stop_endpoint(xhci, cmd, udev->slot_id,
+ ep_index, 0);
+ if (ret)
+ goto err;
+
+ xhci_ring_cmd_db(xhci);
+ spin_unlock_irqrestore(&xhci->lock, flags);
+
+ /* Wait for stop endpoint command to finish */
+ wait_for_completion(cmd->completion);
+
+ if (cmd->status == COMP_COMMAND_ABORTED ||
+ cmd->status == COMP_STOPPED) {
+ xhci_warn(xhci,
+ "stop endpoint command timeout for ep%d%s\n",
+ usb_endpoint_num(&ep->desc),
+ usb_endpoint_dir_in(&ep->desc) ? "in" : "out");
+ ret = -ETIME;
+ }
+ goto free_cmd;
+ }
+
+err:
+ spin_unlock_irqrestore(&xhci->lock, flags);
+free_cmd:
+ xhci_free_command(xhci, cmd);
+
+ return ret;
+}
+
static void xhci_zero_in_ctx(struct xhci_hcd *xhci, struct xhci_virt_device *virt_dev)
{
struct xhci_input_control_ctx *ctrl_ctx;
@@ -5278,6 +5336,110 @@ static void xhci_hcd_init_usb3_data(struct xhci_hcd *xhci, struct usb_hcd *hcd)
xhci->usb3_rhub.hcd = hcd;
}
+/*
+ * Free a XHCI interrupter that was previously allocated. Ensure that the
+ * event ring which was used is reset to the proper state, and recycle the
+ * interrupter for next use by clearing the XHCI reference.
+ */
+static int xhci_free_interrupter(struct usb_hcd *hcd, int intr_num)
+{
+ struct xhci_hcd *xhci = hcd_to_xhci(hcd);
+ struct xhci_sec *sec;
+
+ mutex_lock(&xhci->mutex);
+ list_for_each_entry(sec, &xhci->xhci_sec_list, list) {
+ if (sec->intr_num == intr_num) {
+ xhci_handle_sec_intr_events(xhci, sec->event_ring, sec->ir_set, sec);
+ sec->xhci = 0;
+ }
+ }
+ mutex_unlock(&xhci->mutex);
+
+ return 0;
+}
+
+/*
+ * Reserve a XHCI interrupter, and pass the base address of the event ring for
+ * this pariticular interrupter back to the client.
+ */
+static phys_addr_t xhci_update_interrupter(struct usb_hcd *hcd, int intr_num,
+ dma_addr_t *dma)
+{
+ struct xhci_hcd *xhci = hcd_to_xhci(hcd);
+ struct device *dev = hcd->self.sysdev;
+ struct sg_table sgt;
+ phys_addr_t pa;
+ struct xhci_sec *sec;
+
+ if (!HCD_RH_RUNNING(hcd) ||
+ (xhci->xhc_state & XHCI_STATE_HALTED))
+ return 0;
+
+ mutex_lock(&xhci->mutex);
+ list_for_each_entry(sec, &xhci->xhci_sec_list, list) {
+ if (sec->intr_num == intr_num) {
+ dma_get_sgtable(dev, &sgt, sec->event_ring->first_seg->trbs,
+ sec->event_ring->first_seg->dma, TRB_SEGMENT_SIZE);
+
+ *dma = sec->event_ring->first_seg->dma;
+
+ pa = page_to_phys(sg_page(sgt.sgl));
+ sg_free_table(&sgt);
+ sec->xhci = xhci;
+ mutex_unlock(&xhci->mutex);
+
+ return pa;
+ }
+ }
+ mutex_unlock(&xhci->mutex);
+
+ return 0;
+}
+
+/* Retrieve the transfer ring base address for a specific endpoint. */
+static phys_addr_t xhci_get_xfer_resource(struct usb_device *udev,
+ struct usb_host_endpoint *ep, dma_addr_t *dma)
+{
+ struct usb_hcd *hcd = bus_to_hcd(udev->bus);
+ struct device *dev = hcd->self.sysdev;
+ struct xhci_hcd *xhci = hcd_to_xhci(hcd);
+ struct sg_table sgt;
+ phys_addr_t pa;
+ int ret;
+ unsigned int ep_index;
+ struct xhci_virt_device *virt_dev;
+
+ if (!HCD_RH_RUNNING(hcd))
+ return 0;
+
+ ret = xhci_check_args(hcd, udev, ep, 1, true, __func__);
+ if (ret <= 0) {
+ xhci_err(xhci, "%s: invalid args\n", __func__);
+ return 0;
+ }
+
+ virt_dev = xhci->devs[udev->slot_id];
+ ep_index = xhci_get_endpoint_index(&ep->desc);
+
+ if (virt_dev->eps[ep_index].ring &&
+ virt_dev->eps[ep_index].ring->first_seg) {
+
+ dma_get_sgtable(dev, &sgt,
+ virt_dev->eps[ep_index].ring->first_seg->trbs,
+ virt_dev->eps[ep_index].ring->first_seg->dma,
+ TRB_SEGMENT_SIZE);
+
+ *dma = virt_dev->eps[ep_index].ring->first_seg->dma;
+
+ pa = page_to_phys(sg_page(sgt.sgl));
+ sg_free_table(&sgt);
+
+ return pa;
+ }
+
+ return 0;
+}
+
int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks)
{
struct xhci_hcd *xhci;
@@ -5321,6 +5483,8 @@ int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks)
xhci->hcc_params2 = readl(&xhci->cap_regs->hcc_params2);
xhci->quirks |= quirks;
+ xhci->max_interrupters = min_t(u32, HCS_MAX_INTRS(xhci->hcs_params1),
+ xhci->max_interrupters);
get_quirks(dev, xhci);
@@ -5454,7 +5618,10 @@ static const struct hc_driver xhci_hc_driver = {
.enable_device = xhci_enable_device,
.update_hub_device = xhci_update_hub_device,
.reset_device = xhci_discover_or_reset_device,
-
+ .update_interrupter = xhci_update_interrupter,
+ .free_interrupter = xhci_free_interrupter,
+ .get_transfer_resource = xhci_get_xfer_resource,
+ .stop_endpoint = xhci_stop_endpoint,
/*
* scheduling support
*/
diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h
index c9f06c5e4e9d..2e694686c849 100644
--- a/drivers/usb/host/xhci.h
+++ b/drivers/usb/host/xhci.h
@@ -1926,6 +1926,7 @@ struct xhci_hcd {
struct dentry *debugfs_root;
struct dentry *debugfs_slots;
struct list_head regset_list;
+ struct list_head xhci_sec_list;
void *dbc;
/* platform-specific data -- must come last */
@@ -1945,6 +1946,17 @@ struct xhci_driver_overrides {
void (*reset_bandwidth)(struct usb_hcd *, struct usb_device *);
};
+struct xhci_sec {
+ struct xhci_ring *event_ring;
+ struct xhci_erst erst;
+ /* secondary interrupter */
+ struct xhci_intr_reg __iomem *ir_set;
+ struct xhci_hcd *xhci;
+ int intr_num;
+
+ struct list_head list;
+};
+
#define XHCI_CFC_DELAY 10
/* convert between an HCD pointer and the corresponding EHCI_HCD */
@@ -2034,6 +2046,9 @@ void xhci_dbg_trace(struct xhci_hcd *xhci, void (*trace)(struct va_format *),
/* xHCI memory management */
void xhci_mem_cleanup(struct xhci_hcd *xhci);
int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags);
+void xhci_handle_sec_intr_events(struct xhci_hcd *xhci,
+ struct xhci_ring *ring, struct xhci_intr_reg __iomem *ir_set, struct xhci_sec *sec);
+int xhci_mem_sec_init(struct xhci_hcd *xhci, struct xhci_sec *sec);
void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id);
int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id, struct usb_device *udev, gfp_t flags);
int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *udev);
