Re: [PATCH v2 5/5] usb:cdns3 Add Cadence USB3 DRD Driver

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



> > +
> > +       if (ret)
> > +               return ret;
> > +
> > +       state = readl(&cdns->otg_regs->sts);
> > +       if (OTGSTS_OTG_NRDY(state) != 0) {
> > +               dev_err(cdns->dev, "Cadence USB3 OTG device not ready\n");
> > +               return -ENODEV;
> > +       }
> > +
> > +       ret = cdns3_drd_update_mode(cdns);
> > +
>
> Calling this function, it is timeout for waiting OTGSTS_XHCI_READY at otgsts,
> do you know possible reasons?  After commenting out this function, my
> xHCI function
> works.
>

Pawel, since OTG compliance (Known as HNP/SRP)  has not been used
widely, Linux kernel does not
maintain it from some time ago (maybe 1-2 years). In software design,
we do not need to consider it from
hardware point, eg, kinds of OTG timer. For dual-role switch on the
fly,  through /sys is enough.

Through the debug, we find it needs to wait 1s after setting de-select
the host or gadget before request
XHCI at otg_regs->cmd, and enable fast simulate can reduce delay to
2-3ms. Would you please help
to check with your hardware team this behavior. With below changes, I
can get OTGSTS_XHCI_READY at otgsts.

@@ -141,6 +143,7 @@ static int cdns3_drd_switch_host(struct cdns3 *cdns, int on)
                writel(OTGCMD_HOST_BUS_DROP | OTGCMD_DEV_BUS_DROP |
                       OTGCMD_DEV_POWER_OFF | OTGCMD_HOST_POWER_OFF,
                       &cdns->otg_regs->cmd);
+               usleep_range(3000, 4000);
        }

        return 0;
@@ -178,6 +181,7 @@ static int cdns3_drd_switch_gadget(struct cdns3
*cdns, int on)
                writel(OTGCMD_HOST_BUS_DROP | OTGCMD_DEV_BUS_DROP |
                       OTGCMD_DEV_POWER_OFF | OTGCMD_HOST_POWER_OFF,
                       &cdns->otg_regs->cmd);
+               usleep_range(3000, 4000);
        }

@@ -299,6 +306,7 @@ int cdns3_drd_init(struct cdns3 *cdns)
                cdns->version  = CDNS3_CONTROLLER_V0;
                cdns->otg_v1_regs = NULL;
                cdns->otg_regs = regs;
+               writel(0x1, &cdns->otg_v0_regs->simulate);
                dev_info(cdns->dev, "DRD version v0 (%08x)\n",
                         readl(&cdns->otg_v0_regs->version));
        } else {

Peter

> Peter
>
> > +       return ret;
> > +}
> > +
> > +int cdns3_drd_exit(struct cdns3 *cdns)
> > +{
> > +       return cdns3_drd_switch_host(cdns, 0);
> > +}
> > diff --git a/drivers/usb/cdns3/drd.h b/drivers/usb/cdns3/drd.h
> > new file mode 100644
> > index 000000000000..6a29cdcb492d
> > --- /dev/null
> > +++ b/drivers/usb/cdns3/drd.h
> > @@ -0,0 +1,162 @@
> > +/* SPDX-License-Identifier: GPL-2.0 */
> > +/*
> > + * Cadence USB3 DRD header file.
> > + *
> > + * Copyright (C) 2018 Cadence.
> > + *
> > + * Author: Pawel Laszczak <pawell@xxxxxxxxxxx>
> > + */
> > +#ifndef __LINUX_CDNS3_DRD
> > +#define __LINUX_CDNS3_DRD
> > +
> > +#include <linux/usb/otg.h>
> > +#include <linux/phy/phy.h>
> > +#include "core.h"
> > +
> > +/*  DRD register interface for version v1. */
> > +struct cdns3_otg_regs {
> > +       __le32 did;
> > +       __le32 rid;
> > +       __le32 capabilities;
> > +       __le32 reserved1;
> > +       __le32 cmd;
> > +       __le32 sts;
> > +       __le32 state;
> > +       __le32 reserved2;
> > +       __le32 ien;
> > +       __le32 ivect;
> > +       __le32 refclk;
> > +       __le32 tmr;
> > +       __le32 reserved3[4];
> > +       __le32 simulate;
> > +       __le32 override;
> > +       __le32 susp_ctrl;
> > +       __le32 reserved4;
> > +       __le32 anasts;
> > +       __le32 adp_ramp_time;
> > +       __le32 ctrl1;
> > +       __le32 ctrl2;
> > +};
> > +
> > +/*  DRD register interface for version v0. */
> > +struct cdns3_otg_legacy_regs {
> > +       __le32 cmd;
> > +       __le32 sts;
> > +       __le32 state;
> > +       __le32 refclk;
> > +       __le32 ien;
> > +       __le32 ivect;
> > +       __le32 reserved1[3];
> > +       __le32 tmr;
> > +       __le32 reserved2[2];
> > +       __le32 version;
> > +       __le32 capabilities;
> > +       __le32 reserved3[2];
> > +       __le32 simulate;
> > +       __le32 reserved4[5];
> > +       __le32 ctrl1;
> > +};
> > +
> > +/*
> > + * Common registers interface for both version of DRD.
> > + */
> > +struct cdns3_otg_common_regs {
> > +       __le32 cmd;
> > +       __le32 sts;
> > +       __le32 state;
> > +       __le32 different1;
> > +       __le32 ien;
> > +       __le32 ivect;
> > +};
> > +
> > +/* CDNS_RID - bitmasks */
> > +#define CDNS_RID(p)                    ((p) & GENMASK(15, 0))
> > +
> > +/* CDNS_VID - bitmasks */
> > +#define CDNS_DID(p)                    ((p) & GENMASK(31, 0))
> > +
> > +/* OTGCMD - bitmasks */
> > +/* "Request the bus for Device mode. */
> > +#define OTGCMD_DEV_BUS_REQ             BIT(0)
> > +/* Request the bus for Host mode */
> > +#define OTGCMD_HOST_BUS_REQ            BIT(1)
> > +/* Enable OTG mode. */
> > +#define OTGCMD_OTG_EN                  BIT(2)
> > +/* Disable OTG mode */
> > +#define OTGCMD_OTG_DIS                 BIT(3)
> > +/*"Configure OTG as A-Device. */
> > +#define OTGCMD_A_DEV_EN                        BIT(4)
> > +/*"Configure OTG as A-Device. */
> > +#define OTGCMD_A_DEV_DIS               BIT(5)
> > +/* Drop the bus for Device mod e. */
> > +#define OTGCMD_DEV_BUS_DROP            BIT(8)
> > +/* Drop the bus for Host mode*/
> > +#define OTGCMD_HOST_BUS_DROP           BIT(9)
> > +/* Power Down USBSS-DEV. */
> > +#define OTGCMD_DEV_POWER_OFF           BIT(11)
> > +/* Power Down CDNSXHCI. */
> > +#define OTGCMD_HOST_POWER_OFF          BIT(12)
> > +
> > +/* OTGIEN - bitmasks */
> > +/* ID change interrupt enable */
> > +#define OTGIEN_ID_CHANGE_INT           BIT(0)
> > +/* Vbusvalid fall detected interrupt enable.*/
> > +#define OTGIEN_VBUSVALID_RISE_INT      BIT(4)
> > +/* Vbusvalid fall detected interrupt enable */
> > +#define OTGIEN_VBUSVALID_FALL_INT      BIT(5)
> > +
> > +/* OTGSTS - bitmasks */
> > +/*
> > + * Current value of the ID pin. It is only valid when idpullup in
> > + *  OTGCTRL1_TYPE register is set to '1'.
> > + */
> > +#define OTGSTS_ID_VALUE                        BIT(0)
> > +/* Current value of the vbus_valid */
> > +#define OTGSTS_VBUS_VALID              BIT(1)
> > +/* Current value of the b_sess_vld */
> > +#define OTGSTS_SESSION_VALID           BIT(2)
> > +/*Device mode is active*/
> > +#define OTGSTS_DEV_ACTIVE              BIT(3)
> > +/* Host mode is active. */
> > +#define OTGSTS_HOST_ACTIVE             BIT(4)
> > +/* OTG Controller not ready. */
> > +#define OTGSTS_OTG_NRDY_MASK           BIT(11)
> > +#define OTGSTS_OTG_NRDY(p)             ((p) & OTGSTS_OTG_NRDY_MASK)
> > +/*
> > + * Value of the strap pins.
> > + * 000 - no default configuration
> > + * 010 - Controller initiall configured as Host
> > + * 100 - Controller initially configured as Device
> > + */
> > +#define OTGSTS_STRAP(p)                        (((p) & GENMASK(14, 12)) >> 12)
> > +#define OTGSTS_STRAP_NO_DEFAULT_CFG    0x00
> > +#define OTGSTS_STRAP_HOST_OTG          0x01
> > +#define OTGSTS_STRAP_HOST              0x02
> > +#define OTGSTS_STRAP_GADGET            0x04
> > +/* Host mode is turned on. */
> > +#define OTGSTS_XHCI_READY              BIT(26)
> > +/* "Device mode is turned on .*/
> > +#define OTGSTS_DEV_READY               BIT(27)
> > +
> > +/* OTGSTATE- bitmasks */
> > +#define OTGSTATE_HOST_STATE_MASK       GENMASK(5, 3)
> > +#define OTGSTATE_HOST_STATE_IDLE       0x0
> > +#define OTGSTATE_HOST_STATE_VBUS_FALL   0x7
> > +#define OTGSTATE_HOST_STATE(p)         (((p) & OTGSTATE_HOST_STATE_MASK) >> 3)
> > +
> > +/* OTGREFCLK - bitmasks */
> > +#define OTGREFCLK_STB_CLK_SWITCH_EN    BIT(31)
> > +
> > +/* OVERRIDE - bitmasks */
> > +#define OVERRIDE_IDPULLUP              BIT(0)
> > +/* Only for CDNS3_CONTROLLER_V0 version */
> > +#define OVERRIDE_IDPULLUP_V0           BIT(24)
> > +
> > +int cdns3_is_host(struct cdns3 *cdns);
> > +int cdns3_is_device(struct cdns3 *cdns);
> > +int cdns3_get_id(struct cdns3 *cdns);
> > +int cdns3_drd_init(struct cdns3 *cdns);
> > +int cdns3_drd_exit(struct cdns3 *cdns);
> > +int cdns3_drd_update_mode(struct cdns3 *cdns);
> > +
> > +#endif /* __LINUX_CDNS3_DRD */
> > diff --git a/drivers/usb/cdns3/ep0.c b/drivers/usb/cdns3/ep0.c
> > new file mode 100644
> > index 000000000000..89cf1cde1555
> > --- /dev/null
> > +++ b/drivers/usb/cdns3/ep0.c
> > @@ -0,0 +1,896 @@
> > +// SPDX-License-Identifier: GPL-2.0
> > +/*
> > + * Cadence USBSS DRD Driver - gadget side.
> > + *
> > + * Copyright (C) 2018 Cadence Design Systems.
> > + * Copyright (C) 2017-2018 NXP
> > + *
> > + * Authors: Pawel Jez <pjez@xxxxxxxxxxx>,
> > + *          Pawel Laszczak <pawell@xxxxxxxxxxx>
> > + *         Peter Chen <peter.chen@xxxxxxx>
> > + */
> > +
> > +#include <linux/usb/composite.h>
> > +
> > +#include "gadget.h"
> > +#include "trace.h"
> > +
> > +static struct usb_endpoint_descriptor cdns3_gadget_ep0_desc = {
> > +       .bLength = USB_DT_ENDPOINT_SIZE,
> > +       .bDescriptorType = USB_DT_ENDPOINT,
> > +       .bmAttributes = USB_ENDPOINT_XFER_CONTROL,
> > +};
> > +
> > +/**
> > + * cdns3_ep0_run_transfer - Do transfer on default endpoint hardware
> > + * @priv_dev: extended gadget object
> > + * @dma_addr: physical address where data is/will be stored
> > + * @length: data length
> > + * @erdy: set it to 1 when ERDY packet should be sent -
> > + *        exit from flow control state
> > + */
> > +static void cdns3_ep0_run_transfer(struct cdns3_device *priv_dev,
> > +                                  dma_addr_t dma_addr,
> > +                                  unsigned int length, int erdy)
> > +{
> > +       struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
> > +       struct cdns3_endpoint *priv_ep = priv_dev->eps[0];
> > +
> > +       priv_ep->trb_pool->buffer = TRB_BUFFER(dma_addr);
> > +       priv_ep->trb_pool->length = TRB_LEN(length);
> > +       priv_ep->trb_pool->control = TRB_CYCLE | TRB_IOC | TRB_TYPE(TRB_NORMAL);
> > +
> > +       trace_cdns3_prepare_trb(priv_ep, priv_ep->trb_pool);
> > +
> > +       cdns3_select_ep(priv_dev, priv_dev->ep0_data_dir);
> > +
> > +       writel(EP_STS_TRBERR, &regs->ep_sts);
> > +       writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma), &regs->ep_traddr);
> > +       trace_cdns3_doorbell_ep0(priv_dev->ep0_data_dir ? "ep0in" : "ep0out",
> > +                                readl(&regs->ep_traddr));
> > +
> > +       /* TRB should be prepared before starting transfer */
> > +       writel(EP_CMD_DRDY, &regs->ep_cmd);
> > +
> > +       if (erdy)
> > +               writel(EP_CMD_ERDY, &priv_dev->regs->ep_cmd);
> > +}
> > +
> > +/**
> > + * cdns3_ep0_delegate_req - Returns status of handling setup packet
> > + * Setup is handled by gadget driver
> > + * @priv_dev: extended gadget object
> > + * @ctrl_req: pointer to received setup packet
> > + *
> > + * Returns zero on success or negative value on failure
> > + */
> > +static int cdns3_ep0_delegate_req(struct cdns3_device *priv_dev,
> > +                                 struct usb_ctrlrequest *ctrl_req)
> > +{
> > +       int ret;
> > +
> > +       spin_unlock(&priv_dev->lock);
> > +       priv_dev->setup_pending = 1;
> > +       ret = priv_dev->gadget_driver->setup(&priv_dev->gadget, ctrl_req);
> > +       priv_dev->setup_pending = 0;
> > +       spin_lock(&priv_dev->lock);
> > +       return ret;
> > +}
> > +
> > +static void cdns3_prepare_setup_packet(struct cdns3_device *priv_dev)
> > +{
> > +       priv_dev->ep0_data_dir = 0;
> > +       priv_dev->ep0_stage = CDNS3_SETUP_STAGE;
> > +       cdns3_ep0_run_transfer(priv_dev, priv_dev->setup_dma,
> > +                              sizeof(struct usb_ctrlrequest), 0);
> > +}
> > +
> > +static void cdns3_ep0_complete_setup(struct cdns3_device *priv_dev,
> > +                                    u8 send_stall, u8 send_erdy)
> > +{
> > +       struct cdns3_endpoint *priv_ep = priv_dev->eps[0];
> > +       struct usb_request *request;
> > +
> > +       request = cdns3_next_request(&priv_ep->pending_req_list);
> > +       if (request)
> > +               list_del_init(&request->list);
> > +
> > +       if (send_stall) {
> > +               dev_dbg(priv_dev->dev, "STALL for ep0\n");
> > +               /* set_stall on ep0 */
> > +               cdns3_select_ep(priv_dev, 0x00);
> > +               writel(EP_CMD_SSTALL, &priv_dev->regs->ep_cmd);
> > +       } else {
> > +               cdns3_prepare_setup_packet(priv_dev);
> > +       }
> > +
> > +       priv_dev->ep0_stage = CDNS3_SETUP_STAGE;
> > +       writel((send_erdy ? EP_CMD_ERDY : 0) | EP_CMD_REQ_CMPL,
> > +              &priv_dev->regs->ep_cmd);
> > +}
> > +
> > +/**
> > + * cdns3_req_ep0_set_configuration - Handling of SET_CONFIG standard USB request
> > + * @priv_dev: extended gadget object
> > + * @ctrl_req: pointer to received setup packet
> > + *
> > + * Returns 0 if success, USB_GADGET_DELAYED_STATUS on deferred status stage,
> > + * error code on error
> > + */
> > +static int cdns3_req_ep0_set_configuration(struct cdns3_device *priv_dev,
> > +                                          struct usb_ctrlrequest *ctrl_req)
> > +{
> > +       enum usb_device_state device_state = priv_dev->gadget.state;
> > +       struct cdns3_endpoint *priv_ep;
> > +       u32 config = le16_to_cpu(ctrl_req->wValue);
> > +       int result = 0;
> > +       int i;
> > +
> > +       switch (device_state) {
> > +       case USB_STATE_ADDRESS:
> > +               /* Configure non-control EPs */
> > +               for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) {
> > +                       priv_ep = priv_dev->eps[i];
> > +                       if (!priv_ep)
> > +                               continue;
> > +
> > +                       if (priv_ep->flags & EP_CLAIMED)
> > +                               cdns3_ep_config(priv_ep);
> > +               }
> > +
> > +               result = cdns3_ep0_delegate_req(priv_dev, ctrl_req);
> > +
> > +               if (result)
> > +                       return result;
> > +
> > +               if (config) {
> > +                       cdns3_set_hw_configuration(priv_dev);
> > +               } else {
> > +                       cdns3_hw_reset_eps_config(priv_dev);
> > +                       usb_gadget_set_state(&priv_dev->gadget,
> > +                                            USB_STATE_ADDRESS);
> > +               }
> > +               break;
> > +       case USB_STATE_CONFIGURED:
> > +               result = cdns3_ep0_delegate_req(priv_dev, ctrl_req);
> > +
> > +               if (!config && !result) {
> > +                       cdns3_hw_reset_eps_config(priv_dev);
> > +                       usb_gadget_set_state(&priv_dev->gadget,
> > +                                            USB_STATE_ADDRESS);
> > +               }
> > +               break;
> > +       default:
> > +               result = -EINVAL;
> > +       }
> > +
> > +       return result;
> > +}
> > +
> > +/**
> > + * cdns3_req_ep0_set_address - Handling of SET_ADDRESS standard USB request
> > + * @priv_dev: extended gadget object
> > + * @ctrl_req: pointer to received setup packet
> > + *
> > + * Returns 0 if success, error code on error
> > + */
> > +static int cdns3_req_ep0_set_address(struct cdns3_device *priv_dev,
> > +                                    struct usb_ctrlrequest *ctrl_req)
> > +{
> > +       enum usb_device_state device_state = priv_dev->gadget.state;
> > +       u32 reg;
> > +       u32 addr;
> > +
> > +       addr = le16_to_cpu(ctrl_req->wValue);
> > +
> > +       if (addr > USB_DEVICE_MAX_ADDRESS) {
> > +               dev_err(priv_dev->dev,
> > +                       "Device address (%d) cannot be greater than %d\n",
> > +                       addr, USB_DEVICE_MAX_ADDRESS);
> > +               return -EINVAL;
> > +       }
> > +
> > +       if (device_state == USB_STATE_CONFIGURED) {
> > +               dev_err(priv_dev->dev,
> > +                       "can't set_address from configured state\n");
> > +               return -EINVAL;
> > +       }
> > +
> > +       reg = readl(&priv_dev->regs->usb_cmd);
> > +
> > +       writel(reg | USB_CMD_FADDR(addr) | USB_CMD_SET_ADDR,
> > +              &priv_dev->regs->usb_cmd);
> > +
> > +       usb_gadget_set_state(&priv_dev->gadget,
> > +                            (addr ? USB_STATE_ADDRESS : USB_STATE_DEFAULT));
> > +
> > +       return 0;
> > +}
> > +
> > +/**
> > + * cdns3_req_ep0_get_status - Handling of GET_STATUS standard USB request
> > + * @priv_dev: extended gadget object
> > + * @ctrl_req: pointer to received setup packet
> > + *
> > + * Returns 0 if success, error code on error
> > + */
> > +static int cdns3_req_ep0_get_status(struct cdns3_device *priv_dev,
> > +                                   struct usb_ctrlrequest *ctrl)
> > +{
> > +       __le16 *response_pkt;
> > +       u16 usb_status = 0;
> > +       u32 recip;
> > +       u32 reg;
> > +
> > +       recip = ctrl->bRequestType & USB_RECIP_MASK;
> > +
> > +       switch (recip) {
> > +       case USB_RECIP_DEVICE:
> > +               /* self powered */
> > +               if (priv_dev->is_selfpowered)
> > +                       usb_status = BIT(USB_DEVICE_SELF_POWERED);
> > +
> > +               if (priv_dev->wake_up_flag)
> > +                       usb_status |= BIT(USB_DEVICE_REMOTE_WAKEUP);
> > +
> > +               if (priv_dev->gadget.speed != USB_SPEED_SUPER)
> > +                       break;
> > +
> > +               reg = readl(&priv_dev->regs->usb_sts);
> > +
> > +               if (priv_dev->u1_allowed)
> > +                       usb_status |= BIT(USB_DEV_STAT_U1_ENABLED);
> > +
> > +               if (priv_dev->u2_allowed)
> > +                       usb_status |= BIT(USB_DEV_STAT_U2_ENABLED);
> > +
> > +               break;
> > +       case USB_RECIP_INTERFACE:
> > +               return cdns3_ep0_delegate_req(priv_dev, ctrl);
> > +       case USB_RECIP_ENDPOINT:
> > +               /* check if endpoint is stalled */
> > +               cdns3_select_ep(priv_dev, ctrl->wIndex);
> > +               if (EP_STS_STALL(readl(&priv_dev->regs->ep_sts)))
> > +                       usb_status =  BIT(USB_ENDPOINT_HALT);
> > +               break;
> > +       default:
> > +               return -EINVAL;
> > +       }
> > +
> > +       response_pkt = (__le16 *)priv_dev->setup_buf;
> > +       *response_pkt = cpu_to_le16(usb_status);
> > +
> > +       cdns3_ep0_run_transfer(priv_dev, priv_dev->setup_dma,
> > +                              sizeof(*response_pkt), 1);
> > +       return 0;
> > +}
> > +
> > +static int cdns3_ep0_feature_handle_device(struct cdns3_device *priv_dev,
> > +                                          struct usb_ctrlrequest *ctrl,
> > +                                          int set)
> > +{
> > +       enum usb_device_state state;
> > +       enum usb_device_speed speed;
> > +       int ret = 0;
> > +       u32 wValue;
> > +       u32 wIndex;
> > +       u16 tmode;
> > +
> > +       wValue = le16_to_cpu(ctrl->wValue);
> > +       wIndex = le16_to_cpu(ctrl->wIndex);
> > +       state = priv_dev->gadget.state;
> > +       speed = priv_dev->gadget.speed;
> > +
> > +       switch (ctrl->wValue) {
> > +       case USB_DEVICE_REMOTE_WAKEUP:
> > +               priv_dev->wake_up_flag = !!set;
> > +               break;
> > +       case USB_DEVICE_U1_ENABLE:
> > +               if (state != USB_STATE_CONFIGURED || speed != USB_SPEED_SUPER)
> > +                       return -EINVAL;
> > +
> > +               priv_dev->u1_allowed = !!set;
> > +               break;
> > +       case USB_DEVICE_U2_ENABLE:
> > +               if (state != USB_STATE_CONFIGURED || speed != USB_SPEED_SUPER)
> > +                       return -EINVAL;
> > +
> > +               priv_dev->u2_allowed = !!set;
> > +               break;
> > +       case USB_DEVICE_LTM_ENABLE:
> > +               ret = -EINVAL;
> > +               break;
> > +       case USB_DEVICE_TEST_MODE:
> > +               if (state != USB_STATE_CONFIGURED || speed > USB_SPEED_HIGH)
> > +                       return -EINVAL;
> > +
> > +               tmode = le16_to_cpu(ctrl->wIndex);
> > +
> > +               if (!set || (tmode & 0xff) != 0)
> > +                       return -EINVAL;
> > +
> > +               switch (tmode >> 8) {
> > +               case TEST_J:
> > +               case TEST_K:
> > +               case TEST_SE0_NAK:
> > +               case TEST_PACKET:
> > +                       cdns3_ep0_complete_setup(priv_dev, 0, 1);
> > +                       /**
> > +                        *  Little delay to give the controller some time
> > +                        * for sending status stage.
> > +                        * This time should be less then 3ms.
> > +                        */
> > +                       usleep_range(1000, 2000);
> > +                       cdns3_set_register_bit(&priv_dev->regs->usb_cmd,
> > +                                              USB_CMD_STMODE |
> > +                                              USB_STS_TMODE_SEL(tmode - 1));
> > +                       break;
> > +               default:
> > +                       ret = -EINVAL;
> > +               }
> > +               break;
> > +       default:
> > +               ret = -EINVAL;
> > +       }
> > +
> > +       return ret;
> > +}
> > +
> > +static int cdns3_ep0_feature_handle_intf(struct cdns3_device *priv_dev,
> > +                                        struct usb_ctrlrequest *ctrl,
> > +                                        int set)
> > +{
> > +       u32 wValue;
> > +       int ret = 0;
> > +
> > +       wValue = le16_to_cpu(ctrl->wValue);
> > +
> > +       switch (wValue) {
> > +       case USB_INTRF_FUNC_SUSPEND:
> > +               break;
> > +       default:
> > +               ret = -EINVAL;
> > +       }
> > +
> > +       return ret;
> > +}
> > +
> > +static int cdns3_ep0_feature_handle_endpoint(struct cdns3_device *priv_dev,
> > +                                            struct usb_ctrlrequest *ctrl,
> > +                                            int set)
> > +{
> > +       struct cdns3_endpoint *priv_ep;
> > +       int ret = 0;
> > +       u8 index;
> > +
> > +       if (le16_to_cpu(ctrl->wValue) != USB_ENDPOINT_HALT)
> > +               return -EINVAL;
> > +
> > +       if (!(ctrl->wIndex & ~USB_DIR_IN))
> > +               return 0;
> > +
> > +       index = cdns3_ep_addr_to_index(ctrl->wIndex);
> > +       priv_ep = priv_dev->eps[index];
> > +
> > +       cdns3_select_ep(priv_dev, ctrl->wIndex);
> > +
> > +       if (set) {
> > +               writel(EP_CMD_SSTALL, &priv_dev->regs->ep_cmd);
> > +               priv_ep->flags |= EP_STALL;
> > +       } else {
> > +               struct usb_request *request;
> > +
> > +               if (priv_dev->eps[index]->flags & EP_WEDGE) {
> > +                       cdns3_select_ep(priv_dev, 0x00);
> > +                       return 0;
> > +               }
> > +
> > +               writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
> > +
> > +               /* wait for EPRST cleared */
> > +               ret = cdns3_handshake(&priv_dev->regs->ep_cmd,
> > +                                     EP_CMD_EPRST, 0, 100);
> > +               if (ret)
> > +                       return -EINVAL;
> > +
> > +               priv_ep->flags &= ~EP_STALL;
> > +
> > +               request = cdns3_next_request(&priv_ep->pending_req_list);
> > +               if (request)
> > +                       cdns3_ep_run_transfer(priv_ep, request);
> > +       }
> > +       return ret;
> > +}
> > +
> > +/**
> > + * cdns3_req_ep0_handle_feature -
> > + * Handling of GET/SET_FEATURE standard USB request
> > + *
> > + * @priv_dev: extended gadget object
> > + * @ctrl_req: pointer to received setup packet
> > + * @set: must be set to 1 for SET_FEATURE request
> > + *
> > + * Returns 0 if success, error code on error
> > + */
> > +static int cdns3_req_ep0_handle_feature(struct cdns3_device *priv_dev,
> > +                                       struct usb_ctrlrequest *ctrl,
> > +                                       int set)
> > +{
> > +       int ret = 0;
> > +       u32 recip;
> > +
> > +       recip = ctrl->bRequestType & USB_RECIP_MASK;
> > +
> > +       switch (recip) {
> > +       case USB_RECIP_DEVICE:
> > +               ret = cdns3_ep0_feature_handle_device(priv_dev, ctrl, set);
> > +               break;
> > +       case USB_RECIP_INTERFACE:
> > +               ret = cdns3_ep0_feature_handle_intf(priv_dev, ctrl, set);
> > +               break;
> > +       case USB_RECIP_ENDPOINT:
> > +               ret = cdns3_ep0_feature_handle_endpoint(priv_dev, ctrl, set);
> > +               break;
> > +       default:
> > +               return -EINVAL;
> > +       }
> > +
> > +       return ret;
> > +}
> > +
> > +/**
> > + * cdns3_req_ep0_set_sel - Handling of SET_SEL standard USB request
> > + * @priv_dev: extended gadget object
> > + * @ctrl_req: pointer to received setup packet
> > + *
> > + * Returns 0 if success, error code on error
> > + */
> > +static int cdns3_req_ep0_set_sel(struct cdns3_device *priv_dev,
> > +                                struct usb_ctrlrequest *ctrl_req)
> > +{
> > +       if (priv_dev->gadget.state < USB_STATE_ADDRESS)
> > +               return -EINVAL;
> > +
> > +       if (ctrl_req->wLength != 6) {
> > +               dev_err(priv_dev->dev, "Set SEL should be 6 bytes, got %d\n",
> > +                       ctrl_req->wLength);
> > +               return -EINVAL;
> > +       }
> > +
> > +       cdns3_ep0_run_transfer(priv_dev, priv_dev->setup_dma, 6, 1);
> > +       return 0;
> > +}
> > +
> > +/**
> > + * cdns3_req_ep0_set_isoch_delay -
> > + * Handling of GET_ISOCH_DELAY standard USB request
> > + * @priv_dev: extended gadget object
> > + * @ctrl_req: pointer to received setup packet
> > + *
> > + * Returns 0 if success, error code on error
> > + */
> > +static int cdns3_req_ep0_set_isoch_delay(struct cdns3_device *priv_dev,
> > +                                        struct usb_ctrlrequest *ctrl_req)
> > +{
> > +       if (ctrl_req->wIndex || ctrl_req->wLength)
> > +               return -EINVAL;
> > +
> > +       priv_dev->isoch_delay = ctrl_req->wValue;
> > +
> > +       return 0;
> > +}
> > +
> > +/**
> > + * cdns3_ep0_standard_request - Handling standard USB requests
> > + * @priv_dev: extended gadget object
> > + * @ctrl_req: pointer to received setup packet
> > + *
> > + * Returns 0 if success, error code on error
> > + */
> > +static int cdns3_ep0_standard_request(struct cdns3_device *priv_dev,
> > +                                     struct usb_ctrlrequest *ctrl_req)
> > +{
> > +       int ret;
> > +
> > +       switch (ctrl_req->bRequest) {
> > +       case USB_REQ_SET_ADDRESS:
> > +               ret = cdns3_req_ep0_set_address(priv_dev, ctrl_req);
> > +               break;
> > +       case USB_REQ_SET_CONFIGURATION:
> > +               ret = cdns3_req_ep0_set_configuration(priv_dev, ctrl_req);
> > +               break;
> > +       case USB_REQ_GET_STATUS:
> > +               ret = cdns3_req_ep0_get_status(priv_dev, ctrl_req);
> > +               break;
> > +       case USB_REQ_CLEAR_FEATURE:
> > +               ret = cdns3_req_ep0_handle_feature(priv_dev, ctrl_req, 0);
> > +               break;
> > +       case USB_REQ_SET_FEATURE:
> > +               ret = cdns3_req_ep0_handle_feature(priv_dev, ctrl_req, 1);
> > +               break;
> > +       case USB_REQ_SET_SEL:
> > +               ret = cdns3_req_ep0_set_sel(priv_dev, ctrl_req);
> > +               break;
> > +       case USB_REQ_SET_ISOCH_DELAY:
> > +               ret = cdns3_req_ep0_set_isoch_delay(priv_dev, ctrl_req);
> > +               break;
> > +       default:
> > +               ret = cdns3_ep0_delegate_req(priv_dev, ctrl_req);
> > +               break;
> > +       }
> > +
> > +       return ret;
> > +}
> > +
> > +static void __pending_setup_status_handler(struct cdns3_device *priv_dev)
> > +{
> > +       struct usb_request *request = priv_dev->pending_status_request;
> > +
> > +       if (priv_dev->status_completion_no_call && request &&
> > +           request->complete) {
> > +               request->complete(&priv_dev->eps[0]->endpoint, request);
> > +               priv_dev->status_completion_no_call = 0;
> > +       }
> > +}
> > +
> > +void cdns3_pending_setup_status_handler(struct work_struct *work)
> > +{
> > +       struct cdns3_device *priv_dev = container_of(work, struct cdns3_device,
> > +                       pending_status_wq);
> > +       unsigned long flags;
> > +
> > +       spin_lock_irqsave(&priv_dev->lock, flags);
> > +       __pending_setup_status_handler(priv_dev);
> > +       spin_unlock_irqrestore(&priv_dev->lock, flags);
> > +}
> > +
> > +/**
> > + * cdns3_gadget_ep_giveback - call struct usb_request's ->complete callback
> > + * @priv_ep: The endpoint to whom the request belongs to
> > + * @priv_req: The request we're giving back
> > + * @status: completion code for the request
> > + *
> > + * Must be called with controller's lock held and interrupts disabled. This
> > + * function will unmap @req and call its ->complete() callback to notify upper
> > + * layers that it has completed.
> > + */
> > +
> > +void cdns3_gadget_ep0_giveback(struct cdns3_device *priv_dev,
> > +                              int status)
> > +{
> > +       struct cdns3_endpoint *priv_ep;
> > +       struct usb_request *request;
> > +
> > +       priv_ep = priv_dev->eps[0];
> > +       request = cdns3_next_request(&priv_ep->pending_req_list);
> > +
> > +       priv_ep->dir = priv_dev->ep0_data_dir;
> > +       cdns3_gadget_giveback(priv_ep, to_cdns3_request(request), status);
> > +}
> > +
> > +/**
> > + * cdns3_ep0_setup_phase - Handling setup USB requests
> > + * @priv_dev: extended gadget object
> > + */
> > +static void cdns3_ep0_setup_phase(struct cdns3_device *priv_dev)
> > +{
> > +       struct usb_ctrlrequest *ctrl = priv_dev->setup_buf;
> > +       struct cdns3_endpoint *priv_ep = priv_dev->eps[0];
> > +       int result;
> > +
> > +       priv_dev->ep0_data_dir = ctrl->bRequestType & USB_DIR_IN;
> > +
> > +       trace_cdns3_ctrl_req(ctrl);
> > +
> > +       if (!list_empty(&priv_ep->pending_req_list)) {
> > +               struct usb_request *request;
> > +
> > +               request = cdns3_next_request(&priv_ep->pending_req_list);
> > +               priv_ep->dir = priv_dev->ep0_data_dir;
> > +               cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
> > +                                     -ECONNRESET);
> > +       }
> > +
> > +       if (le16_to_cpu(ctrl->wLength))
> > +               priv_dev->ep0_stage = CDNS3_DATA_STAGE;
> > +       else
> > +               priv_dev->ep0_stage = CDNS3_STATUS_STAGE;
> > +
> > +       if ((ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD)
> > +               result = cdns3_ep0_standard_request(priv_dev, ctrl);
> > +       else
> > +               result = cdns3_ep0_delegate_req(priv_dev, ctrl);
> > +
> > +       if (priv_dev->ep0_stage == CDNS3_STATUS_STAGE &&
> > +           result != USB_GADGET_DELAYED_STATUS) {
> > +               cdns3_ep0_complete_setup(priv_dev, 0, 1);
> > +       } else if (result < 0) {
> > +               cdns3_ep0_complete_setup(priv_dev, 1, 1);
> > +       }
> > +}
> > +
> > +static void cdns3_transfer_completed(struct cdns3_device *priv_dev)
> > +{
> > +       struct cdns3_endpoint *priv_ep = priv_dev->eps[0];
> > +
> > +       if (!list_empty(&priv_ep->pending_req_list)) {
> > +               struct usb_request *request;
> > +
> > +               trace_cdns3_complete_trb(priv_ep, priv_ep->trb_pool);
> > +               request = cdns3_next_request(&priv_ep->pending_req_list);
> > +
> > +               request->actual =
> > +                       TRB_LEN(le32_to_cpu(priv_ep->trb_pool->length));
> > +
> > +               priv_ep->dir = priv_dev->ep0_data_dir;
> > +               cdns3_gadget_giveback(priv_ep, to_cdns3_request(request), 0);
> > +       }
> > +
> > +       cdns3_ep0_complete_setup(priv_dev, 0, 0);
> > +}
> > +
> > +/**
> > + * cdns3_check_new_setup - Check if controller receive new SETUP packet.
> > + * @priv_dev: extended gadget object
> > + *
> > + * The SETUP packet can be kept in on-chip memory or in system memory.
> > + */
> > +static bool cdns3_check_new_setup(struct cdns3_device *priv_dev)
> > +{
> > +       u32 ep_sts_reg;
> > +
> > +       cdns3_select_ep(priv_dev, 0 | USB_DIR_OUT);
> > +       ep_sts_reg = readl(&priv_dev->regs->ep_sts);
> > +
> > +       return !!(ep_sts_reg & (EP_STS_SETUP | EP_STS_STPWAIT));
> > +}
> > +
> > +/**
> > + * cdns3_check_ep0_interrupt_proceed - Processes interrupt related to endpoint 0
> > + * @priv_dev: extended gadget object
> > + * @dir: USB_DIR_IN for IN direction, USB_DIR_OUT for OUT direction
> > + */
> > +void cdns3_check_ep0_interrupt_proceed(struct cdns3_device *priv_dev, int dir)
> > +{
> > +       u32 ep_sts_reg;
> > +
> > +       cdns3_select_ep(priv_dev, dir);
> > +
> > +       ep_sts_reg = readl(&priv_dev->regs->ep_sts);
> > +       writel(ep_sts_reg, &priv_dev->regs->ep_sts);
> > +
> > +       trace_cdns3_ep0_irq(priv_dev, ep_sts_reg);
> > +
> > +       __pending_setup_status_handler(priv_dev);
> > +
> > +       if ((ep_sts_reg & EP_STS_SETUP)) {
> > +               cdns3_ep0_setup_phase(priv_dev);
> > +       } else if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP)) {
> > +               priv_dev->ep0_data_dir = dir;
> > +               cdns3_transfer_completed(priv_dev);
> > +       }
> > +
> > +       if (ep_sts_reg & EP_STS_DESCMIS) {
> > +               if (dir == 0 && !priv_dev->setup_pending)
> > +                       cdns3_prepare_setup_packet(priv_dev);
> > +       }
> > +}
> > +
> > +/**
> > + * cdns3_gadget_ep0_enable
> > + * Function shouldn't be called by gadget driver,
> > + * endpoint 0 is allways active
> > + */
> > +static int cdns3_gadget_ep0_enable(struct usb_ep *ep,
> > +                                  const struct usb_endpoint_descriptor *desc)
> > +{
> > +       return -EINVAL;
> > +}
> > +
> > +/**
> > + * cdns3_gadget_ep0_disable
> > + * Function shouldn't be called by gadget driver,
> > + * endpoint 0 is allways active
> > + */
> > +static int cdns3_gadget_ep0_disable(struct usb_ep *ep)
> > +{
> > +       return -EINVAL;
> > +}
> > +
> > +/**
> > + * cdns3_gadget_ep0_set_halt
> > + * @ep: pointer to endpoint zero object
> > + * @value: 1 for set stall, 0 for clear stall
> > + *
> > + * Returns 0
> > + */
> > +static int cdns3_gadget_ep0_set_halt(struct usb_ep *ep, int value)
> > +{
> > +       /* TODO */
> > +       return 0;
> > +}
> > +
> > +/**
> > + * cdns3_gadget_ep0_queue Transfer data on endpoint zero
> > + * @ep: pointer to endpoint zero object
> > + * @request: pointer to request object
> > + * @gfp_flags: gfp flags
> > + *
> > + * Returns 0 on success, error code elsewhere
> > + */
> > +static int cdns3_gadget_ep0_queue(struct usb_ep *ep,
> > +                                 struct usb_request *request,
> > +                                 gfp_t gfp_flags)
> > +{
> > +       struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
> > +       struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
> > +       unsigned long flags;
> > +       int erdy_sent = 0;
> > +       int ret = 0;
> > +
> > +       dev_dbg(priv_dev->dev, "Queue to Ep0%s L: %d\n",
> > +               priv_dev->ep0_data_dir ? "IN" : "OUT",
> > +               request->length);
> > +
> > +       /* cancel the request if controller receive new SETUP packet. */
> > +       if (cdns3_check_new_setup(priv_dev))
> > +               return -ECONNRESET;
> > +
> > +       /* send STATUS stage. Should be called only for SET_CONFIGURATION */
> > +       if (priv_dev->ep0_stage == CDNS3_STATUS_STAGE) {
> > +               spin_lock_irqsave(&priv_dev->lock, flags);
> > +               cdns3_select_ep(priv_dev, 0x00);
> > +
> > +               erdy_sent = !priv_dev->hw_configured_flag;
> > +               cdns3_set_hw_configuration(priv_dev);
> > +
> > +               if (!erdy_sent)
> > +                       cdns3_ep0_complete_setup(priv_dev, 0, 1);
> > +
> > +               request->actual = 0;
> > +               priv_dev->status_completion_no_call = true;
> > +               priv_dev->pending_status_request = request;
> > +               spin_unlock_irqrestore(&priv_dev->lock, flags);
> > +
> > +               /*
> > +                * Since there is no completion interrupt for status stage,
> > +                * it needs to call ->completion in software after
> > +                * ep0_queue is back.
> > +                */
> > +               queue_work(system_freezable_wq, &priv_dev->pending_status_wq);
> > +               return 0;
> > +       }
> > +
> > +       spin_lock_irqsave(&priv_dev->lock, flags);
> > +       if (!list_empty(&priv_ep->pending_req_list)) {
> > +               dev_err(priv_dev->dev,
> > +                       "can't handle multiple requests for ep0\n");
> > +               spin_unlock_irqrestore(&priv_dev->lock, flags);
> > +               return -EBUSY;
> > +       }
> > +
> > +       ret = usb_gadget_map_request_by_dev(priv_dev->sysdev, request,
> > +                                           priv_dev->ep0_data_dir);
> > +       if (ret) {
> > +               spin_unlock_irqrestore(&priv_dev->lock, flags);
> > +               dev_err(priv_dev->dev, "failed to map request\n");
> > +               return -EINVAL;
> > +       }
> > +
> > +       request->status = -EINPROGRESS;
> > +       list_add_tail(&request->list, &priv_ep->pending_req_list);
> > +       cdns3_ep0_run_transfer(priv_dev, request->dma, request->length, 1);
> > +       spin_unlock_irqrestore(&priv_dev->lock, flags);
> > +
> > +       return ret;
> > +}
> > +
> > +/**
> > + * cdns3_gadget_ep_set_wedge Set wedge on selected endpoint
> > + * @ep: endpoint object
> > + *
> > + * Returns 0
> > + */
> > +int cdns3_gadget_ep_set_wedge(struct usb_ep *ep)
> > +{
> > +       struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
> > +       struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
> > +
> > +       dev_dbg(priv_dev->dev, "Wedge for %s\n", ep->name);
> > +       cdns3_gadget_ep_set_halt(ep, 1);
> > +       priv_ep->flags |= EP_WEDGE;
> > +
> > +       return 0;
> > +}
> > +
> > +const struct usb_ep_ops cdns3_gadget_ep0_ops = {
> > +       .enable = cdns3_gadget_ep0_enable,
> > +       .disable = cdns3_gadget_ep0_disable,
> > +       .alloc_request = cdns3_gadget_ep_alloc_request,
> > +       .free_request = cdns3_gadget_ep_free_request,
> > +       .queue = cdns3_gadget_ep0_queue,
> > +       .dequeue = cdns3_gadget_ep_dequeue,
> > +       .set_halt = cdns3_gadget_ep0_set_halt,
> > +       .set_wedge = cdns3_gadget_ep_set_wedge,
> > +};
> > +
> > +/**
> > + * cdns3_ep0_config - Configures default endpoint
> > + * @priv_dev: extended gadget object
> > + *
> > + * Functions sets parameters: maximal packet size and enables interrupts
> > + */
> > +void cdns3_ep0_config(struct cdns3_device *priv_dev)
> > +{
> > +       struct cdns3_usb_regs __iomem *regs;
> > +       struct cdns3_endpoint *priv_ep;
> > +       u32 max_packet_size = 64;
> > +
> > +       regs = priv_dev->regs;
> > +
> > +       if (priv_dev->gadget.speed == USB_SPEED_SUPER)
> > +               max_packet_size = 512;
> > +
> > +       priv_ep = priv_dev->eps[0];
> > +
> > +       if (!list_empty(&priv_ep->pending_req_list)) {
> > +               struct usb_request *request;
> > +
> > +               request = cdns3_next_request(&priv_ep->pending_req_list);
> > +               list_del_init(&request->list);
> > +       }
> > +
> > +       priv_dev->u1_allowed = 0;
> > +       priv_dev->u2_allowed = 0;
> > +
> > +       priv_dev->gadget.ep0->maxpacket = max_packet_size;
> > +       cdns3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(max_packet_size);
> > +
> > +       /* init ep out */
> > +       cdns3_select_ep(priv_dev, USB_DIR_OUT);
> > +
> > +       writel(EP_CFG_ENABLE | EP_CFG_MAXPKTSIZE(max_packet_size),
> > +              &regs->ep_cfg);
> > +
> > +       writel(EP_STS_EN_SETUPEN | EP_STS_EN_DESCMISEN | EP_STS_EN_TRBERREN,
> > +              &regs->ep_sts_en);
> > +
> > +       /* init ep in */
> > +       cdns3_select_ep(priv_dev, USB_DIR_IN);
> > +
> > +       writel(EP_CFG_ENABLE | EP_CFG_MAXPKTSIZE(max_packet_size),
> > +              &regs->ep_cfg);
> > +
> > +       writel(EP_STS_EN_SETUPEN | EP_STS_EN_TRBERREN, &regs->ep_sts_en);
> > +
> > +       cdns3_set_register_bit(&regs->usb_conf, USB_CONF_U1DS | USB_CONF_U2DS);
> > +}
> > +
> > +/**
> > + * cdns3_init_ep0 Initializes software endpoint 0 of gadget
> > + * @priv_dev: extended gadget object
> > + * @ep_priv: extended endpoint object
> > + *
> > + * Returns 0 on success else error code.
> > + */
> > +int cdns3_init_ep0(struct cdns3_device *priv_dev,
> > +                  struct cdns3_endpoint *priv_ep)
> > +{
> > +       sprintf(priv_ep->name, "ep0");
> > +
> > +       /* fill linux fields */
> > +       priv_ep->endpoint.ops = &cdns3_gadget_ep0_ops;
> > +       priv_ep->endpoint.maxburst = 1;
> > +       usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
> > +                                  CDNS3_EP0_MAX_PACKET_LIMIT);
> > +       priv_ep->endpoint.address = 0;
> > +       priv_ep->endpoint.caps.type_control = 1;
> > +       priv_ep->endpoint.caps.dir_in = 1;
> > +       priv_ep->endpoint.caps.dir_out = 1;
> > +       priv_ep->endpoint.name = priv_ep->name;
> > +       priv_ep->endpoint.desc = &cdns3_gadget_ep0_desc;
> > +       priv_dev->gadget.ep0 = &priv_ep->endpoint;
> > +       priv_ep->type = USB_ENDPOINT_XFER_CONTROL;
> > +
> > +       return cdns3_allocate_trb_pool(priv_ep);
> > +}
> > diff --git a/drivers/usb/cdns3/gadget-export.h b/drivers/usb/cdns3/gadget-export.h
> > new file mode 100644
> > index 000000000000..577469eee961
> > --- /dev/null
> > +++ b/drivers/usb/cdns3/gadget-export.h
> > @@ -0,0 +1,28 @@
> > +/* SPDX-License-Identifier: GPL-2.0 */
> > +/*
> > + * Cadence USBSS DRD Driver - Gadget Export APIs.
> > + *
> > + * Copyright (C) 2017 NXP
> > + * Copyright (C) 2017-2018 NXP
> > + *
> > + * Authors: Peter Chen <peter.chen@xxxxxxx>
> > + */
> > +#ifndef __LINUX_CDNS3_GADGET_EXPORT
> > +#define __LINUX_CDNS3_GADGET_EXPORT
> > +
> > +#ifdef CONFIG_USB_CDNS3_GADGET
> > +
> > +int cdns3_gadget_init(struct cdns3 *cdns);
> > +void cdns3_gadget_exit(struct cdns3 *cdns);
> > +#else
> > +
> > +static inline int cdns3_gadget_init(struct cdns3 *cdns)
> > +{
> > +       return -ENXIO;
> > +}
> > +
> > +static inline void cdns3_gadget_exit(struct cdns3 *cdns) { }
> > +
> > +#endif
> > +
> > +#endif /* __LINUX_CDNS3_GADGET_EXPORT */
> > diff --git a/drivers/usb/cdns3/gadget.c b/drivers/usb/cdns3/gadget.c
> > new file mode 100644
> > index 000000000000..0d95eb00be37
> > --- /dev/null
> > +++ b/drivers/usb/cdns3/gadget.c
> > @@ -0,0 +1,2102 @@
> > +// SPDX-License-Identifier: GPL-2.0
> > +/*
> > + * Cadence USBSS DRD Driver - gadget side.
> > + *
> > + * Copyright (C) 2018 Cadence Design Systems.
> > + * Copyright (C) 2017-2018 NXP
> > + *
> > + * Authors: Pawel Jez <pjez@xxxxxxxxxxx>,
> > + *          Pawel Laszczak <pawell@xxxxxxxxxxx>
> > + *         Peter Chen <peter.chen@xxxxxxx>
> > + */
> > +
> > +#include <linux/dma-mapping.h>
> > +#include <linux/usb/gadget.h>
> > +
> > +#include "core.h"
> > +#include "gadget-export.h"
> > +#include "gadget.h"
> > +
> > +#include "trace.h"
> > +
> > +static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
> > +                                  struct usb_request *request,
> > +                                  gfp_t gfp_flags);
> > +
> > +/**
> > + * cdns3_handshake - spin reading  until handshake completes or fails
> > + * @ptr: address of device controller register to be read
> > + * @mask: bits to look at in result of read
> > + * @done: value of those bits when handshake succeeds
> > + * @usec: timeout in microseconds
> > + *
> > + * Returns negative errno, or zero on success
> > + *
> > + * Success happens when the "mask" bits have the specified value (hardware
> > + * handshake done). There are two failure modes: "usec" have passed (major
> > + * hardware flakeout), or the register reads as all-ones (hardware removed).
> > + */
> > +int cdns3_handshake(void __iomem *ptr, u32 mask, u32 done, int usec)
> > +{
> > +       u32     result;
> > +
> > +       do {
> > +               result = readl(ptr);
> > +               if (result == ~(u32)0)  /* card removed */
> > +                       return -ENODEV;
> > +               result &= mask;
> > +               if (result == done)
> > +                       return 0;
> > +               udelay(1);
> > +               usec--;
> > +       } while (usec > 0);
> > +       return -ETIMEDOUT;
> > +}
> > +
> > +/**
> > + * cdns3_set_register_bit - set bit in given register.
> > + * @ptr: address of device controller register to be read and changed
> > + * @mask: bits requested to set
> > + */
> > +void cdns3_set_register_bit(void __iomem *ptr, u32 mask)
> > +{
> > +       mask = readl(ptr) | mask;
> > +       writel(mask, ptr);
> > +}
> > +
> > +/**
> > + * cdns3_ep_addr_to_index - Macro converts endpoint address to
> > + * index of endpoint object in cdns3_device.eps[] container
> > + * @ep_addr: endpoint address for which endpoint object is required
> > + *
> > + */
> > +u8 cdns3_ep_addr_to_index(u8 ep_addr)
> > +{
> > +       return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0));
> > +}
> > +
> > +/**
> > + * cdns3_next_request - returns next request from list
> > + * @list: list containing requests
> > + *
> > + * Returns request or NULL if no requests in list
> > + */
> > +struct usb_request *cdns3_next_request(struct list_head *list)
> > +{
> > +       if (list_empty(list))
> > +               return NULL;
> > +       return list_first_entry_or_null(list, struct usb_request, list);
> > +}
> > +
> > +/**
> > + * cdns3_next_priv_request - returns next request from list
> > + * @list: list containing requests
> > + *
> > + * Returns request or NULL if no requests in list
> > + */
> > +struct cdns3_request *cdns3_next_priv_request(struct list_head *list)
> > +{
> > +       if (list_empty(list))
> > +               return NULL;
> > +       return list_first_entry_or_null(list, struct cdns3_request, list);
> > +}
> > +
> > +/**
> > + * select_ep - selects endpoint
> > + * @priv_dev:  extended gadget object
> > + * @ep: endpoint address
> > + */
> > +void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep)
> > +{
> > +       if (priv_dev->selected_ep == ep)
> > +               return;
> > +
> > +       priv_dev->selected_ep = ep;
> > +       writel(ep, &priv_dev->regs->ep_sel);
> > +}
> > +
> > +dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
> > +                                struct cdns3_trb *trb)
> > +{
> > +       u32 offset = (char *)trb - (char *)priv_ep->trb_pool;
> > +
> > +       return priv_ep->trb_pool_dma + offset;
> > +}
> > +
> > +int cdns3_ring_size(struct cdns3_endpoint *priv_ep)
> > +{
> > +       switch (priv_ep->type) {
> > +       case USB_ENDPOINT_XFER_ISOC:
> > +               return TRB_ISO_RING_SIZE;
> > +       case USB_ENDPOINT_XFER_CONTROL:
> > +               return TRB_CTRL_RING_SIZE;
> > +       default:
> > +               return TRB_RING_SIZE;
> > +       }
> > +}
> > +
> > +/**
> > + * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint
> > + * @priv_ep:  endpoint object
> > + *
> > + * Function will return 0 on success or -ENOMEM on allocation error
> > + */
> > +int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
> > +{
> > +       struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
> > +       int ring_size = cdns3_ring_size(priv_ep);
> > +       struct cdns3_trb *link_trb;
> > +
> > +       if (!priv_ep->trb_pool) {
> > +               priv_ep->trb_pool = dma_zalloc_coherent(priv_dev->sysdev,
> > +                                                       ring_size,
> > +                                                       &priv_ep->trb_pool_dma,
> > +                                                       GFP_DMA);
> > +               if (!priv_ep->trb_pool)
> > +                       return -ENOMEM;
> > +       } else {
> > +               memset(priv_ep->trb_pool, 0, ring_size);
> > +       }
> > +
> > +       if (!priv_ep->num)
> > +               return 0;
> > +
> > +       if (!priv_ep->aligned_buff) {
> > +               void *buff = dma_alloc_coherent(priv_dev->sysdev,
> > +                                               CDNS3_ALIGNED_BUF_SIZE,
> > +                                               &priv_ep->aligned_dma_addr,
> > +                                               GFP_DMA);
> > +
> > +               priv_ep->aligned_buff  = buff;
> > +               if (!priv_ep->aligned_buff) {
> > +                       dma_free_coherent(priv_dev->sysdev,
> > +                                         ring_size,
> > +                                         priv_ep->trb_pool,
> > +                                         priv_ep->trb_pool_dma);
> > +                       priv_ep->trb_pool = NULL;
> > +
> > +                       return -ENOMEM;
> > +               }
> > +       }
> > +
> > +       priv_ep->num_trbs = ring_size / TRB_SIZE;
> > +       /* Initialize the last TRB as Link TRB */
> > +       link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1));
> > +       link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma);
> > +       link_trb->control = TRB_CYCLE | TRB_TYPE(TRB_LINK) |
> > +                           TRB_CHAIN | TRB_TOGGLE;
> > +
> > +       return 0;
> > +}
> > +
> > +static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep)
> > +{
> > +       struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
> > +
> > +       if (priv_ep->trb_pool) {
> > +               dma_free_coherent(priv_dev->sysdev,
> > +                                 cdns3_ring_size(priv_ep),
> > +                                 priv_ep->trb_pool, priv_ep->trb_pool_dma);
> > +               priv_ep->trb_pool = NULL;
> > +       }
> > +
> > +       if (priv_ep->aligned_buff) {
> > +               dma_free_coherent(priv_dev->sysdev, CDNS3_ALIGNED_BUF_SIZE,
> > +                                 priv_ep->aligned_buff,
> > +                                 priv_ep->aligned_dma_addr);
> > +               priv_ep->aligned_buff = NULL;
> > +       }
> > +}
> > +
> > +/**
> > + * cdns3_data_flush - flush data at onchip buffer
> > + * @priv_ep: endpoint object
> > + *
> > + * Endpoint must be selected before call to this function
> > + *
> > + * Returns zero on success or negative value on failure
> > + */
> > +static int cdns3_data_flush(struct cdns3_endpoint *priv_ep)
> > +{
> > +       struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
> > +
> > +       writel(EP_CMD_DFLUSH, &priv_dev->regs->ep_cmd);
> > +
> > +       /* wait for DFLUSH cleared */
> > +       return cdns3_handshake(&priv_dev->regs->ep_cmd, EP_CMD_DFLUSH, 0, 100);
> > +}
> > +
> > +/**
> > + * cdns3_ep_stall_flush - Stalls and flushes selected endpoint
> > + * @priv_ep: endpoint object
> > + *
> > + * Endpoint must be selected before call to this function
> > + */
> > +static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep)
> > +{
> > +       struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
> > +
> > +       writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL,
> > +              &priv_dev->regs->ep_cmd);
> > +
> > +       /* wait for DFLUSH cleared */
> > +       cdns3_handshake(&priv_dev->regs->ep_cmd, EP_CMD_DFLUSH, 0, 100);
> > +       priv_ep->flags |= EP_STALL;
> > +}
> > +
> > +/**
> > + * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller.
> > + * @priv_dev: extended gadget object
> > + */
> > +void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev)
> > +{
> > +       writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf);
> > +
> > +       cdns3_allow_enable_l1(priv_dev, 0);
> > +       priv_dev->hw_configured_flag = 0;
> > +       priv_dev->onchip_mem_allocated_size = 0;
> > +}
> > +
> > +/**
> > + * cdns3_ep_inc_trb - increment a trb index.
> > + * @index: Pointer to the TRB index to increment.
> > + * @cs: Cycle state
> > + * @trb_in_seg: number of TRBs in segment
> > + *
> > + * The index should never point to the link TRB. After incrementing,
> > + * if it is point to the link TRB, wrap around to the beginning and revert
> > + * cycle state bit The
> > + * link TRB is always at the last TRB entry.
> > + */
> > +static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
> > +{
> > +       (*index)++;
> > +       if (*index == (trb_in_seg - 1)) {
> > +               *index = 0;
> > +               *cs ^=  1;
> > +       }
> > +}
> > +
> > +/**
> > + * cdns3_ep_inc_enq - increment endpoint's enqueue pointer
> > + * @priv_ep: The endpoint whose enqueue pointer we're incrementing
> > + */
> > +static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep)
> > +{
> > +       priv_ep->free_trbs--;
> > +       cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs);
> > +}
> > +
> > +/**
> > + * cdns3_ep_inc_deq - increment endpoint's dequeue pointer
> > + * @priv_ep: The endpoint whose dequeue pointer we're incrementing
> > + */
> > +static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
> > +{
> > +       priv_ep->free_trbs++;
> > +       cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
> > +}
> > +
> > +/**
> > + * cdns3_allow_enable_l1 - enable/disable permits to transition to L1.
> > + * @priv_dev: Extended gadget object
> > + * @enable: Enable/disable permit to transition to L1.
> > + *
> > + * If bit USB_CONF_L1EN is set and device receive Extended Token packet,
> > + * then controller answer with ACK handshake.
> > + * If bit USB_CONF_L1DS is set and device receive Extended Token packet,
> > + * then controller answer with NYET handshake.
> > + */
> > +void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable)
> > +{
> > +       if (enable)
> > +               writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf);
> > +       else
> > +               writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf);
> > +}
> > +
> > +enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev)
> > +{
> > +       u32 reg;
> > +
> > +       reg = readl(&priv_dev->regs->usb_sts);
> > +
> > +       if (DEV_SUPERSPEED(reg))
> > +               return USB_SPEED_SUPER;
> > +       else if (DEV_HIGHSPEED(reg))
> > +               return USB_SPEED_HIGH;
> > +       else if (DEV_FULLSPEED(reg))
> > +               return USB_SPEED_FULL;
> > +       else if (DEV_LOWSPEED(reg))
> > +               return USB_SPEED_LOW;
> > +       return USB_SPEED_UNKNOWN;
> > +}
> > +
> > +/**
> > + * cdns3_start_all_request - add to ring all request not started
> > + * @priv_dev: Extended gadget object
> > + * @priv_ep: The endpoint for whom request will be started.
> > + *
> > + * Returns return ENOMEM if transfer ring i not enough TRBs to start
> > + *         all requests.
> > + */
> > +static int cdns3_start_all_request(struct cdns3_device *priv_dev,
> > +                                  struct cdns3_endpoint *priv_ep)
> > +{
> > +       struct cdns3_request *priv_req;
> > +       struct usb_request *request;
> > +       int ret = 0;
> > +
> > +       while (!list_empty(&priv_ep->deferred_req_list)) {
> > +               request = cdns3_next_request(&priv_ep->deferred_req_list);
> > +               priv_req = to_cdns3_request(request);
> > +
> > +               ret = cdns3_ep_run_transfer(priv_ep, request);
> > +               if (ret)
> > +                       return ret;
> > +
> > +               list_del(&request->list);
> > +               list_add_tail(&request->list,
> > +                             &priv_ep->pending_req_list);
> > +       }
> > +
> > +       priv_ep->flags &= ~EP_RING_FULL;
> > +       return ret;
> > +}
> > +
> > +/**
> > + * cdns3_descmiss_copy_data copy data from internal requests to request queued
> > + * by class driver.
> > + * @priv_ep: extended endpoint object
> > + * @request: request object
> > + */
> > +static void cdns3_descmiss_copy_data(struct cdns3_endpoint *priv_ep,
> > +                                    struct usb_request *request)
> > +{
> > +       struct usb_request *descmiss_req;
> > +       struct cdns3_request *descmiss_priv_req;
> > +
> > +       while (!list_empty(&priv_ep->descmiss_req_list)) {
> > +               int chunk_end;
> > +               int length;
> > +
> > +               descmiss_priv_req =
> > +                       cdns3_next_priv_request(&priv_ep->descmiss_req_list);
> > +               descmiss_req = &descmiss_priv_req->request;
> > +
> > +               /* driver can't touch pending request */
> > +               if (descmiss_priv_req->flags & REQUEST_PENDING)
> > +                       break;
> > +
> > +               chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH;
> > +               length = request->actual + descmiss_req->actual;
> > +
> > +               if (length <= request->length) {
> > +                       memcpy(&((u8 *)request->buf)[request->actual],
> > +                              descmiss_req->buf,
> > +                              descmiss_req->actual);
> > +                       request->actual = length;
> > +               } else {
> > +                       /* It should never occures */
> > +                       request->status = -ENOMEM;
> > +               }
> > +
> > +               list_del_init(&descmiss_priv_req->list);
> > +
> > +               kfree(descmiss_req->buf);
> > +               cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req);
> > +
> > +               if (!chunk_end)
> > +                       break;
> > +       }
> > +}
> > +
> > +/**
> > + * cdns3_gadget_giveback - call struct usb_request's ->complete callback
> > + * @priv_ep: The endpoint to whom the request belongs to
> > + * @priv_req: The request we're giving back
> > + * @status: completion code for the request
> > + *
> > + * Must be called with controller's lock held and interrupts disabled. This
> > + * function will unmap @req and call its ->complete() callback to notify upper
> > + * layers that it has completed.
> > + */
> > +void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
> > +                          struct cdns3_request *priv_req,
> > +                          int status)
> > +{
> > +       struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
> > +       struct usb_request *request = &priv_req->request;
> > +
> > +       list_del_init(&request->list);
> > +
> > +       if (request->status == -EINPROGRESS)
> > +               request->status = status;
> > +
> > +       usb_gadget_unmap_request_by_dev(priv_dev->sysdev, request,
> > +                                       priv_ep->dir);
> > +
> > +       priv_req->flags &= ~REQUEST_PENDING;
> > +       trace_cdns3_gadget_giveback(priv_req);
> > +
> > +       /* WA1: */
> > +       if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN &&
> > +           priv_req->flags & REQUEST_INTERNAL) {
> > +               struct usb_request *req;
> > +
> > +               req = cdns3_next_request(&priv_ep->deferred_req_list);
> > +               request = req;
> > +               priv_ep->descmis_req = NULL;
> > +
> > +               if (!req)
> > +                       return;
> > +
> > +               cdns3_descmiss_copy_data(priv_ep, req);
> > +               if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) &&
> > +                   req->length != req->actual) {
> > +                       /* wait for next part of transfer */
> > +                       return;
> > +               }
> > +
> > +               if (req->status == -EINPROGRESS)
> > +                       req->status = 0;
> > +
> > +               list_del_init(&req->list);
> > +               cdns3_start_all_request(priv_dev, priv_ep);
> > +       }
> > +
> > +       /* Start all not pending request */
> > +       if (priv_ep->flags & EP_RING_FULL)
> > +               cdns3_start_all_request(priv_dev, priv_ep);
> > +
> > +       if (request->complete) {
> > +               spin_unlock(&priv_dev->lock);
> > +               usb_gadget_giveback_request(&priv_ep->endpoint,
> > +                                           request);
> > +               spin_lock(&priv_dev->lock);
> > +       }
> > +
> > +       if (request->buf == priv_dev->zlp_buf)
> > +               cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
> > +}
> > +
> > +/**
> > + * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware
> > + * @priv_ep: endpoint object
> > + *
> > + * Returns zero on success or negative value on failure
> > + */
> > +int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
> > +                         struct usb_request *request)
> > +{
> > +       struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
> > +       struct cdns3_request *priv_req;
> > +       struct cdns3_trb *trb;
> > +       dma_addr_t trb_dma;
> > +       int sg_iter = 0;
> > +       u32 first_pcs;
> > +       int  num_trb;
> > +       int address;
> > +       int pcs;
> > +
> > +       if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
> > +               num_trb = priv_ep->interval;
> > +       else
> > +               num_trb = request->num_sgs ? request->num_sgs : 1;
> > +
> > +       if (num_trb > priv_ep->free_trbs) {
> > +               priv_ep->flags |= EP_RING_FULL;
> > +               return -ENOBUFS;
> > +       }
> > +
> > +       priv_req = to_cdns3_request(request);
> > +       address = priv_ep->endpoint.desc->bEndpointAddress;
> > +
> > +       priv_ep->flags |= EP_PENDING_REQUEST;
> > +       trb_dma = request->dma;
> > +
> > +       /* must allocate buffer aligned to 8 */
> > +       if ((request->dma % 8)) {
> > +               if (request->length <= CDNS3_ALIGNED_BUF_SIZE) {
> > +                       memcpy(priv_ep->aligned_buff, request->buf,
> > +                              request->length);
> > +                       trb_dma = priv_ep->aligned_dma_addr;
> > +               } else {
> > +                       return -ENOMEM;
> > +               }
> > +       }
> > +
> > +       trb = priv_ep->trb_pool + priv_ep->enqueue;
> > +       priv_req->start_trb = priv_ep->enqueue;
> > +       priv_req->trb = trb;
> > +
> > +       /* prepare ring */
> > +       if ((priv_ep->enqueue + num_trb)  >= (priv_ep->num_trbs - 1)) {
> > +               /*updating C bt in  Link TRB before starting DMA*/
> > +               struct cdns3_trb *link_trb = priv_ep->trb_pool +
> > +                                            (priv_ep->num_trbs - 1);
> > +               link_trb->control = ((priv_ep->pcs) ? TRB_CYCLE : 0) |
> > +                                   TRB_TYPE(TRB_LINK) | TRB_CHAIN |
> > +                                   TRB_TOGGLE;
> > +       }
> > +
> > +       first_pcs = priv_ep->pcs ? TRB_CYCLE : 0;
> > +
> > +       do {
> > +       /* fill TRB */
> > +               trb->buffer = TRB_BUFFER(request->num_sgs == 0
> > +                               ? trb_dma : request->sg[sg_iter].dma_address);
> > +
> > +               trb->length = TRB_BURST_LEN(16) |
> > +                   TRB_LEN(request->num_sgs == 0 ?
> > +                               request->length : request->sg[sg_iter].length);
> > +
> > +               trb->control = TRB_TYPE(TRB_NORMAL);
> > +               pcs = priv_ep->pcs ? TRB_CYCLE : 0;
> > +
> > +               /*
> > +                * first trb should be prepared as last to avoid processing
> > +                *  transfer to early
> > +                */
> > +               if (sg_iter != 0)
> > +                       trb->control |= pcs;
> > +
> > +               if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir) {
> > +                       trb->control |= TRB_IOC | TRB_ISP;
> > +               } else {
> > +                       /* for last element in TD or in SG list */
> > +                       if (sg_iter == (num_trb - 1) && sg_iter != 0)
> > +                               trb->control |= pcs | TRB_IOC | TRB_ISP;
> > +               }
> > +               ++sg_iter;
> > +               priv_req->end_trb = priv_ep->enqueue;
> > +               cdns3_ep_inc_enq(priv_ep);
> > +               trb = priv_ep->trb_pool + priv_ep->enqueue;
> > +       } while (sg_iter < num_trb);
> > +
> > +       trb = priv_req->trb;
> > +
> > +       /*
> > +        * Memory barrier = Cycle Bit must be set before trb->length  and
> > +        * trb->buffer fields.
> > +        */
> > +       wmb();
> > +
> > +       priv_req->flags |= REQUEST_PENDING;
> > +
> > +       /* give the TD to the consumer*/
> > +       if (sg_iter == 1)
> > +               trb->control |= first_pcs | TRB_IOC | TRB_ISP;
> > +       else
> > +               trb->control |= first_pcs;
> > +
> > +       trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
> > +       trace_cdns3_ring(priv_ep);
> > +
> > +       /* arm transfer on selected endpoint */
> > +       cdns3_select_ep(priv_ep->cdns3_dev, address);
> > +
> > +       /*
> > +        * For DMULT mode we can set address to transfer ring only once after
> > +        * enabling endpoint.
> > +        */
> > +       if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
> > +               writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma),
> > +                      &priv_dev->regs->ep_traddr);
> > +               priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
> > +       }
> > +
> > +       /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
> > +       writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
> > +       trace_cdns3_doorbell_epx(priv_ep->name,
> > +                                readl(&priv_dev->regs->ep_traddr));
> > +       writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
> > +
> > +       return 0;
> > +}
> > +
> > +void cdns3_set_hw_configuration(struct cdns3_device *priv_dev)
> > +{
> > +       struct cdns3_endpoint *priv_ep;
> > +       struct usb_ep *ep;
> > +       int result = 0;
> > +
> > +       if (priv_dev->hw_configured_flag)
> > +               return;
> > +
> > +       writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf);
> > +       writel(EP_CMD_ERDY | EP_CMD_REQ_CMPL, &priv_dev->regs->ep_cmd);
> > +
> > +       cdns3_set_register_bit(&priv_dev->regs->usb_conf,
> > +                              USB_CONF_U1EN | USB_CONF_U2EN);
> > +
> > +       /* wait until configuration set */
> > +       result = cdns3_handshake(&priv_dev->regs->usb_sts,
> > +                                USB_STS_CFGSTS_MASK, 1, 100);
> > +
> > +       priv_dev->hw_configured_flag = 1;
> > +       cdns3_allow_enable_l1(priv_dev, 1);
> > +
> > +       list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
> > +               if (ep->enabled) {
> > +                       priv_ep = ep_to_cdns3_ep(ep);
> > +                       cdns3_start_all_request(priv_dev, priv_ep);
> > +               }
> > +       }
> > +}
> > +
> > +/**
> > + * cdns3_request_handled - check whether request has been handled by DMA
> > + *
> > + * @priv_ep: extended endpoint object.
> > + * @priv_req: request object for checking
> > + *
> > + * Endpoint must be selected before invoking this function.
> > + *
> > + * Returns false if request has not been handled by DMA, else returns true.
> > + *
> > + * SR - start ring
> > + * ER -  end ring
> > + * DQ = priv_ep->dequeue - dequeue position
> > + * EQ = priv_ep->enqueue -  enqueue position
> > + * ST = priv_req->start_trb - index of first TRB in transfer ring
> > + * ET = priv_req->end_trb - index of last TRB in transfer ring
> > + * CI = current_index - index of processed TRB by DMA.
> > + *
> > + * As first step, function checks if cycle bit for priv_req->start_trb is
> > + * correct.
> > + *
> > + * some rules:
> > + * 1. priv_ep->dequeue never exceed current_index.
> > + * 2  priv_ep->enqueue never exceed priv_ep->dequeue
> > + *
> > + * Then We can split recognition into two parts:
> > + * Case 1 - priv_ep->dequeue < current_index
> > + *      SR ... EQ ... DQ ... CI ... ER
> > + *      SR ... DQ ... CI ... EQ ... ER
> > + *
> > + *      Request has been handled by DMA if ST and ET is between DQ and CI.
> > + *
> > + * Case 2 - priv_ep->dequeue > current_index
> > + * This situation take place when CI go through the LINK TRB at the end of
> > + * transfer ring.
> > + *      SR ... CI ... EQ ... DQ ... ER
> > + *
> > + *      Request has been handled by DMA if ET is less then CI or
> > + *      ET is greater or equal DQ.
> > + */
> > +static bool cdns3_request_handled(struct cdns3_endpoint *priv_ep,
> > +                                 struct cdns3_request *priv_req)
> > +{
> > +       struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
> > +       struct cdns3_trb *trb = priv_req->trb;
> > +       int current_index = 0;
> > +       int handled = 0;
> > +
> > +       current_index = (readl(&priv_dev->regs->ep_traddr) -
> > +                        priv_ep->trb_pool_dma) / TRB_SIZE;
> > +
> > +       trb = &priv_ep->trb_pool[priv_req->start_trb];
> > +
> > +       if ((trb->control  & TRB_CYCLE) != priv_ep->ccs)
> > +               goto finish;
> > +
> > +       if (priv_ep->dequeue < current_index) {
> > +               if ((current_index == (priv_ep->num_trbs - 1)) &&
> > +                   !priv_ep->dequeue)
> > +                       goto finish;
> > +
> > +               if (priv_req->end_trb >= priv_ep->dequeue &&
> > +                   priv_req->end_trb < current_index)
> > +                       handled = 1;
> > +       } else if (priv_ep->dequeue  > current_index) {
> > +               if (priv_req->end_trb  < current_index ||
> > +                   priv_req->end_trb >= priv_ep->dequeue)
> > +                       handled = 1;
> > +       }
> > +
> > +finish:
> > +       trace_cdns3_request_handled(priv_req, current_index, handled);
> > +
> > +       return handled;
> > +}
> > +
> > +static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
> > +                                    struct cdns3_endpoint *priv_ep)
> > +{
> > +       struct cdns3_request *priv_req;
> > +       struct usb_request *request;
> > +       struct cdns3_trb *trb;
> > +       int current_trb;
> > +
> > +       while (!list_empty(&priv_ep->pending_req_list)) {
> > +               request = cdns3_next_request(&priv_ep->pending_req_list);
> > +               priv_req = to_cdns3_request(request);
> > +
> > +               if (!cdns3_request_handled(priv_ep, priv_req))
> > +                       return;
> > +
> > +               if (request->dma % 8 && priv_ep->dir == USB_DIR_OUT)
> > +                       memcpy(request->buf, priv_ep->aligned_buff,
> > +                              request->length);
> > +
> > +               trb = priv_ep->trb_pool + priv_ep->dequeue;
> > +               trace_cdns3_complete_trb(priv_ep, trb);
> > +
> > +               if (trb != priv_req->trb)
> > +                       dev_warn(priv_dev->dev,
> > +                                "request_trb=0x%p, queue_trb=0x%p\n",
> > +                                priv_req->trb, trb);
> > +
> > +               request->actual = TRB_LEN(le32_to_cpu(trb->length));
> > +               current_trb = priv_req->start_trb;
> > +
> > +               while (current_trb != priv_req->end_trb) {
> > +                       cdns3_ep_inc_deq(priv_ep);
> > +                       current_trb = priv_ep->dequeue;
> > +               }
> > +
> > +               cdns3_ep_inc_deq(priv_ep);
> > +               cdns3_gadget_giveback(priv_ep, priv_req, 0);
> > +       }
> > +       priv_ep->flags &= ~EP_PENDING_REQUEST;
> > +}
> > +
> > +/**
> > + * cdns3_descmissing_packet - handles descriptor missing event.
> > + * @priv_dev: extended gadget object
> > + *
> > + * WA1: Controller for OUT endpoints has shared on-chip buffers for all incoming
> > + * packets, including ep0out. It's FIFO buffer, so packets must be handle by DMA
> > + * in correct order. If the first packet in the buffer will not be handled,
> > + * then the following packets directed for other endpoints and  functions
> > + * will be blocked.
> > + * Additionally the packets directed to one endpoint can block entire on-chip
> > + * buffers. In this case transfer to other endpoints also will blocked.
> > + *
> > + * To resolve this issue after raising the descriptor missing interrupt
> > + * driver prepares internal usb_request object and use it to arm DMA transfer.
> > + *
> > + * The problematic situation was observed in case when endpoint has been enabled
> > + * but no usb_request were queued. Driver try detects such endpoints and will
> > + * use this workaround only for these endpoint.
> > + *
> > + * Driver use limited number of buffer. This number can be set by macro
> > + * CDNS_WA1_NUM_BUFFERS.
> > + *
> > + * Such blocking situation was observed on ACM gadget. For this function
> > + * host send OUT data packet but ACM function is not prepared for this packet.
> > + *
> > + * It's limitation of controller but maybe this issues should be fixed in
> > + * function driver.
> > + */
> > +static int cdns3_descmissing_packet(struct cdns3_endpoint *priv_ep)
> > +{
> > +       struct cdns3_request *priv_req;
> > +       struct usb_request *request;
> > +
> > +       if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
> > +               priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
> > +               priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN;
> > +       }
> > +
> > +       request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint,
> > +                                               GFP_ATOMIC);
> > +       if (!request)
> > +               return -ENOMEM;
> > +
> > +       priv_req = to_cdns3_request(request);
> > +       priv_req->flags |= REQUEST_INTERNAL;
> > +
> > +       /* if this field is still assigned it indicate that transfer related
> > +        * with this request has not been finished yet. Driver in this
> > +        * case simply allocate next request and assign flag REQUEST_INTERNAL_CH
> > +        * flag to previous one. It will indicate that current request is
> > +        * part of the previous one.
> > +        */
> > +       if (priv_ep->descmis_req)
> > +               priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH;
> > +
> > +       priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE,
> > +                                       GFP_ATOMIC);
> > +       if (!priv_req) {
> > +               cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
> > +               return -ENOMEM;
> > +       }
> > +
> > +       priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE;
> > +       priv_ep->descmis_req = priv_req;
> > +
> > +       __cdns3_gadget_ep_queue(&priv_ep->endpoint,
> > +                               &priv_ep->descmis_req->request,
> > +                               GFP_ATOMIC);
> > +
> > +       return 0;
> > +}
> > +
> > +/**
> > + * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint
> > + * @priv_ep: endpoint object
> > + *
> > + * Returns 0
> > + */
> > +static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep)
> > +{
> > +       struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
> > +       u32 ep_sts_reg;
> > +
> > +       cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
> > +
> > +       trace_cdns3_epx_irq(priv_dev, priv_ep);
> > +
> > +       ep_sts_reg = readl(&priv_dev->regs->ep_sts);
> > +       writel(ep_sts_reg, &priv_dev->regs->ep_sts);
> > +
> > +       if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP)) {
> > +               if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
> > +                       if (ep_sts_reg & EP_STS_ISP)
> > +                               priv_ep->flags |= EP_QUIRK_END_TRANSFER;
> > +                       else
> > +                               priv_ep->flags &= ~EP_QUIRK_END_TRANSFER;
> > +               }
> > +               cdns3_transfer_completed(priv_dev, priv_ep);
> > +       }
> > +
> > +       /*
> > +        * For isochronous transfer driver completes request on IOC or on
> > +        * TRBERR. IOC appears only when device receive OUT data packet.
> > +        * If host disable stream or lost some packet then the only way to
> > +        * finish all queued transfer is to do it on TRBERR event.
> > +        */
> > +       if ((ep_sts_reg & EP_STS_TRBERR) &&
> > +           priv_ep->type == USB_ENDPOINT_XFER_ISOC)
> > +               cdns3_transfer_completed(priv_dev, priv_ep);
> > +
> > +       /*
> > +        * WA1: this condition should only be meet when
> > +        * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or
> > +        * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN.
> > +        * In other cases this interrupt will be disabled/
> > +        */
> > +       if (ep_sts_reg & EP_STS_DESCMIS) {
> > +               int err;
> > +
> > +               err = cdns3_descmissing_packet(priv_ep);
> > +               if (err)
> > +                       dev_err(priv_dev->dev,
> > +                               "Failed: No sufficient memory for DESCMIS\n");
> > +       }
> > +
> > +       return 0;
> > +}
> > +
> > +/**
> > + * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device
> > + * @priv_dev: extended gadget object
> > + * @usb_ists: bitmap representation of device's reported interrupts
> > + * (usb_ists register value)
> > + */
> > +static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,
> > +                                             u32 usb_ists)
> > +{
> > +       int speed = 0;
> > +
> > +       trace_cdns3_usb_irq(priv_dev, usb_ists);
> > +       /* Connection detected */
> > +       if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) {
> > +               speed = cdns3_get_speed(priv_dev);
> > +               priv_dev->gadget.speed = speed;
> > +               usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED);
> > +               cdns3_ep0_config(priv_dev);
> > +       }
> > +
> > +       /* Disconnection detected */
> > +       if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) {
> > +               if (priv_dev->gadget_driver &&
> > +                   priv_dev->gadget_driver->disconnect) {
> > +                       spin_unlock(&priv_dev->lock);
> > +                       priv_dev->gadget_driver->disconnect(&priv_dev->gadget);
> > +                       spin_lock(&priv_dev->lock);
> > +               }
> > +
> > +               priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
> > +               usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
> > +               cdns3_hw_reset_eps_config(priv_dev);
> > +       }
> > +
> > +       /* reset*/
> > +       if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) {
> > +               /*read again to check the actuall speed*/
> > +               speed = cdns3_get_speed(priv_dev);
> > +               usb_gadget_set_state(&priv_dev->gadget, USB_STATE_DEFAULT);
> > +               priv_dev->gadget.speed = speed;
> > +               cdns3_hw_reset_eps_config(priv_dev);
> > +               cdns3_ep0_config(priv_dev);
> > +       }
> > +}
> > +
> > +/**
> > + * cdns3_device_irq_handler- interrupt handler for device part of controller
> > + *
> > + * @irq: irq number for cdns3 core device
> > + * @data: structure of cdns3
> > + *
> > + * Returns IRQ_HANDLED or IRQ_NONE
> > + */
> > +static irqreturn_t cdns3_device_irq_handler(int irq, void *data)
> > +{
> > +       struct cdns3_device *priv_dev;
> > +       struct cdns3 *cdns = data;
> > +       irqreturn_t ret = IRQ_NONE;
> > +       unsigned long flags;
> > +       u32 reg;
> > +
> > +       priv_dev = cdns->gadget_dev;
> > +       spin_lock_irqsave(&priv_dev->lock, flags);
> > +
> > +       /* check USB device interrupt */
> > +       reg = readl(&priv_dev->regs->usb_ists);
> > +       writel(reg, &priv_dev->regs->usb_ists);
> > +
> > +       if (reg) {
> > +               cdns3_check_usb_interrupt_proceed(priv_dev, reg);
> > +               ret = IRQ_HANDLED;
> > +       }
> > +
> > +       /* check endpoint interrupt */
> > +       reg = readl(&priv_dev->regs->ep_ists);
> > +
> > +       if (reg) {
> > +               reg = ~reg & readl(&priv_dev->regs->ep_ien);
> > +               /* mask deferred interrupt. */
> > +               writel(reg, &priv_dev->regs->ep_ien);
> > +               ret = IRQ_WAKE_THREAD;
> > +       }
> > +
> > +       spin_unlock_irqrestore(&priv_dev->lock, flags);
> > +       return ret;
> > +}
> > +
> > +/**
> > + * cdns3_device_thread_irq_handler- interrupt handler for device part
> > + * of controller
> > + *
> > + * @irq: irq number for cdns3 core device
> > + * @data: structure of cdns3
> > + *
> > + * Returns IRQ_HANDLED or IRQ_NONE
> > + */
> > +static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data)
> > +{
> > +       struct cdns3_device *priv_dev;
> > +       struct cdns3 *cdns = data;
> > +       irqreturn_t ret = IRQ_NONE;
> > +       unsigned long flags;
> > +       u32 ep_ien;
> > +       int bit;
> > +       u32 reg;
> > +
> > +       priv_dev = cdns->gadget_dev;
> > +       spin_lock_irqsave(&priv_dev->lock, flags);
> > +
> > +       reg = readl(&priv_dev->regs->ep_ists);
> > +       ep_ien = reg;
> > +
> > +       /* handle default endpoint OUT */
> > +       if (reg & EP_ISTS_EP_OUT0) {
> > +               cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT);
> > +               ret = IRQ_HANDLED;
> > +       }
> > +
> > +       /* handle default endpoint IN */
> > +       if (reg & EP_ISTS_EP_IN0) {
> > +               cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN);
> > +               ret = IRQ_HANDLED;
> > +       }
> > +
> > +       /* check if interrupt from non default endpoint, if no exit */
> > +       reg &= ~(EP_ISTS_EP_OUT0 | EP_ISTS_EP_IN0);
> > +       if (!reg)
> > +               goto irqend;
> > +
> > +       for_each_set_bit(bit, (unsigned long *)&reg,
> > +                        sizeof(u32) * BITS_PER_BYTE) {
> > +               cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]);
> > +               ret = IRQ_HANDLED;
> > +       }
> > +
> > +irqend:
> > +       ep_ien |= readl(&priv_dev->regs->ep_ien);
> > +       /* Unmask all handled EP interrupts */
> > +       writel(ep_ien, &priv_dev->regs->ep_ien);
> > +       spin_unlock_irqrestore(&priv_dev->lock, flags);
> > +       return ret;
> > +}
> > +
> > +/**
> > + * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP
> > + *
> > + * The real reservation will occur during write to EP_CFG register,
> > + * this function is used to check if the 'size' reservation is allowed.
> > + *
> > + * @priv_dev: extended gadget object
> > + * @size: the size (KB) for EP would like to allocate
> > + *
> > + * Return 0 if the required size can met or negative value on failure
> > + */
> > +static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev,
> > +                                         int size)
> > +{
> > +       u32 onchip_mem;
> > +
> > +       priv_dev->onchip_mem_allocated_size += size;
> > +
> > +       onchip_mem = USB_CAP2_ACTUAL_MEM_SIZE(readl(&priv_dev->regs->usb_cap2));
> > +       if (!onchip_mem)
> > +               onchip_mem = 256;
> > +
> > +       /* 2KB is reserved for EP0*/
> > +       onchip_mem -= 2;
> > +       if (priv_dev->onchip_mem_allocated_size > onchip_mem) {
> > +               priv_dev->onchip_mem_allocated_size -= size;
> > +               return -EPERM;
> > +       }
> > +
> > +       return 0;
> > +}
> > +
> > +/**
> > + * cdns3_ep_config Configure hardware endpoint
> > + * @priv_ep: extended endpoint object
> > + */
> > +void cdns3_ep_config(struct cdns3_endpoint *priv_ep)
> > +{
> > +       bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC);
> > +       struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
> > +       u32 bEndpointAddress = priv_ep->num | priv_ep->dir;
> > +       u32 max_packet_size = 0;
> > +       u32 ep_cfg = 0;
> > +       int ret;
> > +
> > +       switch (priv_ep->type) {
> > +       case USB_ENDPOINT_XFER_INT:
> > +               ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_INT);
> > +               break;
> > +       case USB_ENDPOINT_XFER_BULK:
> > +               ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_BULK);
> > +               break;
> > +       default:
> > +               ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC);
> > +       }
> > +
> > +       switch (priv_dev->gadget.speed) {
> > +       case USB_SPEED_FULL:
> > +               max_packet_size = is_iso_ep ? 1023 : 64;
> > +               break;
> > +       case USB_SPEED_HIGH:
> > +               max_packet_size = is_iso_ep ? 1024 : 512;
> > +               break;
> > +       case USB_SPEED_SUPER:
> > +               max_packet_size = 1024;
> > +               break;
> > +       default:
> > +               /* all other speed are not supported */
> > +               return;
> > +       }
> > +
> > +       ret = cdns3_ep_onchip_buffer_reserve(priv_dev, CDNS3_EP_BUF_SIZE);
> > +       if (ret) {
> > +               dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n");
> > +               return;
> > +       }
> > +
> > +       ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) |
> > +                 EP_CFG_BUFFERING(CDNS3_EP_BUF_SIZE - 1) |
> > +                 EP_CFG_MAXBURST(priv_ep->endpoint.maxburst);
> > +
> > +       cdns3_select_ep(priv_dev, bEndpointAddress);
> > +       writel(ep_cfg, &priv_dev->regs->ep_cfg);
> > +
> > +       dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n",
> > +               priv_ep->name, ep_cfg);
> > +}
> > +
> > +/* Find correct direction for HW endpoint according to description */
> > +static int cdns3_ep_dir_is_correct(struct usb_endpoint_descriptor *desc,
> > +                                  struct cdns3_endpoint *priv_ep)
> > +{
> > +       return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) ||
> > +              (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc));
> > +}
> > +
> > +static struct
> > +cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev,
> > +                                       struct usb_endpoint_descriptor *desc)
> > +{
> > +       struct usb_ep *ep;
> > +       struct cdns3_endpoint *priv_ep;
> > +
> > +       list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
> > +               unsigned long num;
> > +               int ret;
> > +               /* ep name pattern likes epXin or epXout */
> > +               char c[2] = {ep->name[2], '\0'};
> > +
> > +               ret = kstrtoul(c, 10, &num);
> > +               if (ret)
> > +                       return ERR_PTR(ret);
> > +
> > +               priv_ep = ep_to_cdns3_ep(ep);
> > +               if (cdns3_ep_dir_is_correct(desc, priv_ep)) {
> > +                       if (!(priv_ep->flags & EP_CLAIMED)) {
> > +                               priv_ep->num  = num;
> > +                               return priv_ep;
> > +                       }
> > +               }
> > +       }
> > +
> > +       return ERR_PTR(-ENOENT);
> > +}
> > +
> > +/*
> > + *  Cadence IP has one limitation that all endpoints must be configured
> > + * (Type & MaxPacketSize) before setting configuration through hardware
> > + * register, it means we can't change endpoints configuration after
> > + * set_configuration.
> > + *
> > + * This function set EP_CLAIMED flag which is added when the gadget driver
> > + * uses usb_ep_autoconfig to configure specific endpoint;
> > + * When the udc driver receives set_configurion request,
> > + * it goes through all claimed endpoints, and configure all endpoints
> > + * accordingly.
> > + *
> > + * At usb_ep_ops.enable/disable, we only enable and disable endpoint through
> > + * ep_cfg register which can be changed after set_configuration, and do
> > + * some software operation accordingly.
> > + */
> > +static struct
> > +usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget,
> > +                             struct usb_endpoint_descriptor *desc,
> > +                             struct usb_ss_ep_comp_descriptor *comp_desc)
> > +{
> > +       struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
> > +       struct cdns3_endpoint *priv_ep;
> > +       unsigned long flags;
> > +
> > +       priv_ep = cdns3_find_available_ep(priv_dev, desc);
> > +       if (IS_ERR(priv_ep)) {
> > +               dev_err(priv_dev->dev, "no available ep\n");
> > +               return NULL;
> > +       }
> > +
> > +       dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);
> > +
> > +       spin_lock_irqsave(&priv_dev->lock, flags);
> > +       priv_ep->endpoint.desc = desc;
> > +       priv_ep->dir  = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT;
> > +       priv_ep->type = usb_endpoint_type(desc);
> > +       priv_ep->flags |= EP_CLAIMED;
> > +       spin_unlock_irqrestore(&priv_dev->lock, flags);
> > +       return &priv_ep->endpoint;
> > +}
> > +
> > +/**
> > + * cdns3_gadget_ep_alloc_request Allocates request
> > + * @ep: endpoint object associated with request
> > + * @gfp_flags: gfp flags
> > + *
> > + * Returns allocated request address, NULL on allocation error
> > + */
> > +struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep,
> > +                                                 gfp_t gfp_flags)
> > +{
> > +       struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
> > +       struct cdns3_request *priv_req;
> > +
> > +       priv_req = kzalloc(sizeof(*priv_req), gfp_flags);
> > +       if (!priv_req)
> > +               return NULL;
> > +
> > +       priv_req->priv_ep = priv_ep;
> > +
> > +       trace_cdns3_alloc_request(priv_req);
> > +       return &priv_req->request;
> > +}
> > +
> > +/**
> > + * cdns3_gadget_ep_free_request Free memory occupied by request
> > + * @ep: endpoint object associated with request
> > + * @request: request to free memory
> > + */
> > +void cdns3_gadget_ep_free_request(struct usb_ep *ep,
> > +                                 struct usb_request *request)
> > +{
> > +       struct cdns3_request *priv_req = to_cdns3_request(request);
> > +
> > +       trace_cdns3_free_request(priv_req);
> > +       kfree(priv_req);
> > +}
> > +
> > +/**
> > + * cdns3_gadget_ep_enable Enable endpoint
> > + * @ep: endpoint object
> > + * @desc: endpoint descriptor
> > + *
> > + * Returns 0 on success, error code elsewhere
> > + */
> > +static int cdns3_gadget_ep_enable(struct usb_ep *ep,
> > +                                 const struct usb_endpoint_descriptor *desc)
> > +{
> > +       struct cdns3_endpoint *priv_ep;
> > +       struct cdns3_device *priv_dev;
> > +       u32 reg = EP_STS_EN_TRBERREN;
> > +       u32 bEndpointAddress;
> > +       unsigned long flags;
> > +       int ret;
> > +
> > +       priv_ep = ep_to_cdns3_ep(ep);
> > +       priv_dev = priv_ep->cdns3_dev;
> > +
> > +       if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
> > +               dev_dbg(priv_dev->dev, "usbss: invalid parameters\n");
> > +               return -EINVAL;
> > +       }
> > +
> > +       if (!desc->wMaxPacketSize) {
> > +               dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n");
> > +               return -EINVAL;
> > +       }
> > +
> > +       if (dev_WARN_ONCE(priv_dev->dev, priv_ep->flags & EP_ENABLED,
> > +                         "%s is already enabled\n", priv_ep->name))
> > +               return 0;
> > +
> > +       spin_lock_irqsave(&priv_dev->lock, flags);
> > +
> > +       priv_ep->endpoint.desc = desc;
> > +       priv_ep->type = usb_endpoint_type(desc);
> > +       priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
> > +
> > +       if (priv_ep->interval > ISO_MAX_INTERVAL &&
> > +           priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
> > +               dev_err(priv_dev->dev, "Driver is limited to %d period\n",
> > +                       ISO_MAX_INTERVAL);
> > +
> > +               ret =  -EINVAL;
> > +               goto exit;
> > +       }
> > +
> > +       ret = cdns3_allocate_trb_pool(priv_ep);
> > +
> > +       if (ret)
> > +               goto exit;
> > +
> > +       bEndpointAddress = priv_ep->num | priv_ep->dir;
> > +       cdns3_select_ep(priv_dev, bEndpointAddress);
> > +
> > +       trace_cdns3_gadget_ep_enable(priv_ep);
> > +
> > +       writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
> > +
> > +       ret = cdns3_handshake(&priv_dev->regs->ep_cmd,
> > +                             EP_CMD_CSTALL | EP_CMD_EPRST, 0, 100);
> > +
> > +       /* enable interrupt for selected endpoint */
> > +       cdns3_set_register_bit(&priv_dev->regs->ep_ien,
> > +                              BIT(cdns3_ep_addr_to_index(bEndpointAddress)));
> > +       /*
> > +        * WA1: Set flag for all not ISOC OUT endpoints. If this flag is set
> > +        * driver try to detect whether endpoint need additional internal
> > +        * buffer for unblocking on-chip FIFO buffer. This flag will be cleared
> > +        * if before first DESCMISS interrupt the DMA will be armed.
> > +        */
> > +       if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
> > +               priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET;
> > +               reg |= EP_STS_EN_DESCMISEN;
> > +       }
> > +
> > +       writel(reg, &priv_dev->regs->ep_sts_en);
> > +
> > +       cdns3_set_register_bit(&priv_dev->regs->ep_cfg, EP_CFG_ENABLE);
> > +
> > +       ep->desc = desc;
> > +       priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALL |
> > +                           EP_QUIRK_EXTRA_BUF_EN);
> > +       priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR;
> > +       priv_ep->enqueue = 0;
> > +       priv_ep->dequeue = 0;
> > +       reg = readl(&priv_dev->regs->ep_sts);
> > +       priv_ep->pcs = !!EP_STS_CCS(reg);
> > +       priv_ep->ccs = !!EP_STS_CCS(reg);
> > +       /* one TRB is reserved for link TRB used in DMULT mode*/
> > +       priv_ep->free_trbs = priv_ep->num_trbs - 1;
> > +exit:
> > +       spin_unlock_irqrestore(&priv_dev->lock, flags);
> > +
> > +       return ret;
> > +}
> > +
> > +/**
> > + * cdns3_gadget_ep_disable Disable endpoint
> > + * @ep: endpoint object
> > + *
> > + * Returns 0 on success, error code elsewhere
> > + */
> > +static int cdns3_gadget_ep_disable(struct usb_ep *ep)
> > +{
> > +       struct cdns3_endpoint *priv_ep;
> > +       struct cdns3_request *priv_req;
> > +       struct cdns3_device *priv_dev;
> > +       struct usb_request *request;
> > +       unsigned long flags;
> > +       int ret = 0;
> > +       u32 ep_cfg;
> > +
> > +       if (!ep) {
> > +               pr_err("usbss: invalid parameters\n");
> > +               return -EINVAL;
> > +       }
> > +
> > +       priv_ep = ep_to_cdns3_ep(ep);
> > +       priv_dev = priv_ep->cdns3_dev;
> > +
> > +       if (dev_WARN_ONCE(priv_dev->dev, !(priv_ep->flags & EP_ENABLED),
> > +                         "%s is already disabled\n", priv_ep->name))
> > +               return 0;
> > +
> > +       spin_lock_irqsave(&priv_dev->lock, flags);
> > +
> > +       trace_cdns3_gadget_ep_disable(priv_ep);
> > +
> > +       cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
> > +       ret = cdns3_data_flush(priv_ep);
> > +
> > +       ep_cfg = readl(&priv_dev->regs->ep_cfg);
> > +       ep_cfg &= ~EP_CFG_ENABLE;
> > +       writel(ep_cfg, &priv_dev->regs->ep_cfg);
> > +
> > +       while (!list_empty(&priv_ep->pending_req_list)) {
> > +               request = cdns3_next_request(&priv_ep->pending_req_list);
> > +
> > +               cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
> > +                                     -ESHUTDOWN);
> > +       }
> > +
> > +       while (!list_empty(&priv_ep->descmiss_req_list)) {
> > +               priv_req = cdns3_next_priv_request(&priv_ep->descmiss_req_list);
> > +
> > +               kfree(priv_req->request.buf);
> > +               cdns3_gadget_ep_free_request(&priv_ep->endpoint,
> > +                                            &priv_req->request);
> > +       }
> > +
> > +       while (!list_empty(&priv_ep->deferred_req_list)) {
> > +               request = cdns3_next_request(&priv_ep->deferred_req_list);
> > +
> > +               cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
> > +                                     -ESHUTDOWN);
> > +       }
> > +
> > +       priv_ep->descmis_req = NULL;
> > +
> > +       ep->desc = NULL;
> > +       priv_ep->flags &= ~EP_ENABLED;
> > +
> > +       spin_unlock_irqrestore(&priv_dev->lock, flags);
> > +
> > +       return ret;
> > +}
> > +
> > +/**
> > + * cdns3_gadget_ep_queue Transfer data on endpoint
> > + * @ep: endpoint object
> > + * @request: request object
> > + * @gfp_flags: gfp flags
> > + *
> > + * Returns 0 on success, error code elsewhere
> > + */
> > +static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
> > +                                  struct usb_request *request,
> > +                                  gfp_t gfp_flags)
> > +{
> > +       struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
> > +       struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
> > +       struct cdns3_request *priv_req;
> > +       int deferred = 0;
> > +       int ret = 0;
> > +
> > +       request->actual = 0;
> > +       request->status = -EINPROGRESS;
> > +       priv_req = to_cdns3_request(request);
> > +       trace_cdns3_ep_queue(priv_req);
> > +
> > +       /*
> > +        * WA1: if transfer was queued before DESCMISS appear than we
> > +        * can disable handling of DESCMISS interrupt. Driver assumes that it
> > +        * can disable special treatment for this endpoint.
> > +        */
> > +       if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
> > +               u32 reg;
> > +
> > +               cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir);
> > +               reg = readl(&priv_dev->regs->ep_sts_en);
> > +               priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
> > +               reg &= EP_STS_EN_DESCMISEN;
> > +               writel(reg, &priv_dev->regs->ep_sts_en);
> > +       }
> > +
> > +       /* WA1 */
> > +       if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
> > +               u8 pending_empty = list_empty(&priv_ep->pending_req_list);
> > +               u8 descmiss_empty = list_empty(&priv_ep->descmiss_req_list);
> > +
> > +               /*
> > +                *  DESCMISS transfer has been finished, so data will be
> > +                *  directly copied from internal allocated usb_request
> > +                *  objects.
> > +                */
> > +               if (pending_empty && !descmiss_empty &&
> > +                   !(priv_req->flags & REQUEST_INTERNAL)) {
> > +                       cdns3_descmiss_copy_data(priv_ep, request);
> > +                       list_add_tail(&request->list,
> > +                                     &priv_ep->pending_req_list);
> > +                       cdns3_gadget_giveback(priv_ep, priv_req,
> > +                                             request->status);
> > +                       return ret;
> > +               }
> > +
> > +               /*
> > +                * WA1 driver will wait for completion DESCMISS transfer,
> > +                * before starts new, not DESCMISS transfer.
> > +                */
> > +               if (!pending_empty && !descmiss_empty)
> > +                       deferred = 1;
> > +
> > +               if (priv_req->flags & REQUEST_INTERNAL)
> > +                       list_add_tail(&priv_req->list,
> > +                                     &priv_ep->descmiss_req_list);
> > +       }
> > +
> > +       ret = usb_gadget_map_request_by_dev(priv_dev->sysdev, request,
> > +                                           usb_endpoint_dir_in(ep->desc));
> > +       if (ret)
> > +               return ret;
> > +
> > +       /*
> > +        * If hardware endpoint configuration has not been set yet then
> > +        * just queue request in deferred list. Transfer will be started in
> > +        * cdns3_set_hw_configuration.
> > +        */
> > +       if (!priv_dev->hw_configured_flag)
> > +               deferred = 1;
> > +       else
> > +               ret = cdns3_ep_run_transfer(priv_ep, request);
> > +
> > +       if (ret || deferred)
> > +               list_add_tail(&request->list, &priv_ep->deferred_req_list);
> > +       else
> > +               list_add_tail(&request->list, &priv_ep->pending_req_list);
> > +
> > +       return ret;
> > +}
> > +
> > +static int cdns3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
> > +                                gfp_t gfp_flags)
> > +{
> > +       struct usb_request *zlp_request;
> > +       struct cdns3_endpoint *priv_ep;
> > +       struct cdns3_device *priv_dev;
> > +       unsigned long flags;
> > +       int ret;
> > +
> > +       if (!request || !ep)
> > +               return -EINVAL;
> > +
> > +       priv_ep = ep_to_cdns3_ep(ep);
> > +       priv_dev = priv_ep->cdns3_dev;
> > +
> > +       spin_lock_irqsave(&priv_dev->lock, flags);
> > +
> > +       ret = __cdns3_gadget_ep_queue(ep, request, gfp_flags);
> > +
> > +       if (ret == 0 && request->zero && request->length &&
> > +           (request->length % ep->maxpacket == 0)) {
> > +               struct cdns3_request *priv_req;
> > +
> > +               zlp_request = cdns3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
> > +               zlp_request->buf = priv_dev->zlp_buf;
> > +               zlp_request->length = 0;
> > +
> > +               priv_req = to_cdns3_request(zlp_request);
> > +               priv_req->flags |= REQUEST_ZLP;
> > +
> > +               dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n",
> > +                       priv_ep->name);
> > +               ret = __cdns3_gadget_ep_queue(ep, zlp_request, gfp_flags);
> > +       }
> > +
> > +       spin_unlock_irqrestore(&priv_dev->lock, flags);
> > +       return ret;
> > +}
> > +
> > +/**
> > + * cdns3_gadget_ep_dequeue Remove request from transfer queue
> > + * @ep: endpoint object associated with request
> > + * @request: request object
> > + *
> > + * Returns 0 on success, error code elsewhere
> > + */
> > +int cdns3_gadget_ep_dequeue(struct usb_ep *ep,
> > +                           struct usb_request *request)
> > +{
> > +       struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
> > +       struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
> > +       struct usb_request *req, *req_temp;
> > +       struct cdns3_request *priv_req;
> > +       unsigned long flags;
> > +       int ret = 0;
> > +
> > +       if (!ep || !request || !ep->desc)
> > +               return -EINVAL;
> > +
> > +       spin_lock_irqsave(&priv_dev->lock, flags);
> > +
> > +       priv_req = to_cdns3_request(request);
> > +
> > +       trace_cdns3_ep_dequeue(priv_req);
> > +
> > +       cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
> > +
> > +       list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list,
> > +                                list) {
> > +               if (request == req)
> > +                       goto found;
> > +       }
> > +
> > +       list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list,
> > +                                list) {
> > +               if (request == req)
> > +                       goto found;
> > +       }
> > +
> > +found:
> > +       if (request == req)
> > +               cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);
> > +
> > +       spin_unlock_irqrestore(&priv_dev->lock, flags);
> > +       return ret;
> > +}
> > +
> > +/**
> > + * cdns3_gadget_ep_set_halt Sets/clears stall on selected endpoint
> > + * @ep: endpoint object to set/clear stall on
> > + * @value: 1 for set stall, 0 for clear stall
> > + *
> > + * Returns 0 on success, error code elsewhere
> > + */
> > +int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value)
> > +{
> > +       struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
> > +       struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
> > +       unsigned long flags;
> > +       int ret = 0;
> > +
> > +       if (!(priv_ep->flags & EP_ENABLED))
> > +               return -EPERM;
> > +
> > +       spin_lock_irqsave(&priv_dev->lock, flags);
> > +
> > +       /* if actual transfer is pending defer setting stall on this endpoint */
> > +       if ((priv_ep->flags & EP_PENDING_REQUEST) && value) {
> > +               priv_ep->flags |= EP_STALL;
> > +               goto finish;
> > +       }
> > +
> > +       dev_dbg(priv_dev->dev, "Halt endpoint %s\n", priv_ep->name);
> > +
> > +       cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
> > +       if (value) {
> > +               cdns3_ep_stall_flush(priv_ep);
> > +       } else {
> > +               priv_ep->flags &= ~EP_WEDGE;
> > +               writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
> > +
> > +               /* wait for EPRST cleared */
> > +               ret = cdns3_handshake(&priv_dev->regs->ep_cmd,
> > +                                     EP_CMD_EPRST, 0, 100);
> > +               if (unlikely(ret)) {
> > +                       dev_err(priv_dev->dev,
> > +                               "Clearing halt condition failed for %s\n",
> > +                               priv_ep->name);
> > +                       goto finish;
> > +
> > +               } else {
> > +                       priv_ep->flags &= ~EP_STALL;
> > +               }
> > +       }
> > +
> > +       priv_ep->flags &= ~EP_PENDING_REQUEST;
> > +finish:
> > +       spin_unlock_irqrestore(&priv_dev->lock, flags);
> > +
> > +       return ret;
> > +}
> > +
> > +extern const struct usb_ep_ops cdns3_gadget_ep0_ops;
> > +
> > +static const struct usb_ep_ops cdns3_gadget_ep_ops = {
> > +       .enable = cdns3_gadget_ep_enable,
> > +       .disable = cdns3_gadget_ep_disable,
> > +       .alloc_request = cdns3_gadget_ep_alloc_request,
> > +       .free_request = cdns3_gadget_ep_free_request,
> > +       .queue = cdns3_gadget_ep_queue,
> > +       .dequeue = cdns3_gadget_ep_dequeue,
> > +       .set_halt = cdns3_gadget_ep_set_halt,
> > +       .set_wedge = cdns3_gadget_ep_set_wedge,
> > +};
> > +
> > +/**
> > + * cdns3_gadget_get_frame Returns number of actual ITP frame
> > + * @gadget: gadget object
> > + *
> > + * Returns number of actual ITP frame
> > + */
> > +static int cdns3_gadget_get_frame(struct usb_gadget *gadget)
> > +{
> > +       struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
> > +
> > +       return readl(&priv_dev->regs->usb_iptn);
> > +}
> > +
> > +static int cdns3_gadget_wakeup(struct usb_gadget *gadget)
> > +{
> > +       return 0;
> > +}
> > +
> > +static int cdns3_gadget_set_selfpowered(struct usb_gadget *gadget,
> > +                                       int is_selfpowered)
> > +{
> > +       struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
> > +       unsigned long flags;
> > +
> > +       spin_lock_irqsave(&priv_dev->lock, flags);
> > +       priv_dev->is_selfpowered = !!is_selfpowered;
> > +       spin_unlock_irqrestore(&priv_dev->lock, flags);
> > +       return 0;
> > +}
> > +
> > +static int cdns3_gadget_pullup(struct usb_gadget *gadget, int is_on)
> > +{
> > +       struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
> > +
> > +       if (is_on)
> > +               writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
> > +       else
> > +               writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
> > +
> > +       return 0;
> > +}
> > +
> > +static void cdns3_gadget_config(struct cdns3_device *priv_dev)
> > +{
> > +       struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
> > +
> > +       cdns3_ep0_config(priv_dev);
> > +
> > +       /* enable interrupts for endpoint 0 (in and out) */
> > +       writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, &regs->ep_ien);
> > +
> > +       /* enable generic interrupt*/
> > +       writel(USB_IEN_INIT, &regs->usb_ien);
> > +       writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, &regs->usb_conf);
> > +       writel(USB_CONF_DMULT, &regs->usb_conf);
> > +       cdns3_gadget_pullup(&priv_dev->gadget, 1);
> > +}
> > +
> > +/**
> > + * cdns3_gadget_udc_start Gadget start
> > + * @gadget: gadget object
> > + * @driver: driver which operates on this gadget
> > + *
> > + * Returns 0 on success, error code elsewhere
> > + */
> > +static int cdns3_gadget_udc_start(struct usb_gadget *gadget,
> > +                                 struct usb_gadget_driver *driver)
> > +{
> > +       struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
> > +       unsigned long flags;
> > +
> > +       spin_lock_irqsave(&priv_dev->lock, flags);
> > +       priv_dev->gadget_driver = driver;
> > +       cdns3_gadget_config(priv_dev);
> > +       spin_unlock_irqrestore(&priv_dev->lock, flags);
> > +       return 0;
> > +}
> > +
> > +/**
> > + * cdns3_gadget_udc_stop Stops gadget
> > + * @gadget: gadget object
> > + *
> > + * Returns 0
> > + */
> > +static int cdns3_gadget_udc_stop(struct usb_gadget *gadget)
> > +{
> > +       struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
> > +       struct cdns3_endpoint *priv_ep;
> > +       u32 bEndpointAddress;
> > +       struct usb_ep *ep;
> > +       int ret = 0;
> > +
> > +       priv_dev->gadget_driver = NULL;
> > +
> > +       priv_dev->onchip_mem_allocated_size = 0;
> > +       priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
> > +
> > +       list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
> > +               priv_ep = ep_to_cdns3_ep(ep);
> > +               bEndpointAddress = priv_ep->num | priv_ep->dir;
> > +               cdns3_select_ep(priv_dev, bEndpointAddress);
> > +               writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
> > +               ret = cdns3_handshake(&priv_dev->regs->ep_cmd,
> > +                                     EP_CMD_EPRST, 0, 100);
> > +               cdns3_free_trb_pool(priv_ep);
> > +       }
> > +
> > +       /* disable interrupt for device */
> > +       writel(0, &priv_dev->regs->usb_ien);
> > +       writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
> > +
> > +       return ret;
> > +}
> > +
> > +static const struct usb_gadget_ops cdns3_gadget_ops = {
> > +       .get_frame = cdns3_gadget_get_frame,
> > +       .wakeup = cdns3_gadget_wakeup,
> > +       .set_selfpowered = cdns3_gadget_set_selfpowered,
> > +       .pullup = cdns3_gadget_pullup,
> > +       .udc_start = cdns3_gadget_udc_start,
> > +       .udc_stop = cdns3_gadget_udc_stop,
> > +       .match_ep = cdns3_gadget_match_ep,
> > +};
> > +
> > +static void cdns3_free_all_eps(struct cdns3_device *priv_dev)
> > +{
> > +       int i;
> > +
> > +       /*ep0 OUT point to ep0 IN*/
> > +       priv_dev->eps[16] = NULL;
> > +
> > +       cdns3_free_trb_pool(priv_dev->eps[0]);
> > +
> > +       for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
> > +               if (priv_dev->eps[i])
> > +                       devm_kfree(priv_dev->dev, priv_dev->eps[i]);
> > +}
> > +
> > +/**
> > + * cdns3_init_eps Initializes software endpoints of gadget
> > + * @cdns3: extended gadget object
> > + *
> > + * Returns 0 on success, error code elsewhere
> > + */
> > +static int cdns3_init_eps(struct cdns3_device *priv_dev)
> > +{
> > +       u32 ep_enabled_reg, iso_ep_reg;
> > +       struct cdns3_endpoint *priv_ep;
> > +       int ep_dir, ep_number;
> > +       u32 ep_mask;
> > +       int ret = 0;
> > +       int i;
> > +
> > +       /* Read it from USB_CAP3 to USB_CAP5 */
> > +       ep_enabled_reg = readl(&priv_dev->regs->usb_cap3);
> > +       iso_ep_reg = readl(&priv_dev->regs->usb_cap4);
> > +
> > +       dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n");
> > +
> > +       for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) {
> > +               ep_dir = i >> 4;        /* i div 16 */
> > +               ep_number = i & 0xF;    /* i % 16 */
> > +               ep_mask = BIT(i);
> > +
> > +               if (!(ep_enabled_reg & ep_mask))
> > +                       continue;
> > +
> > +               if (ep_dir && !ep_number) {
> > +                       priv_dev->eps[i] = priv_dev->eps[0];
> > +                       continue;
> > +               }
> > +
> > +               priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),
> > +                                      GFP_KERNEL);
> > +               if (!priv_ep) {
> > +                       ret = -ENOMEM;
> > +                       goto err;
> > +               }
> > +
> > +               /* set parent of endpoint object */
> > +               priv_ep->cdns3_dev = priv_dev;
> > +               priv_dev->eps[i] = priv_ep;
> > +               priv_ep->num = ep_number;
> > +               priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT;
> > +
> > +               if (!ep_number) {
> > +                       ret = cdns3_init_ep0(priv_dev, priv_ep);
> > +                       if (ret) {
> > +                               dev_err(priv_dev->dev, "Failed to init ep0\n");
> > +                               goto err;
> > +                       }
> > +               } else {
> > +                       snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s",
> > +                                ep_number, !!ep_dir ? "in" : "out");
> > +                       priv_ep->endpoint.name = priv_ep->name;
> > +
> > +                       usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
> > +                                                  CDNS3_EP_MAX_PACKET_LIMIT);
> > +                       priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS;
> > +                       priv_ep->endpoint.ops = &cdns3_gadget_ep_ops;
> > +                       if (ep_dir)
> > +                               priv_ep->endpoint.caps.dir_in = 1;
> > +                       else
> > +                               priv_ep->endpoint.caps.dir_out = 1;
> > +
> > +                       if (iso_ep_reg & ep_mask)
> > +                               priv_ep->endpoint.caps.type_iso = 1;
> > +
> > +                       priv_ep->endpoint.caps.type_bulk = 1;
> > +                       priv_ep->endpoint.caps.type_int = 1;
> > +                       priv_ep->endpoint.maxburst = CDNS3_EP_BUF_SIZE - 1;
> > +
> > +                       list_add_tail(&priv_ep->endpoint.ep_list,
> > +                                     &priv_dev->gadget.ep_list);
> > +               }
> > +
> > +               priv_ep->flags = 0;
> > +
> > +               dev_info(priv_dev->dev, "Initialized  %s support: %s %s\n",
> > +                        priv_ep->name,
> > +                        priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "",
> > +                        priv_ep->endpoint.caps.type_iso ? "ISO" : "");
> > +
> > +               INIT_LIST_HEAD(&priv_ep->pending_req_list);
> > +               INIT_LIST_HEAD(&priv_ep->deferred_req_list);
> > +               INIT_LIST_HEAD(&priv_ep->descmiss_req_list);
> > +       }
> > +
> > +       return 0;
> > +err:
> > +       cdns3_free_all_eps(priv_dev);
> > +       return -ENOMEM;
> > +}
> > +
> > +static void cdns3_gadget_disable(struct cdns3 *cdns)
> > +{
> > +       struct cdns3_device *priv_dev;
> > +
> > +       priv_dev = cdns->gadget_dev;
> > +
> > +       if (priv_dev->gadget_driver)
> > +               priv_dev->gadget_driver->disconnect(&priv_dev->gadget);
> > +
> > +       usb_gadget_disconnect(&priv_dev->gadget);
> > +       priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
> > +}
> > +
> > +void cdns3_gadget_exit(struct cdns3 *cdns)
> > +{
> > +       struct cdns3_device *priv_dev;
> > +
> > +       priv_dev = cdns->gadget_dev;
> > +
> > +       cdns3_gadget_disable(cdns);
> > +
> > +       devm_free_irq(cdns->dev, cdns->irq, cdns);
> > +
> > +       pm_runtime_mark_last_busy(cdns->dev);
> > +       pm_runtime_put_autosuspend(cdns->dev);
> > +
> > +       usb_del_gadget_udc(&priv_dev->gadget);
> > +
> > +       cdns3_free_all_eps(priv_dev);
> > +
> > +       dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
> > +                         priv_dev->setup_dma);
> > +
> > +       kfree(priv_dev->zlp_buf);
> > +       kfree(priv_dev);
> > +       cdns->gadget_dev = NULL;
> > +}
> > +
> > +static int cdns3_gadget_start(struct cdns3 *cdns)
> > +{
> > +       struct cdns3_device *priv_dev;
> > +       u32 max_speed;
> > +       int ret;
> > +
> > +       priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL);
> > +       if (!priv_dev)
> > +               return -ENOMEM;
> > +
> > +       cdns->gadget_dev = priv_dev;
> > +       priv_dev->sysdev = cdns->dev;
> > +       priv_dev->dev = cdns->dev;
> > +       priv_dev->regs = cdns->dev_regs;
> > +
> > +       max_speed = usb_get_maximum_speed(cdns->dev);
> > +
> > +       /* Check the maximum_speed parameter */
> > +       switch (max_speed) {
> > +       case USB_SPEED_FULL:
> > +       case USB_SPEED_HIGH:
> > +       case USB_SPEED_SUPER:
> > +               break;
> > +       default:
> > +               dev_err(cdns->dev, "invalid maximum_speed parameter %d\n",
> > +                       max_speed);
> > +               /* fall through */
> > +       case USB_SPEED_UNKNOWN:
> > +               /* default to superspeed */
> > +               max_speed = USB_SPEED_SUPER;
> > +               break;
> > +       }
> > +
> > +       /* fill gadget fields */
> > +       priv_dev->gadget.max_speed = max_speed;
> > +       priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
> > +       priv_dev->gadget.ops = &cdns3_gadget_ops;
> > +       priv_dev->gadget.name = "usb-ss-gadget";
> > +       priv_dev->gadget.sg_supported = 1;
> > +
> > +       spin_lock_init(&priv_dev->lock);
> > +       INIT_WORK(&priv_dev->pending_status_wq,
> > +                 cdns3_pending_setup_status_handler);
> > +
> > +       /* initialize endpoint container */
> > +       INIT_LIST_HEAD(&priv_dev->gadget.ep_list);
> > +
> > +       ret = cdns3_init_eps(priv_dev);
> > +       if (ret) {
> > +               dev_err(priv_dev->dev, "Failed to create endpoints\n");
> > +               goto err1;
> > +       }
> > +
> > +       /* allocate memory for setup packet buffer */
> > +       priv_dev->setup_buf = dma_alloc_coherent(priv_dev->sysdev, 8,
> > +                                                &priv_dev->setup_dma, GFP_DMA);
> > +       if (!priv_dev->setup_buf) {
> > +               dev_err(priv_dev->dev, "Failed to allocate memory for SETUP buffer\n");
> > +               ret = -ENOMEM;
> > +               goto err2;
> > +       }
> > +
> > +       priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6);
> > +       dev_dbg(priv_dev->dev, "Device Controller version: %08x\n",
> > +               readl(&priv_dev->regs->usb_cap6));
> > +       dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n",
> > +               readl(&priv_dev->regs->usb_cap1));
> > +       dev_dbg(priv_dev->dev, "On-Chip memory cnfiguration: %08x\n",
> > +               readl(&priv_dev->regs->usb_cap2));
> > +
> > +       priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL);
> > +       if (!priv_dev->zlp_buf) {
> > +               ret = -ENOMEM;
> > +               goto err3;
> > +       }
> > +
> > +       /* add USB gadget device */
> > +       ret = usb_add_gadget_udc(priv_dev->dev, &priv_dev->gadget);
> > +       if (ret < 0) {
> > +               dev_err(priv_dev->dev,
> > +                       "Failed to register USB device controller\n");
> > +               goto err4;
> > +       }
> > +
> > +       return 0;
> > +err4:
> > +       kfree(priv_dev->zlp_buf);
> > +err3:
> > +       dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
> > +                         priv_dev->setup_dma);
> > +err2:
> > +       cdns3_free_all_eps(priv_dev);
> > +err1:
> > +       cdns->gadget_dev = NULL;
> > +       return ret;
> > +}
> > +
> > +static int __cdns3_gadget_init(struct cdns3 *cdns)
> > +{
> > +       struct cdns3_device *priv_dev;
> > +       unsigned long flags;
> > +       int ret = 0;
> > +
> > +       ret = cdns3_gadget_start(cdns);
> > +       if (ret)
> > +               return ret;
> > +
> > +       priv_dev = cdns->gadget_dev;
> > +       ret = devm_request_threaded_irq(cdns->dev, cdns->irq,
> > +                                       cdns3_device_irq_handler,
> > +                                       cdns3_device_thread_irq_handler,
> > +                                       IRQF_SHARED, dev_name(cdns->dev), cdns);
> > +       if (ret)
> > +               goto err0;
> > +
> > +       pm_runtime_get_sync(cdns->dev);
> > +       spin_lock_irqsave(&priv_dev->lock, flags);
> > +       spin_unlock_irqrestore(&priv_dev->lock, flags);
> > +       return 0;
> > +err0:
> > +       cdns3_gadget_exit(cdns);
> > +       return ret;
> > +}
> > +
> > +static int cdns3_gadget_suspend(struct cdns3 *cdns, bool do_wakeup)
> > +{
> > +       cdns3_gadget_disable(cdns);
> > +       return 0;
> > +}
> > +
> > +static int cdns3_gadget_resume(struct cdns3 *cdns, bool hibernated)
> > +{
> > +       struct cdns3_device *priv_dev;
> > +       unsigned long flags;
> > +
> > +       priv_dev = cdns->gadget_dev;
> > +       spin_lock_irqsave(&priv_dev->lock, flags);
> > +
> > +       if (!priv_dev->gadget_driver) {
> > +               spin_unlock_irqrestore(&priv_dev->lock, flags);
> > +               return 0;
> > +       }
> > +
> > +       cdns3_gadget_config(priv_dev);
> > +       spin_unlock_irqrestore(&priv_dev->lock, flags);
> > +       return 0;
> > +}
> > +
> > +/**
> > + * cdns3_gadget_init - initialize device structure
> > + *
> > + * cdns: cdns3 instance
> > + *
> > + * This function initializes the gadget.
> > + */
> > +int cdns3_gadget_init(struct cdns3 *cdns)
> > +{
> > +       struct cdns3_role_driver *rdrv;
> > +
> > +       rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
> > +       if (!rdrv)
> > +               return -ENOMEM;
> > +
> > +       rdrv->start     = __cdns3_gadget_init;
> > +       rdrv->stop      = cdns3_gadget_exit;
> > +       rdrv->suspend   = cdns3_gadget_suspend;
> > +       rdrv->resume    = cdns3_gadget_resume;
> > +       rdrv->state     = CDNS3_ROLE_STATE_INACTIVE;
> > +       rdrv->name      = "gadget";
> > +       cdns->roles[CDNS3_ROLE_GADGET] = rdrv;
> > +
> > +       return 0;
> > +}
> > diff --git a/drivers/usb/cdns3/gadget.h b/drivers/usb/cdns3/gadget.h
> > new file mode 100644
> > index 000000000000..41cec7f085ad
> > --- /dev/null
> > +++ b/drivers/usb/cdns3/gadget.h
> > @@ -0,0 +1,1206 @@
> > +/* SPDX-License-Identifier: GPL-2.0 */
> > +/*
> > + * USBSS device controller driver header file
> > + *
> > + * Copyright (C) 2018 Cadence.
> > + * Copyright (C) 2017-2018 NXP
> > + *
> > + * Author: Pawel Laszczak <pawell@xxxxxxxxxxx>
> > + *         Pawel Jez <pjez@xxxxxxxxxxx>
> > + *         Peter Chen <peter.chen@xxxxxxx>
> > + */
> > +#ifndef __LINUX_CDNS3_GADGET
> > +#define __LINUX_CDNS3_GADGET
> > +#include <linux/usb/gadget.h>
> > +
> > +/*
> > + * USBSS-DEV register interface.
> > + * This corresponds to the USBSS Device Controller Interface
> > + */
> > +
> > +/**
> > + * struct cdns3_usb_regs - device controller registers.
> > + * @usb_conf:      Global Configuration Register.
> > + * @usb_sts:       Global Status Register.
> > + * @usb_cmd:       Global Command Register.
> > + * @usb_iptn:      ITP/SOF number Register.
> > + * @usb_lpm:       Global Command Register.
> > + * @usb_ien:       USB Interrupt Enable Register.
> > + * @usb_ists:      USB Interrupt Status Register.
> > + * @ep_sel:        Endpoint Select Register.
> > + * @ep_traddr:     Endpoint Transfer Ring Address Register.
> > + * @ep_cfg:        Endpoint Configuration Register.
> > + * @ep_cmd:        Endpoint Command Register.
> > + * @ep_sts:        Endpoint Status Register.
> > + * @ep_sts_sid:    Endpoint Status Register.
> > + * @ep_sts_en:     Endpoint Status Register Enable.
> > + * @drbl:          Doorbell Register.
> > + * @ep_ien:        EP Interrupt Enable Register.
> > + * @ep_ists:       EP Interrupt Status Register.
> > + * @usb_pwr:       Global Power Configuration Register.
> > + * @usb_conf2:     Global Configuration Register 2.
> > + * @usb_cap1:      Capability Register 1.
> > + * @usb_cap2:      Capability Register 2.
> > + * @usb_cap3:      Capability Register 3.
> > + * @usb_cap4:      Capability Register 4.
> > + * @usb_cap5:      Capability Register 5.
> > + * @usb_cap6:      Capability Register 6.
> > + * @usb_cpkt1:     Custom Packet Register 1.
> > + * @usb_cpkt2:     Custom Packet Register 2.
> > + * @usb_cpkt3:     Custom Packet Register 3.
> > + * @reserved1:     Reserved.
> > + * @cfg_regs:      Configuration registers.
> > + * @reserved2:     Reserved.
> > + * @dma_axi_ctrl:  AXI Control register.
> > + * @dma_axi_id:    AXI ID register.
> > + * @dma_axi_cap:   AXI Capability register.
> > + * @dma_axi_ctrl0: AXI Control 0 register.
> > + * @dma_axi_ctrl1: AXI Control 1 register.
> > + */
> > +struct cdns3_usb_regs {
> > +       __le32 usb_conf;
> > +       __le32 usb_sts;
> > +       __le32 usb_cmd;
> > +       __le32 usb_iptn;
> > +       __le32 usb_lpm;
> > +       __le32 usb_ien;
> > +       __le32 usb_ists;
> > +       __le32 ep_sel;
> > +       __le32 ep_traddr;
> > +       __le32 ep_cfg;
> > +       __le32 ep_cmd;
> > +       __le32 ep_sts;
> > +       __le32 ep_sts_sid;
> > +       __le32 ep_sts_en;
> > +       __le32 drbl;
> > +       __le32 ep_ien;
> > +       __le32 ep_ists;
> > +       __le32 usb_pwr;
> > +       __le32 usb_conf2;
> > +       __le32 usb_cap1;
> > +       __le32 usb_cap2;
> > +       __le32 usb_cap3;
> > +       __le32 usb_cap4;
> > +       __le32 usb_cap5;
> > +       __le32 usb_cap6;
> > +       __le32 usb_cpkt1;
> > +       __le32 usb_cpkt2;
> > +       __le32 usb_cpkt3;
> > +       __le32 reserved1[36];
> > +       __le32 cfg_reg1;
> > +       __le32 dbg_link1;
> > +       __le32 dbg_link2;
> > +       __le32 cfg_regs[74];
> > +       __le32 reserved2[34];
> > +       __le32 dma_axi_ctrl;
> > +       __le32 dma_axi_id;
> > +       __le32 dma_axi_cap;
> > +       __le32 dma_axi_ctrl0;
> > +       __le32 dma_axi_ctrl1;
> > +};
> > +
> > +/* USB_CONF - bitmasks */
> > +/* Reset USB device configuration. */
> > +#define USB_CONF_CFGRST                BIT(0)
> > +/* Set Configuration. */
> > +#define USB_CONF_CFGSET                BIT(1)
> > +/* Disconnect USB device in SuperSpeed. */
> > +#define USB_CONF_USB3DIS       BIT(3)
> > +/* Disconnect USB device in HS/FS */
> > +#define USB_CONF_USB2DIS       BIT(4)
> > +/* Little Endian access - default */
> > +#define USB_CONF_LENDIAN       BIT(5)
> > +/*
> > + * Big Endian access. Driver assume that byte order for
> > + * SFRs access always is as Little Endian so this bit
> > + * is not used.
> > + */
> > +#define USB_CONF_BENDIAN       BIT(6)
> > +/* Device software reset. */
> > +#define USB_CONF_SWRST         BIT(7)
> > +/* Singular DMA transfer mode. */
> > +#define USB_CONF_DSING         BIT(8)
> > +/* Multiple DMA transfers mode. */
> > +#define USB_CONF_DMULT         BIT(9)
> > +/* DMA clock turn-off enable. */
> > +#define USB_CONF_DMAOFFEN      BIT(10)
> > +/* DMA clock turn-off disable. */
> > +#define USB_CONF_DMAOFFDS      BIT(11)
> > +/* Clear Force Full Speed. */
> > +#define USB_CONF_CFORCE_FS     BIT(12)
> > +/* Set Force Full Speed. */
> > +#define USB_CONF_SFORCE_FS     BIT(13)
> > +/* Device enable. */
> > +#define USB_CONF_DEVEN         BIT(14)
> > +/* Device disable. */
> > +#define USB_CONF_DEVDS         BIT(15)
> > +/* L1 LPM state entry enable (used in HS/FS mode). */
> > +#define USB_CONF_L1EN          BIT(16)
> > +/* L1 LPM state entry disable (used in HS/FS mode). */
> > +#define USB_CONF_L1DS          BIT(17)
> > +/* USB 2.0 clock gate disable. */
> > +#define USB_CONF_CLK2OFFEN     BIT(18)
> > +/* USB 2.0 clock gate enable. */
> > +#define USB_CONF_CLK2OFFDS     BIT(19)
> > +/* L0 LPM state entry request (used in HS/FS mode). */
> > +#define USB_CONF_LGO_L0                BIT(20)
> > +/* USB 3.0 clock gate disable. */
> > +#define USB_CONF_CLK3OFFEN     BIT(21)
> > +/* USB 3.0 clock gate enable. */
> > +#define USB_CONF_CLK3OFFDS     BIT(22)
> > +/* Bit 23 is reserved*/
> > +/* U1 state entry enable (used in SS mode). */
> > +#define USB_CONF_U1EN          BIT(24)
> > +/* U1 state entry disable (used in SS mode). */
> > +#define USB_CONF_U1DS          BIT(25)
> > +/* U2 state entry enable (used in SS mode). */
> > +#define USB_CONF_U2EN          BIT(26)
> > +/* U2 state entry disable (used in SS mode). */
> > +#define USB_CONF_U2DS          BIT(27)
> > +/* U0 state entry request (used in SS mode). */
> > +#define USB_CONF_LGO_U0                BIT(28)
> > +/* U1 state entry request (used in SS mode). */
> > +#define USB_CONF_LGO_U1                BIT(29)
> > +/* U2 state entry request (used in SS mode). */
> > +#define USB_CONF_LGO_U2                BIT(30)
> > +/* SS.Inactive state entry request (used in SS mode) */
> > +#define USB_CONF_LGO_SSINACT   BIT(31)
> > +
> > +/* USB_STS - bitmasks */
> > +/*
> > + * Configuration status.
> > + * 1 - device is in the configured state.
> > + * 0 - device is not configured.
> > + */
> > +#define USB_STS_CFGSTS_MASK    BIT(0)
> > +#define USB_STS_CFGSTS(p)      ((p) & USB_STS_CFGSTS_MASK)
> > +/*
> > + * On-chip memory overflow.
> > + * 0 - On-chip memory status OK.
> > + * 1 - On-chip memory overflow.
> > + */
> > +#define USB_STS_OV_MASK                BIT(1)
> > +#define USB_STS_OV(p)          ((p) & USB_STS_OV_MASK)
> > +/*
> > + * SuperSpeed connection status.
> > + * 0 - USB in SuperSpeed mode disconnected.
> > + * 1 - USB in SuperSpeed mode connected.
> > + */
> > +#define USB_STS_USB3CONS_MASK  BIT(2)
> > +#define USB_STS_USB3CONS(p)    ((p) & USB_STS_USB3CONS_MASK)
> > +/*
> > + * DMA transfer configuration status.
> > + * 0 - single request.
> > + * 1 - multiple TRB chain
> > + */
> > +#define USB_STS_DTRANS_MASK    BIT(3)
> > +#define USB_STS_DTRANS(p)      ((p) & USB_STS_DTRANS_MASK)
> > +/*
> > + * Device speed.
> > + * 0 - Undefined (value after reset).
> > + * 1 - Low speed
> > + * 2 - Full speed
> > + * 3 - High speed
> > + * 4 - Super speed
> > + */
> > +#define USB_STS_USBSPEED_MASK  GENMASK(6, 4)
> > +#define USB_STS_USBSPEED(p)    (((p) & USB_STS_USBSPEED_MASK) >> 4)
> > +#define USB_STS_LS             (0x1 << 4)
> > +#define USB_STS_FS             (0x2 << 4)
> > +#define USB_STS_HS             (0x3 << 4)
> > +#define USB_STS_SS             (0x4 << 4)
> > +#define DEV_UNDEFSPEED(p)      (((p) & USB_STS_USBSPEED_MASK) == (0x0 << 4))
> > +#define DEV_LOWSPEED(p)                (((p) & USB_STS_USBSPEED_MASK) == USB_STS_LS)
> > +#define DEV_FULLSPEED(p)       (((p) & USB_STS_USBSPEED_MASK) == USB_STS_FS)
> > +#define DEV_HIGHSPEED(p)       (((p) & USB_STS_USBSPEED_MASK) == USB_STS_HS)
> > +#define DEV_SUPERSPEED(p)      (((p) & USB_STS_USBSPEED_MASK) == USB_STS_SS)
> > +/*
> > + * Endianness for SFR access.
> > + * 0 - Little Endian order (default after hardware reset).
> > + * 1 - Big Endian order
> > + */
> > +#define USB_STS_ENDIAN_MASK    BIT(7)
> > +#define USB_STS_ENDIAN(p)      ((p) & USB_STS_ENDIAN_MASK)
> > +/*
> > + * HS/FS clock turn-off status.
> > + * 0 - hsfs clock is always on.
> > + * 1 - hsfs clock turn-off in L2 (HS/FS mode) is enabled
> > + *          (default after hardware reset).
> > + */
> > +#define USB_STS_CLK2OFF_MASK   BIT(8)
> > +#define USB_STS_CLK2OFF(p)     ((p) & USB_STS_CLK2OFF_MASK)
> > +/*
> > + * PCLK clock turn-off status.
> > + * 0 - pclk clock is always on.
> > + * 1 - pclk clock turn-off in U3 (SS mode) is enabled
> > + *          (default after hardware reset).
> > + */
> > +#define USB_STS_CLK3OFF_MASK   BIT(9)
> > +#define USB_STS_CLK3OFF(p)     ((p) & USB_STS_CLK3OFF_MASK)
> > +/*
> > + * Controller in reset state.
> > + * 0 - Internal reset is active.
> > + * 1 - Internal reset is not active and controller is fully operational.
> > + */
> > +#define USB_STS_IN_RST_MASK    BIT(10)
> > +#define USB_STS_IN_RST(p)      ((p) & USB_STS_IN_RST_MASK)
> > +/*
> > + * Device enable Status.
> > + * 0 - USB device is disabled (VBUS input is disconnected from internal logic).
> > + * 1 - USB device is enabled (VBUS input is connected to the internal logic).
> > + */
> > +#define USB_STS_DEVS_MASK      BIT(14)
> > +#define USB_STS_DEVS(p)                ((p) & USB_STS_DEVS_MASK)
> > +/*
> > + * DAddress statuss.
> > + * 0 - USB device is default state.
> > + * 1 - USB device is at least in address state.
> > + */
> > +#define USB_STS_ADDRESSED_MASK BIT(15)
> > +#define USB_STS_ADDRESSED(p)   ((p) & USB_STS_ADDRESSED_MASK)
> > +/*
> > + * L1 LPM state enable status (used in HS/FS mode).
> > + * 0 - Entering to L1 LPM state disabled.
> > + * 1 - Entering to L1 LPM state enabled.
> > + */
> > +#define USB_STS_L1ENS_MASK     BIT(16)
> > +#define USB_STS_L1ENS(p)       ((p) & USB_STS_L1ENS_MASK)
> > +/*
> > + * Internal VBUS connection status (used both in HS/FS  and SS mode).
> > + * 0 - internal VBUS is not detected.
> > + * 1 - internal VBUS is detected.
> > + */
> > +#define USB_STS_VBUSS_MASK     BIT(17)
> > +#define USB_STS_VBUSS(p)       ((p) & USB_STS_VBUSS_MASK)
> > +/*
> > + * HS/FS LPM  state (used in FS/HS mode).
> > + * 0 - L0 State
> > + * 1 - L1 State
> > + * 2 - L2 State
> > + * 3 - L3 State
> > + */
> > +#define USB_STS_LPMST_MASK     GENMASK(19, 18)
> > +#define DEV_L0_STATE(p)                (((p) & USB_STS_LPMST_MASK) == (0x0 << 18))
> > +#define DEV_L1_STATE(p)                (((p) & USB_STS_LPMST_MASK) == (0x1 << 18))
> > +#define DEV_L2_STATE(p)                (((p) & USB_STS_LPMST_MASK) == (0x2 << 18))
> > +#define DEV_L3_STATE(p)                (((p) & USB_STS_LPMST_MASK) == (0x3 << 18))
> > +/*
> > + * Disable HS status (used in FS/HS mode).
> > + * 0 - the disconnect bit for HS/FS mode is set .
> > + * 1 - the disconnect bit for HS/FS mode is not set.
> > + */
> > +#define USB_STS_USB2CONS_MASK  BIT(20)
> > +#define USB_STS_USB2CONS(p)    ((p) & USB_STS_USB2CONS_MASK)
> > +/*
> > + * HS/FS mode connection status (used in FS/HS mode).
> > + * 0 - High Speed operations in USB2.0 (FS/HS) mode not disabled.
> > + * 1 - High Speed operations in USB2.0 (FS/HS).
> > + */
> > +#define USB_STS_DISABLE_HS_MASK        BIT(21)
> > +#define USB_STS_DISABLE_HS(p)  ((p) & USB_STS_DISABLE_HS_MASK)
> > +/*
> > + * U1 state enable status (used in SS mode).
> > + * 0 - Entering to  U1 state disabled.
> > + * 1 - Entering to  U1 state enabled.
> > + */
> > +#define USB_STS_U1ENS_MASK     BIT(24)
> > +#define USB_STS_U1ENS(p)       ((p) & USB_STS_U1ENS_MASK)
> > +/*
> > + * U2 state enable status (used in SS mode).
> > + * 0 - Entering to  U2 state disabled.
> > + * 1 - Entering to  U2 state enabled.
> > + */
> > +#define USB_STS_U2ENS_MASK     BIT(25)
> > +#define USB_STS_U2ENS(p)       ((p) & USB_STS_U2ENS_MASK)
> > +/*
> > + * SuperSpeed Link LTSSM state. This field reflects USBSS-DEV current
> > + * SuperSpeed link state
> > + */
> > +#define USB_STS_LST_MASK       GENMASK(29, 26)
> > +#define DEV_LST_U0             (((p) & USB_STS_LST_MASK) == (0x0 << 26))
> > +#define DEV_LST_U1             (((p) & USB_STS_LST_MASK) == (0x1 << 26))
> > +#define DEV_LST_U2             (((p) & USB_STS_LST_MASK) == (0x2 << 26))
> > +#define DEV_LST_U3             (((p) & USB_STS_LST_MASK) == (0x3 << 26))
> > +#define DEV_LST_DISABLED       (((p) & USB_STS_LST_MASK) == (0x4 << 26))
> > +#define DEV_LST_RXDETECT       (((p) & USB_STS_LST_MASK) == (0x5 << 26))
> > +#define DEV_LST_INACTIVE       (((p) & USB_STS_LST_MASK) == (0x6 << 26))
> > +#define DEV_LST_POLLING                (((p) & USB_STS_LST_MASK) == (0x7 << 26))
> > +#define DEV_LST_RECOVERY       (((p) & USB_STS_LST_MASK) == (0x8 << 26))
> > +#define DEV_LST_HOT_RESET      (((p) & USB_STS_LST_MASK) == (0x9 << 26))
> > +#define DEV_LST_COMP_MODE      (((p) & USB_STS_LST_MASK) == (0xa << 26))
> > +#define DEV_LST_LB_STATE       (((p) & USB_STS_LST_MASK) == (0xb << 26))
> > +/*
> > + * DMA clock turn-off status.
> > + * 0 - DMA clock is always on (default after hardware reset).
> > + * 1 - DMA clock turn-off in U1, U2 and U3 (SS mode) is enabled.
> > + */
> > +#define USB_STS_DMAOFF_MASK    BIT(30)
> > +#define USB_STS_DMAOFF(p)      ((p) & USB_STS_DMAOFF_MASK)
> > +/*
> > + * SFR Endian statuss.
> > + * 0 - Little Endian order (default after hardware reset).
> > + * 1 - Big Endian order.
> > + */
> > +#define USB_STS_ENDIAN2_MASK   BIT(31)
> > +#define USB_STS_ENDIAN2(p)     ((p) & USB_STS_ENDIAN2_MASK)
> > +
> > +/* USB_CMD -  bitmasks */
> > +/* Set Function Address */
> > +#define USB_CMD_SET_ADDR       BIT(0)
> > +/*
> > + * Function Address This field is saved to the device only when the field
> > + * SET_ADDR is set '1 ' during write to USB_CMD register.
> > + * Software is responsible for entering the address of the device during
> > + * SET_ADDRESS request service. This field should be set immediately after
> > + * the SETUP packet is decoded, and prior to confirmation of the status phase
> > + */
> > +#define USB_CMD_FADDR_MASK     GENMASK(7, 1)
> > +#define USB_CMD_FADDR(p)       (((p) << 1) & USB_CMD_FADDR_MASK)
> > +/* Send Function Wake Device Notification TP (used only in SS mode). */
> > +#define USB_CMD_SDNFW          BIT(8)
> > +/* Set Test Mode (used only in HS/FS mode). */
> > +#define USB_CMD_STMODE         BIT(9)
> > +/* Test mode selector (used only in HS/FS mode) */
> > +#define USB_STS_TMODE_SEL_MASK GENMASK(11, 10)
> > +#define USB_STS_TMODE_SEL(p)   (((p) << 10) & USB_STS_TMODE_SEL_MASK)
> > +/*
> > + *  Send Latency Tolerance Message Device Notification TP (used only
> > + *  in SS mode).
> > + */
> > +#define USB_CMD_SDNLTM         BIT(12)
> > +/* Send Custom Transaction Packet (used only in SS mode) */
> > +#define USB_CMD_SPKT           BIT(13)
> > +/*Device Notification 'Function Wake' - Interface value (only in SS mode. */
> > +#define USB_CMD_DNFW_INT_MASK  GENMASK(23, 16)
> > +#define USB_STS_DNFW_INT(p)    (((p) << 16) & USB_CMD_DNFW_INT_MASK)
> > +/*
> > + * Device Notification 'Latency Tolerance Message' -373 BELT value [7:0]
> > + * (used only in SS mode).
> > + */
> > +#define USB_CMD_DNLTM_BELT_MASK        GENMASK(27, 16)
> > +#define USB_STS_DNLTM_BELT(p)  (((p) << 16) & USB_CMD_DNLTM_BELT_MASK)
> > +
> > +/* USB_ITPN - bitmasks */
> > +/*
> > + * ITP(SS) / SOF (HS/FS) number
> > + * In SS mode this field represent number of last ITP received from host.
> > + * In HS/FS mode this field represent number of last SOF received from host.
> > + */
> > +#define USB_ITPN_MASK          GENMASK(13, 0)
> > +#define USB_ITPN(p)            ((p) & USB_ITPN_MASK)
> > +
> > +/* USB_LPM - bitmasks */
> > +/* Host Initiated Resume Duration. */
> > +#define USB_LPM_HIRD_MASK      GENMASK(3, 0)
> > +#define USB_LPM_HIRD(p)                ((p) & USB_LPM_HIRD_MASK)
> > +/* Remote Wakeup Enable (bRemoteWake). */
> > +#define USB_LPM_BRW            BIT(4)
> > +
> > +/* USB_IEN - bitmasks */
> > +/* SS connection interrupt enable */
> > +#define USB_IEN_CONIEN         BIT(0)
> > +/* SS disconnection interrupt enable. */
> > +#define USB_IEN_DISIEN         BIT(1)
> > +/* USB SS warm reset interrupt enable. */
> > +#define USB_IEN_UWRESIEN       BIT(2)
> > +/* USB SS hot reset interrupt enable */
> > +#define USB_IEN_UHRESIEN       BIT(3)
> > +/* SS link U3 state enter interrupt enable (suspend).*/
> > +#define USB_IEN_U3ENTIEN       BIT(4)
> > +/* SS link U3 state exit interrupt enable (wakeup). */
> > +#define USB_IEN_U3EXTIEN       BIT(5)
> > +/* SS link U2 state enter interrupt enable.*/
> > +#define USB_IEN_U2ENTIEN       BIT(6)
> > +/* SS link U2 state exit interrupt enable.*/
> > +#define USB_IEN_U2EXTIEN       BIT(7)
> > +/* SS link U1 state enter interrupt enable.*/
> > +#define USB_IEN_U1ENTIEN       BIT(8)
> > +/* SS link U1 state exit interrupt enable.*/
> > +#define USB_IEN_U1EXTIEN       BIT(9)
> > +/* ITP/SOF packet detected interrupt enable.*/
> > +#define USB_IEN_ITPIEN         BIT(10)
> > +/* Wakeup interrupt enable.*/
> > +#define USB_IEN_WAKEIEN                BIT(11)
> > +/* Send Custom Packet interrupt enable.*/
> > +#define USB_IEN_SPKTIEN                BIT(12)
> > +/* HS/FS mode connection interrupt enable.*/
> > +#define USB_IEN_CON2IEN                BIT(16)
> > +/* HS/FS mode disconnection interrupt enable.*/
> > +#define USB_IEN_DIS2IEN                BIT(17)
> > +/* USB reset (HS/FS mode) interrupt enable.*/
> > +#define USB_IEN_U2RESIEN       BIT(18)
> > +/* LPM L2 state enter interrupt enable.*/
> > +#define USB_IEN_L2ENTIEN       BIT(20)
> > +/* LPM  L2 state exit interrupt enable.*/
> > +#define USB_IEN_L2EXTIEN       BIT(21)
> > +/* LPM L1 state enter interrupt enable.*/
> > +#define USB_IEN_L1ENTIEN       BIT(24)
> > +/* LPM  L1 state exit interrupt enable.*/
> > +#define USB_IEN_L1EXTIEN       BIT(25)
> > +/* Configuration reset interrupt enable.*/
> > +#define USB_IEN_CFGRESIEN      BIT(26)
> > +/* Start of the USB SS warm reset interrupt enable.*/
> > +#define USB_IEN_UWRESSIEN      BIT(28)
> > +/* End of the USB SS warm reset interrupt enable.*/
> > +#define USB_IEN_UWRESEIEN      BIT(29)
> > +
> > +#define USB_IEN_INIT  (USB_IEN_U2RESIEN | USB_ISTS_DIS2I | USB_IEN_CON2IEN \
> > +                      | USB_IEN_UHRESIEN | USB_IEN_UWRESIEN | USB_IEN_DISIEN \
> > +                      | USB_IEN_CONIEN | USB_IEN_U3EXTIEN | USB_IEN_L2ENTIEN \
> > +                      | USB_IEN_L2EXTIEN)
> > +
> > +/* USB_ISTS - bitmasks */
> > +/* SS Connection detected. */
> > +#define USB_ISTS_CONI          BIT(0)
> > +/* SS Disconnection detected. */
> > +#define USB_ISTS_DISI          BIT(1)
> > +/* UUSB warm reset detectede. */
> > +#define USB_ISTS_UWRESI                BIT(2)
> > +/* USB hot reset detected. */
> > +#define USB_ISTS_UHRESI                BIT(3)
> > +/* U3 link state enter detected (suspend).*/
> > +#define USB_ISTS_U3ENTI                BIT(4)
> > +/* U3 link state exit detected (wakeup). */
> > +#define USB_ISTS_U3EXTI                BIT(5)
> > +/* U2 link state enter detected.*/
> > +#define USB_ISTS_U2ENTI                BIT(6)
> > +/* U2 link state exit detected.*/
> > +#define USB_ISTS_U2EXTI                BIT(7)
> > +/* U1 link state enter detected.*/
> > +#define USB_ISTS_U1ENTI                BIT(8)
> > +/* U1 link state exit detected.*/
> > +#define USB_ISTS_U1EXTI                BIT(9)
> > +/* ITP/SOF packet detected.*/
> > +#define USB_ISTS_ITPI          BIT(10)
> > +/* Wakeup detected.*/
> > +#define USB_ISTS_WAKEI         BIT(11)
> > +/* Send Custom Packet detected.*/
> > +#define USB_ISTS_SPKTI         BIT(12)
> > +/* HS/FS mode connection detected.*/
> > +#define USB_ISTS_CON2I         BIT(16)
> > +/* HS/FS mode disconnection detected.*/
> > +#define USB_ISTS_DIS2I         BIT(17)
> > +/* USB reset (HS/FS mode) detected.*/
> > +#define USB_ISTS_U2RESI                BIT(18)
> > +/* LPM L2 state enter detected.*/
> > +#define USB_ISTS_L2ENTI                BIT(20)
> > +/* LPM  L2 state exit detected.*/
> > +#define USB_ISTS_L2EXTI                BIT(21)
> > +/* LPM L1 state enter detected.*/
> > +#define USB_ISTS_L1ENTI                BIT(24)
> > +/* LPM L1 state exit detected.*/
> > +#define USB_ISTS_L1EXTI                BIT(25)
> > +/* USB configuration reset detected.*/
> > +#define USB_ISTS_CFGRESI       BIT(26)
> > +/* Start of the USB warm reset detected.*/
> > +#define USB_ISTS_UWRESSI       BIT(28)
> > +/* End of the USB warm reset detected.*/
> > +#define USB_ISTS_UWRESEI       BIT(29)
> > +
> > +/* USB_SEL - bitmasks */
> > +#define EP_SEL_EPNO_MASK       GENMASK(3, 0)
> > +/* Endpoint number. */
> > +#define EP_SEL_EPNO(p)         ((p) & EP_SEL_EPNO_MASK)
> > +/* Endpoint direction bit - 0 - OUT, 1 - IN. */
> > +#define EP_SEL_DIR             BIT(7)
> > +
> > +#define select_ep_in(nr)       (EP_SEL_EPNO(p) | EP_SEL_DIR)
> > +#define select_ep_out          (EP_SEL_EPNO(p))
> > +
> > +/* EP_TRADDR - bitmasks */
> > +/* Transfer Ring address. */
> > +#define EP_TRADDR_TRADDR(p)    ((p))
> > +
> > +/* EP_CFG - bitmasks */
> > +/* Endpoint enable */
> > +#define EP_CFG_ENABLE          BIT(0)
> > +/*
> > + *  Endpoint type.
> > + * 1 - isochronous
> > + * 2 - bulk
> > + * 3 - interrupt
> > + */
> > +#define EP_CFG_EPTYPE_MASK     GENMASK(2, 1)
> > +#define EP_CFG_EPTYPE(p)       (((p) << 1)  & EP_CFG_EPTYPE_MASK)
> > +/* Stream support enable (only in SS mode). */
> > +#define EP_CFG_STREAM_EN       BIT(3)
> > +/* TDL check (only in SS mode for BULK EP). */
> > +#define EP_CFG_TDL_CHK         BIT(4)
> > +/* SID check (only in SS mode for BULK OUT EP). */
> > +#define EP_CFG_SID_CHK         BIT(5)
> > +/* DMA transfer endianness. */
> > +#define EP_CFG_EPENDIAN                BIT(7)
> > +/* Max burst size (used only in SS mode). */
> > +#define EP_CFG_MAXBURST_MASK   GENMASK(11, 8)
> > +#define EP_CFG_MAXBURST(p)     (((p) << 8) & EP_CFG_MAXBURST_MASK)
> > +/* ISO max burst. */
> > +#define EP_CFG_MULT_MASK       GENMASK(15, 14)
> > +#define EP_CFG_MULT(p)         (((p) << 14) & EP_CFG_MULT)
> > +/* ISO max burst. */
> > +#define EP_CFG_MAXPKTSIZE_MASK GENMASK(26, 16)
> > +#define EP_CFG_MAXPKTSIZE(p)   (((p) << 16) & EP_CFG_MAXPKTSIZE_MASK)
> > +/* Max number of buffered packets. */
> > +#define EP_CFG_BUFFERING_MASK  GENMASK(31, 27)
> > +#define EP_CFG_BUFFERING(p)    (((p) << 27) & EP_CFG_BUFFERING_MASK)
> > +
> > +/* EP_CMD - bitmasks */
> > +/* Endpoint reset. */
> > +#define EP_CMD_EPRST           BIT(0)
> > +/* Endpoint STALL set. */
> > +#define EP_CMD_SSTALL          BIT(1)
> > +/* Endpoint STALL clear. */
> > +#define EP_CMD_CSTALL          BIT(2)
> > +/* Send ERDY TP. */
> > +#define EP_CMD_ERDY            BIT(3)
> > +/* Request complete. */
> > +#define EP_CMD_REQ_CMPL                BIT(5)
> > +/* Transfer descriptor ready. */
> > +#define EP_CMD_DRDY            BIT(6)
> > +/* Data flush. */
> > +#define EP_CMD_DFLUSH          BIT(7)
> > +/*
> > + * Transfer Descriptor Length write  (used only for Bulk Stream capable
> > + * endpoints in SS mode).
> > + */
> > +#define EP_CMD_STDL            BIT(8)
> > +/* Transfer Descriptor Length (used only in SS mode for bulk endpoints). */
> > +#define EP_CMD_TDL_MASK                GENMASK(15, 9)
> > +#define EP_CMD_TDL(p)          (((p) << 9) & EP_CMD_TDL_MASK)
> > +/* ERDY Stream ID value (used in SS mode). */
> > +#define EP_CMD_ERDY_SID_MASK   GENMASK(31, 16)
> > +#define EP_CMD_ERDY_SID(p)     (((p) << 16) & EP_CMD_SID_MASK)
> > +
> > +/* EP_STS - bitmasks */
> > +/* Setup transfer complete. */
> > +#define EP_STS_SETUP           BIT(0)
> > +/* Endpoint STALL status. */
> > +#define EP_STS_STALL(p)                ((p) & BIT(1))
> > +/* Interrupt On Complete. */
> > +#define EP_STS_IOC             BIT(2)
> > +/* Interrupt on Short Packet. */
> > +#define EP_STS_ISP             BIT(3)
> > +/* Transfer descriptor missing. */
> > +#define EP_STS_DESCMIS         BIT(4)
> > +/* Stream Rejected (used only in SS mode) */
> > +#define EP_STS_STREAMR         BIT(5)
> > +/* EXIT from MOVE DATA State (used only for stream transfers in SS mode). */
> > +#define EP_STS_MD_EXIT         BIT(6)
> > +/* TRB error. */
> > +#define EP_STS_TRBERR          BIT(7)
> > +/* Not ready (used only in SS mode). */
> > +#define EP_STS_NRDY            BIT(8)
> > +/* DMA busy. */
> > +#define EP_STS_DBUSY(p)                ((p) & BIT(9))
> > +/* Endpoint Buffer Empty */
> > +#define EP_STS_BUFFEMPTY(p)    ((p) & BIT(10))
> > +/* Current Cycle Status */
> > +#define EP_STS_CCS(p)          ((p) & BIT(11))
> > +/* Prime (used only in SS mode. */
> > +#define EP_STS_PRIME           BIT(12)
> > +/* Stream error (used only in SS mode). */
> > +#define EP_STS_SIDERR          BIT(13)
> > +/* OUT size mismatch. */
> > +#define EP_STS_OUTSMM          BIT(14)
> > +/* ISO transmission error. */
> > +#define EP_STS_ISOERR          BIT(15)
> > +/* Host Packet Pending (only for SS mode). */
> > +#define EP_STS_HOSTPP(p)       ((p) & BIT(16))
> > +/* Stream Protocol State Machine State (only for Bulk stream endpoints). */
> > +#define EP_STS_SPSMST_MASK             GENMASK(18, 17)
> > +#define EP_STS_SPSMST_DISABLED(p)      (((p) & EP_STS_SPSMST_MASK) >> 17)
> > +#define EP_STS_SPSMST_IDLE(p)          (((p) & EP_STS_SPSMST_MASK) >> 17)
> > +#define EP_STS_SPSMST_START_STREAM(p)  (((p) & EP_STS_SPSMST_MASK) >> 17)
> > +#define EP_STS_SPSMST_MOVE_DATA(p)     (((p) & EP_STS_SPSMST_MASK) >> 17)
> > +/* Interrupt On Transfer complete. */
> > +#define EP_STS_IOT             BIT(19)
> > +/* OUT queue endpoint number. */
> > +#define EP_STS_OUTQ_NO_MASK    GENMASK(27, 24)
> > +#define EP_STS_OUTQ_NO(p)      (((p) & EP_STS_OUTQ_NO_MASK) >> 24)
> > +/* OUT queue valid flag. */
> > +#define EP_STS_OUTQ_VAL_MASK   BIT(28)
> > +#define EP_STS_OUTQ_VAL(p)     ((p) & EP_STS_OUTQ_VAL_MASK)
> > +/* SETUP WAIT. */
> > +#define EP_STS_STPWAIT         BIT(31)
> > +
> > +/* EP_STS_SID - bitmasks */
> > +/* Stream ID (used only in SS mode). */
> > +#define EP_STS_SID_MASK                GENMASK(15, 0)
> > +#define EP_STS_SID(p)          ((p) & EP_STS_SID_MASK)
> > +
> > +/* EP_STS_EN - bitmasks */
> > +/* SETUP interrupt enable. */
> > +#define EP_STS_EN_SETUPEN      BIT(0)
> > +/* OUT transfer missing descriptor enable. */
> > +#define EP_STS_EN_DESCMISEN    BIT(4)
> > +/* Stream Rejected enable. */
> > +#define EP_STS_EN_STREAMREN    BIT(5)
> > +/* Move Data Exit enable.*/
> > +#define EP_STS_EN_MD_EXITEN    BIT(6)
> > +/* TRB enable. */
> > +#define EP_STS_EN_TRBERREN     BIT(7)
> > +/* NRDY enable. */
> > +#define EP_STS_EN_NRDYEN       BIT(8)
> > +/* Prime enable. */
> > +#define EP_STS_EN_PRIMEEEN     BIT(12)
> > +/* Stream error enable. */
> > +#define EP_STS_EN_SIDERREN     BIT(13)
> > +/* OUT size mismatch enable. */
> > +#define EP_STS_EN_OUTSMMEN     BIT(14)
> > +/* ISO transmission error enable. */
> > +#define EP_STS_EN_ISOERREN     BIT(15)
> > +/* Interrupt on Transmission complete enable. */
> > +#define EP_STS_EN_IOTEN                BIT(19)
> > +/* Setup Wait interrupt enable. */
> > +#define EP_STS_EN_STPWAITEN    BIT(31)
> > +
> > +/* DRBL- bitmasks */
> > +#define DB_VALUE_BY_INDEX(index) (1 << (index))
> > +#define DB_VALUE_EP0_OUT       BIT(0)
> > +#define DB_VALUE_EP0_IN                BIT(16)
> > +
> > +/* EP_IEN - bitmasks */
> > +#define EP_IEN(index)          (1 << (index))
> > +#define EP_IEN_EP_OUT0         BIT(0)
> > +#define EP_IEN_EP_IN0          BIT(16)
> > +
> > +/* EP_ISTS - bitmasks */
> > +#define EP_ISTS(index)         (1 << (index))
> > +#define EP_ISTS_EP_OUT0                BIT(0)
> > +#define EP_ISTS_EP_IN0         BIT(16)
> > +
> > +/* EP_PWR- bitmasks */
> > +/*Power Shut Off capability enable*/
> > +#define PUSB_PWR_PSO_EN                BIT(0)
> > +/*Power Shut Off capability disable*/
> > +#define PUSB_PWR_PSO_DS                BIT(1)
> > +/*
> > + * Enables turning-off Reference Clock.
> > + * This bit is optional and implemented only when support for OTG is
> > + * implemented (indicated by OTG_READY bit set to '1').
> > + */
> > +#define PUSB_PWR_STB_CLK_SWITCH_EN     BIT(8)
> > +/*
> > + * Status bit indicating that operation required by STB_CLK_SWITCH_EN write
> > + * is completed
> > + */
> > +#define PUSB_PWR_STB_CLK_SWITCH_DONE   BIT(9)
> > +/* This bit informs if Fast Registers Access is enabled. */
> > +#define PUSB_PWR_FST_REG_ACCESS_STAT   BIT(30)
> > +/* Fast Registers Access Enable. */
> > +#define PUSB_PWR_FST_REG_ACCESS        BIT(31)
> > +
> > +/* USB_CAP1- bitmasks */
> > +/*
> > + * SFR Interface type
> > + * These field reflects type of SFR interface implemented:
> > + * 0x0 - OCP
> > + * 0x1 - AHB,
> > + * 0x2 - PLB
> > + * 0x3 - AXI
> > + * 0x4-0xF - reserved
> > + */
> > +#define USB_CAP1_SFR_TYPE_MASK GENMASK(3, 0)
> > +#define DEV_SFR_TYPE_OCP(p)    (((p) & USB_CAP1_SFR_TYPE_MASK) == 0x0)
> > +#define DEV_SFR_TYPE_AHB(p)    (((p) & USB_CAP1_SFR_TYPE_MASK) == 0x1)
> > +#define DEV_SFR_TYPE_PLB(p)    (((p) & USB_CAP1_SFR_TYPE_MASK) == 0x2)
> > +#define DEV_SFR_TYPE_AXI(p)    (((p) & USB_CAP1_SFR_TYPE_MASK) == 0x3)
> > +/*
> > + * SFR Interface width
> > + * These field reflects width of SFR interface implemented:
> > + * 0x0 - 8 bit interface,
> > + * 0x1 - 16 bit interface,
> > + * 0x2 - 32 bit interface
> > + * 0x3 - 64 bit interface
> > + * 0x4-0xF - reserved
> > + */
> > +#define USB_CAP1_SFR_WIDTH_MASK        GENMASK(7, 4)
> > +#define DEV_SFR_WIDTH_8(p)     (((p) & USB_CAP1_SFR_WIDTH_MASK) == (0x0 << 4))
> > +#define DEV_SFR_WIDTH_16(p)    (((p) & USB_CAP1_SFR_WIDTH_MASK) == (0x1 << 4))
> > +#define DEV_SFR_WIDTH_32(p)    (((p) & USB_CAP1_SFR_WIDTH_MASK) == (0x2 << 4))
> > +#define DEV_SFR_WIDTH_64(p)    (((p) & USB_CAP1_SFR_WIDTH_MASK) == (0x3 << 4))
> > +/*
> > + * DMA Interface type
> > + * These field reflects type of DMA interface implemented:
> > + * 0x0 - OCP
> > + * 0x1 - AHB,
> > + * 0x2 - PLB
> > + * 0x3 - AXI
> > + * 0x4-0xF - reserved
> > + */
> > +#define USB_CAP1_DMA_TYPE_MASK GENMASK(11, 8)
> > +#define DEV_DMA_TYPE_OCP(p)    (((p) & USB_CAP1_DMA_TYPE_MASK) == (0x0 << 8))
> > +#define DEV_DMA_TYPE_AHB(p)    (((p) & USB_CAP1_DMA_TYPE_MASK) == (0x1 << 8))
> > +#define DEV_DMA_TYPE_PLB(p)    (((p) & USB_CAP1_DMA_TYPE_MASK) == (0x2 << 8))
> > +#define DEV_DMA_TYPE_AXI(p)    (((p) & USB_CAP1_DMA_TYPE_MASK) == (0x3 << 8))
> > +/*
> > + * DMA Interface width
> > + * These field reflects width of DMA interface implemented:
> > + * 0x0 - reserved,
> > + * 0x1 - reserved,
> > + * 0x2 - 32 bit interface
> > + * 0x3 - 64 bit interface
> > + * 0x4-0xF - reserved
> > + */
> > +#define USB_CAP1_DMA_WIDTH_MASK        GENMASK(15, 12)
> > +#define DEV_DMA_WIDTH_32(p)    (((p) & USB_CAP1_DMA_WIDTH_MASK) == (0x2 << 12))
> > +#define DEV_DMA_WIDTH_64(p)    (((p) & USB_CAP1_DMA_WIDTH_MASK) == (0x3 << 12))
> > +/*
> > + * USB3 PHY Interface type
> > + * These field reflects type of USB3 PHY interface implemented:
> > + * 0x0 - USB PIPE,
> > + * 0x1 - RMMI,
> > + * 0x2-0xF - reserved
> > + */
> > +#define USB_CAP1_U3PHY_TYPE_MASK GENMASK(19, 16)
> > +#define DEV_U3PHY_PIPE(p) (((p) & USB_CAP1_U3PHY_TYPE_MASK) == (0x0 << 16))
> > +#define DEV_U3PHY_RMMI(p) (((p) & USB_CAP1_U3PHY_TYPE_MASK) == (0x1 << 16))
> > +/*
> > + * USB3 PHY Interface width
> > + * These field reflects width of USB3 PHY interface implemented:
> > + * 0x0 - 8 bit PIPE interface,
> > + * 0x1 - 16 bit PIPE interface,
> > + * 0x2 - 32 bit PIPE interface,
> > + * 0x3 - 64 bit PIPE interface
> > + * 0x4-0xF - reserved
> > + * Note: When SSIC interface is implemented this field shows the width of
> > + * internal PIPE interface. The RMMI interface is always 20bit wide.
> > + */
> > +#define USB_CAP1_U3PHY_WIDTH_MASK GENMASK(23, 20)
> > +#define DEV_U3PHY_WIDTH_8(p) \
> > +       (((p) & USB_CAP1_U3PHY_WIDTH_MASK) == (0x0 << 20))
> > +#define DEV_U3PHY_WIDTH_16(p) \
> > +       (((p) & USB_CAP1_U3PHY_WIDTH_MASK) == (0x1 << 16))
> > +#define DEV_U3PHY_WIDTH_32(p) \
> > +       (((p) & USB_CAP1_U3PHY_WIDTH_MASK) == (0x2 << 20))
> > +#define DEV_U3PHY_WIDTH_64(p) \
> > +       (((p) & USB_CAP1_U3PHY_WIDTH_MASK) == (0x3 << 16))
> > +
> > +/*
> > + * USB2 PHY Interface enable
> > + * These field informs if USB2 PHY interface is implemented:
> > + * 0x0 - interface NOT implemented,
> > + * 0x1 - interface implemented
> > + */
> > +#define USB_CAP1_U2PHY_EN(p)   ((p) & BIT(24))
> > +/*
> > + * USB2 PHY Interface type
> > + * These field reflects type of USB2 PHY interface implemented:
> > + * 0x0 - UTMI,
> > + * 0x1 - ULPI
> > + */
> > +#define DEV_U2PHY_ULPI(p)      ((p) & BIT(25))
> > +/*
> > + * USB2 PHY Interface width
> > + * These field reflects width of USB2 PHY interface implemented:
> > + * 0x0 - 8 bit interface,
> > + * 0x1 - 16 bit interface,
> > + * Note: The ULPI interface is always 8bit wide.
> > + */
> > +#define DEV_U2PHY_WIDTH_16(p)  ((p) & BIT(26))
> > +/*
> > + * OTG Ready
> > + * 0x0 - pure device mode
> > + * 0x1 - some features and ports for CDNS USB OTG controller are implemented.
> > + */
> > +#define USB_CAP1_OTG_READY(p)  ((p) & BIT(27))
> > +
> > +/* USB_CAP2- bitmasks */
> > +/*
> > + * The actual size of the connected On-chip RAM memory in kB:
> > + * - 0 means 256 kB (max supported mem size)
> > + * - value other than 0 reflects the mem size in kB
> > + */
> > +#define USB_CAP2_ACTUAL_MEM_SIZE(p) ((p) & GENMASK(7, 0))
> > +/*
> > + * Max supported mem size
> > + * These field reflects width of on-chip RAM address bus width,
> > + * which determines max supported mem size:
> > + * 0x0-0x7 - reserved,
> > + * 0x8 - support for 4kB mem,
> > + * 0x9 - support for 8kB mem,
> > + * 0xA - support for 16kB mem,
> > + * 0xB - support for 32kB mem,
> > + * 0xC - support for 64kB mem,
> > + * 0xD - support for 128kB mem,
> > + * 0xE - support for 256kB mem,
> > + * 0xF - reserved
> > + */
> > +#define USB_CAP2_MAX_MEM_SIZE(p) ((p) & GENMASK(11, 8))
> > +
> > +/* USB_CAP3- bitmasks */
> > +#define EP_IS_IMPLEMENTED(reg, index) ((reg) & (1 << (index)))
> > +
> > +/* USB_CAP4- bitmasks */
> > +#define EP_SUPPORT_ISO(reg, index) ((reg) & (1 << (index)))
> > +
> > +/* USB_CAP5- bitmasks */
> > +#define EP_SUPPORT_STREAM(reg, index) ((reg) & (1 << (index)))
> > +
> > +/* USB_CAP6- bitmasks */
> > +/* The USBSS-DEV Controller  Internal build number. */
> > +#define GET_DEV_VERSION__INTERNAL_NUMBER(p) ((p) & GENMASK(7, 0))
> > +/* The USBSS-DEV Controller  version number. */
> > +#define GET_DEV_VERSION(p) ((p) & GENMASK(31, 8))
> > +
> > +/* DBG_LINK1- bitmasks */
> > +/*
> > + * LFPS_MIN_DET_U1_EXIT value This parameter configures the minimum
> > + * time required for decoding the received LFPS as an LFPS.U1_Exit.
> > + */
> > +#define DBG_LINK1_LFPS_MIN_DET_U1_EXIT(p)      ((p) & GENMASK(7, 0))
> > +/*
> > + * LFPS_MIN_GEN_U1_EXIT value This parameter configures the minimum time for
> > + * phytxelecidle deassertion when LFPS.U1_Exit
> > + */
> > +#define DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(p)      (((p) << 8) & GENMASK(15, 8))
> > +/*
> > + * RXDET_BREAK_DIS value This parameter configures terminating the Far-end
> > + * Receiver termination detection sequence:
> > + * 0: it is possible that USBSS_DEV will terminate Farend receiver
> > + *    termination detection sequence
> > + * 1: USBSS_DEV will not terminate Far-end receiver termination
> > + *    detection sequence
> > + */
> > +#define DBG_LINK1_RXDET_BREAK_DIS              BIT(16)
> > +/* LFPS_GEN_PING value This parameter configures the LFPS.Ping generation */
> > +#define DBG_LINK1_LFPS_GEN_PING(p)             (((p) << 17) & GENMASK(21, 17))
> > +/*
> > + * Set the LFPS_MIN_DET_U1_EXIT value Writing '1' to this bit writes the
> > + * LFPS_MIN_DET_U1_EXIT field value to the device. This bit is automatically
> > + * cleared. Writing '0' has no effect
> > + */
> > +#define DBG_LINK1_LFPS_MIN_DET_U1_EXIT_SET     BIT(24)
> > +/*
> > + * Set the LFPS_MIN_GEN_U1_EXIT value. Writing '1' to this bit writes the
> > + * LFPS_MIN_GEN_U1_EXIT field value to the device. This bit is automatically
> > + * cleared. Writing '0' has no effect
> > + */
> > +#define DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET     BIT(25)
> > +/*
> > + * Set the RXDET_BREAK_DIS value Writing '1' to this bit writes
> > + * the RXDET_BREAK_DIS field value to the device. This bit is automatically
> > + * cleared. Writing '0' has no effect
> > + */
> > +#define DBG_LINK1_RXDET_BREAK_DIS_SET          BIT(26)
> > +/*
> > + * Set the LFPS_GEN_PING_SET value Writing '1' to this bit writes
> > + * the LFPS_GEN_PING field value to the device. This bit is automatically
> > + * cleared. Writing '0' has no effect."
> > + */
> > +#define DBG_LINK1_LFPS_GEN_PING_SET            BIT(27)
> > +
> > +#define gadget_to_cdns3_device(g) (container_of(g, struct cdns3_device, gadget))
> > +
> > +#define ep_to_cdns3_ep(ep) (container_of(ep, struct cdns3_endpoint, endpoint))
> > +
> > +/*-------------------------------------------------------------------------*/
> > +/*
> > + * USBSS-DEV DMA interface.
> > + */
> > +#define TRBS_PER_SEGMENT       40
> > +
> > +#define ISO_MAX_INTERVAL       10
> > +
> > +/*
> > + *Only for ISOC endpoints - maximum number of TRBs is calculated as
> > + * pow(2, bInterval-1) * number of usb requests. It is limitation made by
> > + * driver to save memory. Controller must prepare TRB for each ITP even
> > + * if bInterval > 1. It's the reason why driver needs so many TRBs for
> > + * isochronous endpoints.
> > + */
> > +#define TRBS_PER_ISOC_SEGMENT  (ISO_MAX_INTERVAL * 8)
> > +
> > +#define GET_TRBS_PER_SEGMENT(ep_type) ((ep_type) == USB_ENDPOINT_XFER_ISOC ? \
> > +                                     TRBS_PER_ISOC_SEGMENT : TRBS_PER_SEGMENT)
> > +/**
> > + * struct cdns3_trb - represent Transfer Descriptor block.
> > + * @buffer:    pointer to buffer data
> > + * @length:    length of data
> > + * @control:   control flags.
> > + *
> > + * This structure describes transfer block serviced by DMA module.
> > + */
> > +struct cdns3_trb {
> > +       __le32 buffer;
> > +       __le32 length;
> > +       __le32 control;
> > +};
> > +
> > +#define TRB_SIZE               (sizeof(struct cdns3_trb))
> > +#define TRB_RING_SIZE          (TRB_SIZE * TRBS_PER_SEGMENT)
> > +#define TRB_ISO_RING_SIZE      (TRB_SIZE * TRBS_PER_ISOC_SEGMENT)
> > +#define TRB_CTRL_RING_SIZE     (TRB_SIZE * 2)
> > +
> > +/* TRB bit mask */
> > +#define TRB_TYPE_BITMASK       GENMASK(15, 10)
> > +#define TRB_TYPE(p)            ((p) << 10)
> > +#define TRB_FIELD_TO_TYPE(p)   (((p) & TRB_TYPE_BITMASK) >> 10)
> > +
> > +/* TRB type IDs */
> > +/* bulk, interrupt, isoc , and control data stage */
> > +#define TRB_NORMAL             1
> > +/* TRB for linking ring segments */
> > +#define TRB_LINK               6
> > +
> > +/* Cycle bit - indicates TRB ownership by driver or hw*/
> > +#define TRB_CYCLE              BIT(0)
> > +/*
> > + * When set to '1', the device will toggle its interpretation of the Cycle bit
> > + */
> > +#define TRB_TOGGLE             BIT(1)
> > +
> > +/* Interrupt on short packet*/
> > +#define TRB_ISP                        BIT(2)
> > +/*Setting this bit enables FIFO DMA operation mode*/
> > +#define TRB_FIFO_MODE          BIT(3)
> > +/* Set PCIe no snoop attribute */
> > +#define TRB_CHAIN              BIT(4)
> > +/* Interrupt on completion */
> > +#define TRB_IOC                        BIT(5)
> > +
> > +/* stream ID bitmasks. */
> > +#define TRB_STREAM_ID(p)       ((p) & GENMASK(31, 16))
> > +
> > +/* transfer_len bitmasks. */
> > +#define TRB_LEN(p)             ((p) & GENMASK(16, 0))
> > +
> > +/* transfer_len bitmasks - bits 31:24 */
> > +#define TRB_BURST_LEN(p)       ((p) & GENMASK(31, 24))
> > +
> > +/* Data buffer pointer bitmasks*/
> > +#define TRB_BUFFER(p)          ((p) & GENMASK(31, 0))
> > +
> > +/*-------------------------------------------------------------------------*/
> > +/* Driver numeric constants */
> > +
> > +/* Such declaration should be added to ch9.h */
> > +#define USB_DEVICE_MAX_ADDRESS         127
> > +
> > +/* Endpoint init values */
> > +#define CDNS3_EP_MAX_PACKET_LIMIT      1024
> > +#define CDNS3_EP_MAX_STREAMS           15
> > +#define CDNS3_EP0_MAX_PACKET_LIMIT     512
> > +
> > +/* All endpoints including EP0 */
> > +#define CDNS3_ENDPOINTS_MAX_COUNT      32
> > +#define CDNS3_EP_ZLP_BUF_SIZE          1024
> > +
> > +#define CDNS3_EP_BUF_SIZE              2       /* KB */
> > +#define CDNS3_ALIGNED_BUF_SIZE         16384   /* Bytes */
> > +#define CDNS3_MAX_NUM_DESCMISS_BUF     32
> > +#define CDNS3_DESCMIS_BUF_SIZE         2048    /* Bytes */
> > +/*-------------------------------------------------------------------------*/
> > +/* Used structs */
> > +
> > +struct cdns3_device;
> > +
> > +/**
> > + * struct cdns3_endpoint - extended device side representation of USB endpoint.
> > + * @endpoint: usb endpoint
> > + * @pending_req_list: list of requests queuing on transfer ring.
> > + * @deferred_req_list: list of requests waiting for queuing on transfer ring.
> > + * @descmiss_req_list: list of requests internally allocated by driver (WA1).
> > + * @trb_pool: transfer ring - array of transaction buffers
> > + * @trb_pool_dma: dma address of transfer ring
> > + * @cdns3_dev: device associated with this endpoint
> > + * @name: a human readable name e.g. ep1out
> > + * @flags: specify the current state of endpoint
> > + * @descmis_req: internal transfer object used for getting data from on-chip
> > + *     buffer. It can happen only if function driver doesn't send usb_request
> > + *     object on time.
> > + * @aligned_buff: aligned to 8 bytes data buffer. Buffer address used in
> > + *     TRB shall be aligned to 8.
> > + * @aligned_dma_addr: dma address of aligned_buff
> > + * @dir: endpoint direction
> > + * @num: endpoint number (1 - 15)
> > + * @type: set to bmAttributes & USB_ENDPOINT_XFERTYPE_MASK
> > + * @interval: interval between packets used for ISOC endpoint.
> > + * @free_trbs: number of free TRBs in transfer ring
> > + * @num_trbs: number of all TRBs in transfer ring
> > + * @pcs: producer cycle state
> > + * @ccs: consumer cycle state
> > + * @enqueue: enqueue index in transfer ring
> > + * @dequeue: dequeue index in transfer ring
> > + */
> > +struct cdns3_endpoint {
> > +       struct usb_ep           endpoint;
> > +       struct list_head        pending_req_list;
> > +       struct list_head        deferred_req_list;
> > +       struct list_head        descmiss_req_list;
> > +
> > +       struct cdns3_trb        *trb_pool;
> > +       dma_addr_t              trb_pool_dma;
> > +
> > +       struct cdns3_device     *cdns3_dev;
> > +       char                    name[20];
> > +
> > +#define EP_ENABLED             BIT(0)
> > +#define EP_STALL               BIT(1)
> > +#define EP_WEDGE               BIT(2)
> > +#define EP_TRANSFER_STARTED    BIT(3)
> > +#define EP_UPDATE_EP_TRBADDR   BIT(4)
> > +#define EP_PENDING_REQUEST     BIT(5)
> > +#define EP_RING_FULL           BIT(6)
> > +#define EP_CLAIMED             BIT(7)
> > +#define EP_QUIRK_EXTRA_BUF_DET BIT(8)
> > +#define EP_QUIRK_EXTRA_BUF_EN  BIT(9)
> > +#define EP_QUIRK_END_TRANSFER  BIT(10)
> > +
> > +       u32                     flags;
> > +
> > +       struct cdns3_request    *descmis_req;
> > +
> > +       void                    *aligned_buff;
> > +       dma_addr_t              aligned_dma_addr;
> > +       u8                      dir;
> > +       u8                      num;
> > +       u8                      type;
> > +       int                     interval;
> > +
> > +       int                     free_trbs;
> > +       int                     num_trbs;
> > +       u8                      pcs;
> > +       u8                      ccs;
> > +       int                     enqueue;
> > +       int                     dequeue;
> > +};
> > +
> > +/**
> > + * struct cdns3_request - extended device side representation of usb_request
> > + *                        object .
> > + * @request: generic usb_request object describing single I/O request.
> > + * @priv_ep: extended representation of usb_ep object
> > + * @trb: the first TRB association with this request
> > + * @start_trb: number of the first TRB in transfer ring
> > + * @end_trb: number of the last TRB in transfer ring
> > + * @flags: flag specifying special usage of request
> > + * @list: used by internally allocated request to add to descmiss_req_list.
> > + */
> > +struct cdns3_request {
> > +       struct usb_request      request;
> > +       struct cdns3_endpoint   *priv_ep;
> > +       struct cdns3_trb        *trb;
> > +       int                     start_trb;
> > +       int                     end_trb;
> > +#define REQUEST_PENDING                BIT(0)
> > +#define REQUEST_INTERNAL       BIT(1)
> > +#define REQUEST_INTERNAL_CH    BIT(2)
> > +#define REQUEST_ZLP            BIT(3)
> > +       u32                     flags;
> > +       struct list_head        list;
> > +};
> > +
> > +#define to_cdns3_request(r) (container_of(r, struct cdns3_request, request))
> > +
> > +/*Stages used during enumeration process.*/
> > +#define CDNS3_SETUP_STAGE              0x0
> > +#define CDNS3_DATA_STAGE               0x1
> > +#define CDNS3_STATUS_STAGE             0x2
> > +
> > +/**
> > + * struct cdns3_device - represent USB device.
> > + * @dev: pointer to device structure associated whit this controller
> > + * @sysdev: pointer to the DMA capable device
> > + * @gadget: device side representation of the peripheral controller
> > + * @gadget_driver: pointer to the gadget driver
> > + * @dev_ver: device controller version.
> > + * @lock: for synchronizing
> > + * @regs: base address for device side registers
> > + * @setup_buf: used while processing usb control requests
> > + * @setup_dma: dma address for setup_buf
> > + * @zlp_buf - zlp buffer
> > + * @ep0_stage: ep0 stage during enumeration process.
> > + * @ep0_data_dir: direction for control transfer
> > + * @eps: array of pointers to all endpoints with exclusion ep0
> > + * @selected_ep: actually selected endpoint. It's used only to improve
> > + *               performance.
> > + * @isoch_delay: value from Set Isoch Delay request. Only valid on SS/SSP.
> > + * @u1_allowed: allow device transition to u1 state
> > + * @u2_allowed: allow device transition to u2 state
> > + * @is_selfpowered: device is self powered
> > + * @setup_pending: setup packet is processing by gadget driver
> > + * @hw_configured_flag: hardware endpoint configuration was set.
> > + * @wake_up_flag: allow device to remote up the host
> > + * @status_completion_no_call: indicate that driver is waiting for status s
> > + *     stage completion. It's used in deferred SET_CONFIGURATION request.
> > + * @onchip_mem_allocated_size: actual size of on-chip memory assigned
> > + *     to endpoints
> > + * @pending_status_wq: workqueue handling status stage for deferred requests.
> > + * @pending_status_request: request for which status stage was deferred
> > + */
> > +struct cdns3_device {
> > +       struct device                   *dev;
> > +       struct device                   *sysdev;
> > +
> > +       struct usb_gadget               gadget;
> > +       struct usb_gadget_driver        *gadget_driver;
> > +
> > +#define CDNS_REVISION_V0               0x00024501
> > +#define CDNS_REVISION_V1               0x00024509
> > +       u32                             dev_ver;
> > +
> > +       /* generic spin-lock for drivers */
> > +       spinlock_t                      lock;
> > +
> > +       struct cdns3_usb_regs           __iomem *regs;
> > +
> > +       struct usb_ctrlrequest          *setup_buf;
> > +       dma_addr_t                      setup_dma;
> > +       void                            *zlp_buf;
> > +
> > +       u8                              ep0_stage;
> > +       int                             ep0_data_dir;
> > +
> > +       struct cdns3_endpoint           *eps[CDNS3_ENDPOINTS_MAX_COUNT];
> > +
> > +       u32                             selected_ep;
> > +       u16                             isoch_delay;
> > +
> > +       unsigned                        u1_allowed:1;
> > +       unsigned                        u2_allowed:1;
> > +       unsigned                        is_selfpowered:1;
> > +       unsigned                        setup_pending:1;
> > +       int                             hw_configured_flag:1;
> > +       int                             wake_up_flag:1;
> > +       unsigned                        status_completion_no_call:1;
> > +
> > +       struct work_struct              pending_status_wq;
> > +       struct usb_request              *pending_status_request;
> > +
> > +       /*in KB */
> > +       int                             onchip_mem_allocated_size;
> > +};
> > +
> > +int cdns3_handshake(void __iomem *ptr, u32 mask, u32 done, int usec);
> > +void cdns3_set_register_bit(void __iomem *ptr, u32 mask);
> > +dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
> > +                                struct cdns3_trb *trb);
> > +enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev);
> > +void cdns3_pending_setup_status_handler(struct work_struct *work);
> > +void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev);
> > +void cdns3_set_hw_configuration(struct cdns3_device *priv_dev);
> > +void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep);
> > +void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable);
> > +struct usb_request *cdns3_next_request(struct list_head *list);
> > +int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
> > +                         struct usb_request *request);
> > +int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep);
> > +u8 cdns3_ep_addr_to_index(u8 ep_addr);
> > +int cdns3_gadget_ep_set_wedge(struct usb_ep *ep);
> > +int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value);
> > +struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep,
> > +                                                 gfp_t gfp_flags);
> > +void cdns3_gadget_ep_free_request(struct usb_ep *ep,
> > +                                 struct usb_request *request);
> > +int cdns3_gadget_ep_dequeue(struct usb_ep *ep, struct usb_request *request);
> > +void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
> > +                          struct cdns3_request *priv_req,
> > +                          int status);
> > +
> > +int cdns3_init_ep0(struct cdns3_device *priv_dev,
> > +                  struct cdns3_endpoint *priv_ep);
> > +void cdns3_ep0_config(struct cdns3_device *priv_dev);
> > +void cdns3_ep_config(struct cdns3_endpoint *priv_ep);
> > +void cdns3_check_ep0_interrupt_proceed(struct cdns3_device *priv_dev, int dir);
> > +
> > +#endif /* __LINUX_CDNS3_GADGET */
> > diff --git a/drivers/usb/cdns3/host-export.h b/drivers/usb/cdns3/host-export.h
> > new file mode 100644
> > index 000000000000..b498a170b7e8
> > --- /dev/null
> > +++ b/drivers/usb/cdns3/host-export.h
> > @@ -0,0 +1,28 @@
> > +/* SPDX-License-Identifier: GPL-2.0 */
> > +/*
> > + * Cadence USBSS DRD Driver - Host Export APIs
> > + *
> > + * Copyright (C) 2017-2018 NXP
> > + *
> > + * Authors: Peter Chen <peter.chen@xxxxxxx>
> > + */
> > +#ifndef __LINUX_CDNS3_HOST_EXPORT
> > +#define __LINUX_CDNS3_HOST_EXPORT
> > +
> > +#ifdef CONFIG_USB_CDNS3_HOST
> > +
> > +int cdns3_host_init(struct cdns3 *cdns);
> > +void cdns3_host_exit(struct cdns3 *cdns);
> > +
> > +#else
> > +
> > +static inline int cdns3_host_init(struct cdns3 *cdns)
> > +{
> > +       return -ENXIO;
> > +}
> > +
> > +static inline void cdns3_host_exit(struct cdns3 *cdns) { }
> > +
> > +#endif /* CONFIG_USB_CDNS3_HOST */
> > +
> > +#endif /* __LINUX_CDNS3_HOST_EXPORT */
> > diff --git a/drivers/usb/cdns3/host.c b/drivers/usb/cdns3/host.c
> > new file mode 100644
> > index 000000000000..b43b0236a885
> > --- /dev/null
> > +++ b/drivers/usb/cdns3/host.c
> > @@ -0,0 +1,72 @@
> > +// SPDX-License-Identifier: GPL-2.0
> > +/*
> > + * Cadence USBSS DRD Driver - host side
> > + *
> > + * Copyright (C) 2018 Cadence Design Systems.
> > + * Copyright (C) 2017-2018 NXP
> > + *
> > + * Authors: Peter Chen <peter.chen@xxxxxxx>
> > + *         Pawel Laszczak <pawell@xxxxxxxxxxx>
> > + */
> > +
> > +#include <linux/platform_device.h>
> > +#include "core.h"
> > +
> > +static int __cdns3_host_init(struct cdns3 *cdns)
> > +{
> > +       struct platform_device *xhci;
> > +       int ret;
> > +
> > +       xhci = platform_device_alloc("xhci-hcd", PLATFORM_DEVID_AUTO);
> > +       if (!xhci) {
> > +               dev_err(cdns->dev, "couldn't allocate xHCI device\n");
> > +               return -ENOMEM;
> > +       }
> > +
> > +       xhci->dev.parent = cdns->dev;
> > +       cdns->host_dev = xhci;
> > +
> > +       ret = platform_device_add_resources(xhci, cdns->xhci_res,
> > +                                           CDNS3_XHCI_RESOURCES_NUM);
> > +       if (ret) {
> > +               dev_err(cdns->dev, "couldn't add resources to xHCI device\n");
> > +               goto err1;
> > +       }
> > +
> > +       ret = platform_device_add(xhci);
> > +       if (ret) {
> > +               dev_err(cdns->dev, "failed to register xHCI device\n");
> > +               goto err1;
> > +       }
> > +
> > +       return 0;
> > +err1:
> > +       platform_device_put(xhci);
> > +       return ret;
> > +}
> > +
> > +static void cdns3_host_exit(struct cdns3 *cdns)
> > +{
> > +       platform_device_unregister(cdns->host_dev);
> > +       cdns->host_dev = NULL;
> > +}
> > +
> > +int cdns3_host_init(struct cdns3 *cdns)
> > +{
> > +       struct cdns3_role_driver *rdrv;
> > +
> > +       rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
> > +       if (!rdrv)
> > +               return -ENOMEM;
> > +
> > +       rdrv->start     = __cdns3_host_init;
> > +       rdrv->stop      = cdns3_host_exit;
> > +       rdrv->state     = CDNS3_ROLE_STATE_INACTIVE;
> > +       rdrv->suspend   = NULL;
> > +       rdrv->resume    = NULL;
> > +       rdrv->name      = "host";
> > +
> > +       cdns->roles[CDNS3_ROLE_HOST] = rdrv;
> > +
> > +       return 0;
> > +}
> > diff --git a/drivers/usb/cdns3/trace.c b/drivers/usb/cdns3/trace.c
> > new file mode 100644
> > index 000000000000..587ae08e019d
> > --- /dev/null
> > +++ b/drivers/usb/cdns3/trace.c
> > @@ -0,0 +1,11 @@
> > +// SPDX-License-Identifier: GPL-2.0
> > +/*
> > + * USBSS device controller driver Trace Support
> > + *
> > + * Copyright (C) 2018 Cadence.
> > + *
> > + * Author: Pawel Laszczak <pawell@xxxxxxxxxxx>
> > + */
> > +
> > +#define CREATE_TRACE_POINTS
> > +#include "trace.h"
> > diff --git a/drivers/usb/cdns3/trace.h b/drivers/usb/cdns3/trace.h
> > new file mode 100644
> > index 000000000000..f3d7a91fae86
> > --- /dev/null
> > +++ b/drivers/usb/cdns3/trace.h
> > @@ -0,0 +1,389 @@
> > +/* SPDX-License-Identifier: GPL-2.0 */
> > +/*
> > + * USBSS device controller driver.
> > + * Trace support header file.
> > + *
> > + * Copyright (C) 2018 Cadence.
> > + *
> > + * Author: Pawel Laszczak <pawell@xxxxxxxxxxx>
> > + */
> > +
> > +#undef TRACE_SYSTEM
> > +#define TRACE_SYSTEM cdns3
> > +
> > +#if !defined(__LINUX_CDNS3_TRACE) || defined(TRACE_HEADER_MULTI_READ)
> > +#define __LINUX_CDNS3_TRACE
> > +
> > +#include <linux/types.h>
> > +#include <linux/tracepoint.h>
> > +#include <asm/byteorder.h>
> > +#include <linux/usb/ch9.h>
> > +#include "core.h"
> > +#include "gadget.h"
> > +#include "debug.h"
> > +
> > +#define CDNS3_MSG_MAX  500
> > +
> > +DECLARE_EVENT_CLASS(cdns3_log_doorbell,
> > +       TP_PROTO(const char *ep_name, u32 ep_trbaddr),
> > +       TP_ARGS(ep_name, ep_trbaddr),
> > +       TP_STRUCT__entry(
> > +               __string(name, ep_name)
> > +               __field(u32, ep_trbaddr)
> > +       ),
> > +       TP_fast_assign(
> > +               __assign_str(name, ep_name);
> > +               __entry->ep_trbaddr = ep_trbaddr;
> > +       ),
> > +       TP_printk("//Ding Dong %s, ep_trbaddr %08x", __get_str(name),
> > +                 __entry->ep_trbaddr)
> > +);
> > +
> > +DEFINE_EVENT(cdns3_log_doorbell, cdns3_doorbell_ep0,
> > +       TP_PROTO(const char *ep_name, u32 ep_trbaddr),
> > +       TP_ARGS(ep_name, ep_trbaddr)
> > +);
> > +
> > +DEFINE_EVENT(cdns3_log_doorbell, cdns3_doorbell_epx,
> > +       TP_PROTO(const char *ep_name, u32 ep_trbaddr),
> > +       TP_ARGS(ep_name, ep_trbaddr)
> > +);
> > +
> > +DECLARE_EVENT_CLASS(cdns3_log_usb_irq,
> > +       TP_PROTO(struct cdns3_device *priv_dev, u32 usb_ists),
> > +       TP_ARGS(priv_dev, usb_ists),
> > +       TP_STRUCT__entry(
> > +               __field(enum usb_device_speed, speed)
> > +               __field(u32, usb_ists)
> > +               __dynamic_array(char, str, CDNS3_MSG_MAX)
> > +       ),
> > +       TP_fast_assign(
> > +               __entry->speed = cdns3_get_speed(priv_dev);
> > +               __entry->usb_ists = usb_ists;
> > +       ),
> > +       TP_printk("%s", cdns3_decode_usb_irq(__get_str(str), __entry->speed,
> > +                                            __entry->usb_ists))
> > +);
> > +
> > +DEFINE_EVENT(cdns3_log_usb_irq, cdns3_usb_irq,
> > +       TP_PROTO(struct cdns3_device *priv_dev, u32 usb_ists),
> > +       TP_ARGS(priv_dev, usb_ists)
> > +);
> > +
> > +DECLARE_EVENT_CLASS(cdns3_log_epx_irq,
> > +       TP_PROTO(struct cdns3_device *priv_dev, struct cdns3_endpoint *priv_ep),
> > +       TP_ARGS(priv_dev, priv_ep),
> > +       TP_STRUCT__entry(
> > +               __string(ep_name, priv_ep->name)
> > +               __field(u32, ep_sts)
> > +               __field(u32, ep_traddr)
> > +               __dynamic_array(char, str, CDNS3_MSG_MAX)
> > +       ),
> > +       TP_fast_assign(
> > +               __assign_str(ep_name, priv_ep->name);
> > +               __entry->ep_sts = readl(&priv_dev->regs->ep_sts);
> > +               __entry->ep_traddr = readl(&priv_dev->regs->ep_traddr);
> > +       ),
> > +       TP_printk("%s, ep_traddr: %08x",
> > +                 cdns3_decode_epx_irq(__get_str(str),
> > +                                      __get_str(ep_name),
> > +                                      __entry->ep_sts),
> > +                 __entry->ep_traddr)
> > +);
> > +
> > +DEFINE_EVENT(cdns3_log_epx_irq, cdns3_epx_irq,
> > +       TP_PROTO(struct cdns3_device *priv_dev, struct cdns3_endpoint *priv_ep),
> > +       TP_ARGS(priv_dev, priv_ep)
> > +);
> > +
> > +DECLARE_EVENT_CLASS(cdns3_log_ep0_irq,
> > +       TP_PROTO(struct cdns3_device *priv_dev,  u32 ep_sts),
> > +       TP_ARGS(priv_dev, ep_sts),
> > +       TP_STRUCT__entry(
> > +               __field(int, ep_dir)
> > +               __field(u32, ep_sts)
> > +               __dynamic_array(char, str, CDNS3_MSG_MAX)
> > +       ),
> > +       TP_fast_assign(
> > +               __entry->ep_dir = priv_dev->ep0_data_dir;
> > +               __entry->ep_sts = ep_sts;
> > +       ),
> > +       TP_printk("%s", cdns3_decode_ep0_irq(__get_str(str),
> > +                                            __entry->ep_dir,
> > +                                            __entry->ep_sts))
> > +);
> > +
> > +DEFINE_EVENT(cdns3_log_ep0_irq, cdns3_ep0_irq,
> > +       TP_PROTO(struct cdns3_device *priv_dev, u32 ep_sts),
> > +       TP_ARGS(priv_dev, ep_sts)
> > +);
> > +
> > +DECLARE_EVENT_CLASS(cdns3_log_ctrl,
> > +       TP_PROTO(struct usb_ctrlrequest *ctrl),
> > +       TP_ARGS(ctrl),
> > +       TP_STRUCT__entry(
> > +               __field(u8, bRequestType)
> > +               __field(u8, bRequest)
> > +               __field(u16, wValue)
> > +               __field(u16, wIndex)
> > +               __field(u16, wLength)
> > +               __dynamic_array(char, str, CDNS3_MSG_MAX)
> > +       ),
> > +       TP_fast_assign(
> > +               __entry->bRequestType = ctrl->bRequestType;
> > +               __entry->bRequest = ctrl->bRequest;
> > +               __entry->wValue = le16_to_cpu(ctrl->wValue);
> > +               __entry->wIndex = le16_to_cpu(ctrl->wIndex);
> > +               __entry->wLength = le16_to_cpu(ctrl->wLength);
> > +       ),
> > +       TP_printk("%s", usb_decode_ctrl(__get_str(str), __entry->bRequestType,
> > +                                       __entry->bRequest, __entry->wValue,
> > +                                       __entry->wIndex, __entry->wLength)
> > +       )
> > +);
> > +
> > +DEFINE_EVENT(cdns3_log_ctrl, cdns3_ctrl_req,
> > +       TP_PROTO(struct usb_ctrlrequest *ctrl),
> > +       TP_ARGS(ctrl)
> > +);
> > +
> > +DECLARE_EVENT_CLASS(cdns3_log_request,
> > +       TP_PROTO(struct cdns3_request *req),
> > +       TP_ARGS(req),
> > +       TP_STRUCT__entry(
> > +               __string(name, req->priv_ep->name)
> > +               __field(struct cdns3_request *, req)
> > +               __field(unsigned int, actual)
> > +               __field(unsigned int, length)
> > +               __field(int, status)
> > +               __field(int, zero)
> > +               __field(int, short_not_ok)
> > +               __field(int, no_interrupt)
> > +               __field(int, start_trb)
> > +               __field(int, end_trb)
> > +               __field(struct cdns3_trb *, start_trb_addr)
> > +               __field(int, flags)
> > +       ),
> > +       TP_fast_assign(
> > +               __assign_str(name, req->priv_ep->name);
> > +               __entry->req = req;
> > +               __entry->actual = req->request.actual;
> > +               __entry->length = req->request.length;
> > +               __entry->status = req->request.status;
> > +               __entry->zero = req->request.zero;
> > +               __entry->short_not_ok = req->request.short_not_ok;
> > +               __entry->no_interrupt = req->request.no_interrupt;
> > +               __entry->start_trb = req->start_trb;
> > +               __entry->end_trb = req->end_trb;
> > +               __entry->start_trb_addr = req->trb;
> > +               __entry->flags = req->flags;
> > +       ),
> > +       TP_printk("%s: req: %p, length: %u/%u %s%s%s, status: %d,"
> > +                 " trb: [start:%d, end:%d: virt addr %pa], flags:%x ",
> > +               __get_str(name), __entry->req, __entry->actual, __entry->length,
> > +               __entry->zero ? "zero | " : "",
> > +               __entry->short_not_ok ? "short | " : "",
> > +               __entry->no_interrupt ? "no int" : "",
> > +               __entry->status,
> > +               __entry->start_trb,
> > +               __entry->end_trb,
> > +               __entry->start_trb_addr,
> > +               __entry->flags
> > +       )
> > +);
> > +
> > +DEFINE_EVENT(cdns3_log_request, cdns3_alloc_request,
> > +       TP_PROTO(struct cdns3_request *req),
> > +       TP_ARGS(req)
> > +);
> > +
> > +DEFINE_EVENT(cdns3_log_request, cdns3_free_request,
> > +       TP_PROTO(struct cdns3_request *req),
> > +       TP_ARGS(req)
> > +);
> > +
> > +DEFINE_EVENT(cdns3_log_request, cdns3_ep_queue,
> > +       TP_PROTO(struct cdns3_request *req),
> > +       TP_ARGS(req)
> > +);
> > +
> > +DEFINE_EVENT(cdns3_log_request, cdns3_ep_dequeue,
> > +       TP_PROTO(struct cdns3_request *req),
> > +       TP_ARGS(req)
> > +);
> > +
> > +DEFINE_EVENT(cdns3_log_request, cdns3_gadget_giveback,
> > +       TP_PROTO(struct cdns3_request *req),
> > +       TP_ARGS(req)
> > +);
> > +
> > +DECLARE_EVENT_CLASS(cdns3_log_trb,
> > +       TP_PROTO(struct cdns3_endpoint *priv_ep, struct cdns3_trb *trb),
> > +       TP_ARGS(priv_ep, trb),
> > +       TP_STRUCT__entry(
> > +               __string(name, priv_ep->name)
> > +               __field(struct cdns3_trb *, trb)
> > +               __field(u32, buffer)
> > +               __field(u32, length)
> > +               __field(u32, control)
> > +               __field(u32, type)
> > +       ),
> > +       TP_fast_assign(
> > +               __assign_str(name, priv_ep->name);
> > +               __entry->trb = trb;
> > +               __entry->buffer = trb->buffer;
> > +               __entry->length = trb->length;
> > +               __entry->control = trb->control;
> > +               __entry->type = usb_endpoint_type(priv_ep->endpoint.desc);
> > +       ),
> > +       TP_printk("%s: trb 0x%pa, dma buf: 0x%08x, size: %ld, ctrl: 0x%08x (%s%s%s%s%s%s%s)",
> > +               __get_str(name), __entry->trb, __entry->buffer,
> > +               TRB_LEN(__entry->length), __entry->control,
> > +               __entry->control & TRB_CYCLE ? "C=1, " : "C=0, ",
> > +               __entry->control & TRB_TOGGLE ? "T=1, " : "T=0, ",
> > +               __entry->control & TRB_ISP ? "ISP, " : "",
> > +               __entry->control & TRB_FIFO_MODE ? "FIFO, " : "",
> > +               __entry->control & TRB_CHAIN ? "CHAIN, " : "",
> > +               __entry->control & TRB_IOC ? "IOC, " : "",
> > +               TRB_FIELD_TO_TYPE(__entry->control) == TRB_NORMAL ? "Normal" : "LINK"
> > +       )
> > +);
> > +
> > +DEFINE_EVENT(cdns3_log_trb, cdns3_prepare_trb,
> > +       TP_PROTO(struct cdns3_endpoint *priv_ep, struct cdns3_trb *trb),
> > +       TP_ARGS(priv_ep, trb)
> > +);
> > +
> > +DEFINE_EVENT(cdns3_log_trb, cdns3_complete_trb,
> > +       TP_PROTO(struct cdns3_endpoint *priv_ep, struct cdns3_trb *trb),
> > +       TP_ARGS(priv_ep, trb)
> > +);
> > +
> > +DECLARE_EVENT_CLASS(cdns3_log_ring,
> > +       TP_PROTO(struct cdns3_endpoint *priv_ep),
> > +       TP_ARGS(priv_ep),
> > +       TP_STRUCT__entry(
> > +               __dynamic_array(u8, ring, TRB_RING_SIZE)
> > +               __dynamic_array(u8, priv_ep, sizeof(struct cdns3_endpoint))
> > +               __dynamic_array(char, buffer,
> > +                               (TRBS_PER_SEGMENT * 65) + CDNS3_MSG_MAX)
> > +       ),
> > +       TP_fast_assign(
> > +               memcpy(__get_dynamic_array(priv_ep), priv_ep,
> > +                      sizeof(struct cdns3_endpoint));
> > +               memcpy(__get_dynamic_array(ring), priv_ep->trb_pool,
> > +                      TRB_RING_SIZE);
> > +       ),
> > +
> > +       TP_printk("%s",
> > +                 cdns3_dbg_ring((struct cdns3_endpoint *)__get_str(priv_ep),
> > +                                (struct cdns3_trb *)__get_str(ring),
> > +                                __get_str(buffer)))
> > +);
> > +
> > +DEFINE_EVENT(cdns3_log_ring, cdns3_ring,
> > +       TP_PROTO(struct cdns3_endpoint *priv_ep),
> > +       TP_ARGS(priv_ep)
> > +);
> > +
> > +DECLARE_EVENT_CLASS(cdns3_log_ep,
> > +       TP_PROTO(struct cdns3_endpoint *priv_ep),
> > +       TP_ARGS(priv_ep),
> > +       TP_STRUCT__entry(
> > +               __string(name, priv_ep->name)
> > +               __field(unsigned int, maxpacket)
> > +               __field(unsigned int, maxpacket_limit)
> > +               __field(unsigned int, max_streams)
> > +               __field(unsigned int, maxburst)
> > +               __field(unsigned int, flags)
> > +               __field(unsigned int, dir)
> > +               __field(u8, enqueue)
> > +               __field(u8, dequeue)
> > +       ),
> > +       TP_fast_assign(
> > +               __assign_str(name, priv_ep->name);
> > +               __entry->maxpacket = priv_ep->endpoint.maxpacket;
> > +               __entry->maxpacket_limit = priv_ep->endpoint.maxpacket_limit;
> > +               __entry->max_streams = priv_ep->endpoint.max_streams;
> > +               __entry->maxburst = priv_ep->endpoint.maxburst;
> > +               __entry->flags = priv_ep->flags;
> > +               __entry->dir = priv_ep->dir;
> > +               __entry->enqueue = priv_ep->enqueue;
> > +               __entry->dequeue = priv_ep->dequeue;
> > +       ),
> > +       TP_printk("%s: mps: %d/%d. streams: %d, burst: %d, enq idx: %d, "
> > +                 "deq idx: %d, flags %s%s%s%s%s%s%s%s, dir: %s",
> > +               __get_str(name), __entry->maxpacket,
> > +               __entry->maxpacket_limit, __entry->max_streams,
> > +               __entry->maxburst, __entry->enqueue,
> > +               __entry->dequeue,
> > +               __entry->flags & EP_ENABLED ? "EN | " : "",
> > +               __entry->flags & EP_STALL ? "STALL | " : "",
> > +               __entry->flags & EP_WEDGE ? "WEDGE | " : "",
> > +               __entry->flags & EP_TRANSFER_STARTED ? "STARTED | " : "",
> > +               __entry->flags & EP_UPDATE_EP_TRBADDR ? "UPD TRB | " : "",
> > +               __entry->flags & EP_PENDING_REQUEST ? "REQ PEN | " : "",
> > +               __entry->flags & EP_RING_FULL ? "RING FULL |" : "",
> > +               __entry->flags & EP_CLAIMED ?  "CLAIMED " : "",
> > +               __entry->dir ? "IN" : "OUT"
> > +       )
> > +);
> > +
> > +DEFINE_EVENT(cdns3_log_ep, cdns3_gadget_ep_enable,
> > +       TP_PROTO(struct cdns3_endpoint *priv_ep),
> > +       TP_ARGS(priv_ep)
> > +);
> > +
> > +DEFINE_EVENT(cdns3_log_ep, cdns3_gadget_ep_disable,
> > +       TP_PROTO(struct cdns3_endpoint *priv_ep),
> > +       TP_ARGS(priv_ep)
> > +);
> > +
> > +DECLARE_EVENT_CLASS(cdns3_log_request_handled,
> > +       TP_PROTO(struct cdns3_request *priv_req, int current_index,
> > +                int handled),
> > +       TP_ARGS(priv_req, current_index, handled),
> > +       TP_STRUCT__entry(
> > +               __field(struct cdns3_request *, priv_req)
> > +               __field(unsigned int, dma_position)
> > +               __field(unsigned int, handled)
> > +               __field(unsigned int, dequeue_idx)
> > +               __field(unsigned int, enqueue_idx)
> > +               __field(unsigned int, start_trb)
> > +               __field(unsigned int, end_trb)
> > +       ),
> > +       TP_fast_assign(
> > +               __entry->priv_req = priv_req;
> > +               __entry->dma_position = current_index;
> > +               __entry->handled = handled;
> > +               __entry->dequeue_idx = priv_req->priv_ep->dequeue;
> > +               __entry->enqueue_idx = priv_req->priv_ep->enqueue;
> > +               __entry->start_trb = priv_req->start_trb;
> > +               __entry->end_trb = priv_req->end_trb;
> > +       ),
> > +       TP_printk("Req: %p %s, DMA pos: %d, ep deq: %d, ep enq: %d,"
> > +                 " start trb: %d, end trb: %d",
> > +               __entry->priv_req,
> > +               __entry->handled ? "handled" : "not handled",
> > +               __entry->dma_position, __entry->dequeue_idx,
> > +               __entry->enqueue_idx, __entry->start_trb,
> > +               __entry->end_trb
> > +       )
> > +);
> > +
> > +DEFINE_EVENT(cdns3_log_request_handled, cdns3_request_handled,
> > +       TP_PROTO(struct cdns3_request *priv_req, int current_index,
> > +                int handled),
> > +       TP_ARGS(priv_req, current_index, handled)
> > +);
> > +#endif /* __LINUX_CDNS3_TRACE */
> > +
> > +/* this part must be outside header guard */
> > +
> > +#undef TRACE_INCLUDE_PATH
> > +#define TRACE_INCLUDE_PATH .
> > +
> > +#undef TRACE_INCLUDE_FILE
> > +#define TRACE_INCLUDE_FILE trace
> > +
> > +#include <trace/define_trace.h>
> > --
> > 2.17.1
> >




[Index of Archives]     [Linux Media]     [Linux Input]     [Linux Audio Users]     [Yosemite News]     [Linux Kernel]     [Linux SCSI]     [Old Linux USB Devel Archive]

  Powered by Linux