+
> +static int pvrdma_pci_probe(struct pci_dev *pdev,
> + const struct pci_device_id *id)
> +{
> + struct pci_dev *pdev_net;
> + struct pvrdma_dev *dev;
> + int ret;
> + unsigned long start;
> + unsigned long len;
> + unsigned int version;
> + dma_addr_t slot_dma = 0;
> +
> + dev_dbg(&pdev->dev, "initializing driver %s\n", pci_name(pdev));
> +
> + /* Allocate zero-out device */
> + dev = (struct pvrdma_dev *)ib_alloc_device(sizeof(*dev));
> + if (!dev) {
> + dev_err(&pdev->dev, "failed to allocate IB device\n");
> + return -ENOMEM;
> + }
> +
> + mutex_lock(&pvrdma_device_list_lock);
> + list_add(&dev->device_link, &pvrdma_device_list);
> + mutex_unlock(&pvrdma_device_list_lock);
> +
> + ret = pvrdma_init_device(dev);
> + if (ret)
> + goto err_free_device;
> +
> + dev->pdev = pdev;
> + pci_set_drvdata(pdev, dev);
> +
> + ret = pci_enable_device(pdev);
> + if (ret) {
> + dev_err(&pdev->dev, "cannot enable PCI device\n");
> + goto err_free_device;
> + }
> +
> + dev_dbg(&pdev->dev, "PCI resource flags BAR0 %#lx\n",
> + pci_resource_flags(pdev, 0));
> + dev_dbg(&pdev->dev, "PCI resource len %#llx\n",
> + (unsigned long long)pci_resource_len(pdev, 0));
> + dev_dbg(&pdev->dev, "PCI resource start %#llx\n",
> + (unsigned long long)pci_resource_start(pdev, 0));
> + dev_dbg(&pdev->dev, "PCI resource flags BAR1 %#lx\n",
> + pci_resource_flags(pdev, 1));
> + dev_dbg(&pdev->dev, "PCI resource len %#llx\n",
> + (unsigned long long)pci_resource_len(pdev, 1));
> + dev_dbg(&pdev->dev, "PCI resource start %#llx\n",
> + (unsigned long long)pci_resource_start(pdev, 1));
> +
> + if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) ||
> + !(pci_resource_flags(pdev, 1) & IORESOURCE_MEM)) {
> + dev_err(&pdev->dev, "PCI BAR region not MMIO\n");
> + ret = -ENOMEM;
> + goto err_free_device;
> + }
> +
> + ret = pci_request_regions(pdev, DRV_NAME);
> + if (ret) {
> + dev_err(&pdev->dev, "cannot request PCI resources\n");
> + goto err_disable_pdev;
> + }
> +
> + /* Enable 64-Bit DMA */
> + if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) == 0) {
> + ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
> + if (ret != 0) {
> + dev_err(&pdev->dev,
> + "pci_set_consistent_dma_mask failed\n");
> + goto err_free_resource;
> + }
> + } else {
> + ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
> + if (ret != 0) {
> + dev_err(&pdev->dev,
> + "pci_set_dma_mask failed\n");
> + goto err_free_resource;
> + }
> + }
> +
> + pci_set_master(pdev);
> +
> + /* Map register space */
> + start = pci_resource_start(dev->pdev, PVRDMA_PCI_RESOURCE_REG);
> + len = pci_resource_len(dev->pdev, PVRDMA_PCI_RESOURCE_REG);
> + dev->regs = ioremap(start, len);
> + if (!dev->regs) {
> + dev_err(&pdev->dev, "register mapping failed\n");
> + ret = -ENOMEM;
> + goto err_free_resource;
> + }
> +
> + /* Setup per-device UAR. */
> + dev->driver_uar.index = 0;
> + dev->driver_uar.pfn =
> + pci_resource_start(dev->pdev, PVRDMA_PCI_RESOURCE_UAR) >>
> + PAGE_SHIFT;
> + dev->driver_uar.map =
> + ioremap(dev->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE);
> + if (!dev->driver_uar.map) {
> + dev_err(&pdev->dev, "failed to remap UAR pages\n");
> + ret = -ENOMEM;
> + goto err_unmap_regs;
> + }
> +
> + version = pvrdma_read_reg(dev, PVRDMA_REG_VERSION);
> + dev_info(&pdev->dev, "device version %d, driver version %d\n",
> + version, PVRDMA_VERSION);
> + if (version < PVRDMA_VERSION) {
> + dev_err(&pdev->dev, "incompatible device version\n");
> + goto err_uar_unmap;
> + }
> +
> + dev->dsr = dma_alloc_coherent(&pdev->dev, sizeof(*dev->dsr),
> + &dev->dsrbase, GFP_KERNEL);
> + if (!dev->dsr) {
> + dev_err(&pdev->dev, "failed to allocate shared region\n");
> + ret = -ENOMEM;
> + goto err_uar_unmap;
> + }
> +
> + /* Setup the shared region */
> + memset(dev->dsr, 0, sizeof(*dev->dsr));
> + dev->dsr->driver_version = PVRDMA_VERSION;
> + dev->dsr->gos_info.gos_bits = sizeof(void *) == 4 ?
> + PVRDMA_GOS_BITS_32 :
> + PVRDMA_GOS_BITS_64;
> + dev->dsr->gos_info.gos_type = PVRDMA_GOS_TYPE_LINUX;
> + dev->dsr->gos_info.gos_ver = 1;
> + dev->dsr->uar_pfn = dev->driver_uar.pfn;
> +
> + /* Command slot. */
> + dev->cmd_slot = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
> + &slot_dma, GFP_KERNEL);
> + if (!dev->cmd_slot) {
> + ret = -ENOMEM;
> + goto err_free_dsr;
> + }
> +
> + dev->dsr->cmd_slot_dma = (u64)slot_dma;
> +
> + /* Response slot. */
> + dev->resp_slot = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
> + &slot_dma, GFP_KERNEL);
> + if (!dev->resp_slot) {
> + ret = -ENOMEM;
> + goto err_free_slots;
> + }
> +
> + dev->dsr->resp_slot_dma = (u64)slot_dma;
> +
> + /* Async event ring */
> + dev->dsr->async_ring_pages.num_pages = 4;
Why 4?
Actually question is why driver hard-coded this value as apposed to other
HW resources rings (such as CQ, QP) that is calculated based on element
size and max element number that supported by the HW?
> + ret = pvrdma_page_dir_init(dev, &dev->async_pdir,
> + dev->dsr->async_ring_pages.num_pages, true);
> + if (ret)
> + goto err_free_slots;
> + dev->async_ring_state = dev->async_pdir.pages[0];
> + dev->dsr->async_ring_pages.pdir_dma = dev->async_pdir.dir_dma;
> +
> + /* CQ notification ring */
> + dev->dsr->cq_ring_pages.num_pages = 4;
Ditto.
> + ret = pvrdma_page_dir_init(dev, &dev->cq_pdir,
> + dev->dsr->cq_ring_pages.num_pages, true);
> + if (ret)
> + goto err_free_async_ring;
> + dev->cq_ring_state = dev->cq_pdir.pages[0];
> + dev->dsr->cq_ring_pages.pdir_dma = dev->cq_pdir.dir_dma;
> +
> + /*
> + * Write the PA of the shared region to the device. The writes must be
> + * ordered such that the high bits are written last. When the writes
> + * complete, the device will have filled out the capabilities.
> + */
> +
> + pvrdma_write_reg(dev, PVRDMA_REG_DSRLOW, (u32)dev->dsrbase);
> + pvrdma_write_reg(dev, PVRDMA_REG_DSRHIGH,
> + (u32)((u64)(dev->dsrbase) >> 32));
> +
> + /* Make sure the write is complete before reading status. */
> + mb();
> +
> + /* Currently, the driver only supports RoCE mode. */
> + if (dev->dsr->caps.mode != PVRDMA_DEVICE_MODE_ROCE) {
> + dev_err(&pdev->dev, "unsupported transport %d\n",
> + dev->dsr->caps.mode);
> + ret = -EFAULT;
> + goto err_free_cq_ring;
> + }
> +
> + /* Currently, the driver only supports RoCE V1. */
> + if (!(dev->dsr->caps.gid_types & PVRDMA_GID_TYPE_FLAG_ROCE_V1)) {
> + dev_err(&pdev->dev, "driver needs RoCE v1 support\n");
> + ret = -EFAULT;
> + goto err_free_cq_ring;
> + }
> +
> + /* Paired vmxnet3 will have same bus, slot. But func will be 0 */
> + pdev_net = pci_get_slot(pdev->bus, PCI_DEVFN(PCI_SLOT(pdev->devfn), 0));
> + if (!pdev_net) {
> + dev_err(&pdev->dev, "failed to find paired net device\n");
> + ret = -ENODEV;
> + goto err_free_cq_ring;
> + }
> +
> + if (pdev_net->vendor != PCI_VENDOR_ID_VMWARE ||
> + pdev_net->device != PCI_DEVICE_ID_VMWARE_VMXNET3) {
> + dev_err(&pdev->dev, "failed to find paired vmxnet3 device\n");
> + pci_dev_put(pdev_net);
> + ret = -ENODEV;
> + goto err_free_cq_ring;
> + }
> +
> + dev->netdev = pci_get_drvdata(pdev_net);
> + pci_dev_put(pdev_net);
> + if (!dev->netdev) {
> + dev_err(&pdev->dev, "failed to get vmxnet3 device\n");
> + ret = -ENODEV;
> + goto err_free_cq_ring;
> + }
> +
> + dev_info(&pdev->dev, "paired device to %s\n", dev->netdev->name);
> +
> + /* Interrupt setup */
> + ret = pvrdma_alloc_intrs(dev);
> + if (ret) {
> + dev_err(&pdev->dev, "failed to allocate interrupts\n");
> + ret = -ENOMEM;
> + goto err_netdevice;
> + }
> +
> + /* Allocate UAR table. */
> + ret = pvrdma_uar_table_init(dev);
> + if (ret) {
> + dev_err(&pdev->dev, "failed to allocate UAR table\n");
> + ret = -ENOMEM;
> + goto err_free_intrs;
> + }
> +
> + /* Allocate GID table */
> + dev->sgid_tbl = kcalloc(dev->dsr->caps.gid_tbl_len,
> + sizeof(union ib_gid), GFP_KERNEL);
> + if (!dev->sgid_tbl) {
> + ret = -ENOMEM;
> + goto err_free_uar_table;
> + }
> + dev_dbg(&pdev->dev, "gid table len %d\n", dev->dsr->caps.gid_tbl_len);
> +
> + pvrdma_enable_intrs(dev);
> +
> + /* Activate pvrdma device */
> + pvrdma_write_reg(dev, PVRDMA_REG_CTL, PVRDMA_DEVICE_CTL_ACTIVATE);
> +
> + /* Make sure the write is complete before reading status. */
> + mb();
> +
> + /* Check if device was successfully activated */
> + ret = pvrdma_read_reg(dev, PVRDMA_REG_ERR);
> + if (ret != 0) {
> + dev_err(&pdev->dev, "failed to activate device\n");
> + ret = -EFAULT;
> + goto err_disable_intr;
> + }
> +
> + /* Register IB device */
> + ret = pvrdma_register_device(dev);
> + if (ret) {
> + dev_err(&pdev->dev, "failed to register IB device\n");
> + goto err_disable_intr;
> + }
> +
> + dev->nb_netdev.notifier_call = pvrdma_netdevice_event;
> + ret = register_netdevice_notifier(&dev->nb_netdev);
> + if (ret) {
> + dev_err(&pdev->dev, "failed to register netdevice events\n");
> + goto err_unreg_ibdev;
> + }
> +
> + dev_info(&pdev->dev, "attached to device\n");
> + return 0;
> +
> +err_unreg_ibdev:
> + ib_unregister_device(&dev->ib_dev);
> +err_disable_intr:
> + pvrdma_disable_intrs(dev);
> + kfree(dev->sgid_tbl);
> +err_free_uar_table:
> + pvrdma_uar_table_cleanup(dev);
> +err_free_intrs:
> + pvrdma_free_irq(dev);
> + pvrdma_disable_msi_all(dev);
> +err_netdevice:
> + unregister_netdevice_notifier(&dev->nb_netdev);
> +err_free_cq_ring:
> + pvrdma_page_dir_cleanup(dev, &dev->cq_pdir);
> +err_free_async_ring:
> + pvrdma_page_dir_cleanup(dev, &dev->async_pdir);
> +err_free_slots:
> + pvrdma_free_slots(dev);
> +err_free_dsr:
> + dma_free_coherent(&pdev->dev, sizeof(*dev->dsr), dev->dsr,
> + dev->dsrbase);
> +err_uar_unmap:
> + iounmap(dev->driver_uar.map);
> +err_unmap_regs:
> + iounmap(dev->regs);
> +err_free_resource:
> + pci_release_regions(pdev);
> +err_disable_pdev:
> + pci_disable_device(pdev);
> + pci_set_drvdata(pdev, NULL);
> +err_free_device:
> + mutex_lock(&pvrdma_device_list_lock);
> + list_del(&dev->device_link);
> + mutex_unlock(&pvrdma_device_list_lock);
> + ib_dealloc_device(&dev->ib_dev);
> + return ret;
> +}
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
