On 26-Feb-19 21:28, Leon Romanovsky wrote:
> On Thu, Feb 21, 2019 at 05:33:12PM +0200, Gal Pressman wrote:
>> Add the main EFA module file which takes care of device
>> probe/initialization/registration/etc.
>>
>> Signed-off-by: Gal Pressman <galpress@xxxxxxxxxx>
>> ---
>> drivers/infiniband/hw/efa/efa_main.c | 577 +++++++++++++++++++++++++++++++++++
>> 1 file changed, 577 insertions(+)
>> create mode 100644 drivers/infiniband/hw/efa/efa_main.c
>>
>> diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c
>> new file mode 100644
>> index 000000000000..a4ccebc9665d
>> --- /dev/null
>> +++ b/drivers/infiniband/hw/efa/efa_main.c
>> @@ -0,0 +1,577 @@
>> +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
>
> Are you sure that this is the license and Linux-OpenIB?
What do you mean?
>
>> +/*
>> + * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
>> + */
>> +
>> +
>> +static const struct ib_device_ops efa_dev_ops = {
>> + .alloc_pd = efa_alloc_pd,
>> + .alloc_ucontext = efa_alloc_ucontext,
>> + .create_ah = efa_create_ah,
>> + .create_cq = efa_create_cq,
>> + .create_qp = efa_create_qp,
>> + .dealloc_pd = efa_dealloc_pd,
>> + .dealloc_ucontext = efa_dealloc_ucontext,
>> + .dereg_mr = efa_dereg_mr,
>> + .destroy_ah = efa_destroy_ah,
>> + .destroy_cq = efa_destroy_cq,
>> + .destroy_qp = efa_destroy_qp,
>> + .get_link_layer = efa_port_link_layer,
>> + .get_port_immutable = efa_get_port_immutable,
>> + .mmap = efa_mmap,
>> + .modify_qp = efa_modify_qp,
>> + .query_device = efa_query_device,
>> + .query_gid = efa_query_gid,
>> + .query_pkey = efa_query_pkey,
>> + .query_port = efa_query_port,
>> + .query_qp = efa_query_qp,
>> + .reg_user_mr = efa_reg_mr,
>> + INIT_RDMA_OBJ_SIZE(ib_pd, efa_pd, ibpd),
>
> ucontext needs this INIT_RDMA_OBJ_SIZE() too.
Trying to keep up with these changes :).
>
>> +};
>> +
>> +static int efa_probe_device(struct pci_dev *pdev)
>> +{
>> + struct efa_com_dev *edev;
>> + struct efa_dev *dev;
>> + int bars;
>> + int err;
>> +
>> + err = pci_enable_device_mem(pdev);
>> + if (err) {
>> + efa_err(&pdev->dev, "pci_enable_device_mem() failed!\n");
>> + return err;
>> + }
>> +
>> + pci_set_master(pdev);
>> +
>> + dev = ib_alloc_device(efa_dev, ibdev);
>
> It return or NULL or real pointer,
ACK.
>
>> + if (IS_ERR_OR_NULL(dev)) {
>> + efa_err(&pdev->dev, "Device alloc failed\n");
>> + err = dev ? PTR_ERR(dev) : -ENOMEM;
>> + goto err_disable_device;
>> + }
>> +
>> + edev = kzalloc(sizeof(*edev), GFP_KERNEL);
>> + if (!edev) {
>> + err = -ENOMEM;
>> + goto err_ibdev_destroy;
>> + }
>> +
>> + pci_set_drvdata(pdev, dev);
>> + edev->dmadev = &pdev->dev;
>> + dev->edev = edev;
>> + dev->pdev = pdev;
>> +
>> + bars = pci_select_bars(pdev, IORESOURCE_MEM) & EFA_BASE_BAR_MASK;
>> + err = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);
>> + if (err) {
>> + efa_err(&pdev->dev, "pci_request_selected_regions failed %d\n",
>> + err);
>> + goto err_free_efa_dev;
>> + }
>> +
>> + dev->reg_bar_addr = pci_resource_start(pdev, EFA_REG_BAR);
>> + dev->reg_bar_len = pci_resource_len(pdev, EFA_REG_BAR);
>> + dev->mem_bar_addr = pci_resource_start(pdev, EFA_MEM_BAR);
>> + dev->mem_bar_len = pci_resource_len(pdev, EFA_MEM_BAR);
>> +
>> + edev->reg_bar = devm_ioremap(&pdev->dev,
>> + dev->reg_bar_addr,
>> + dev->reg_bar_len);
>> + if (!edev->reg_bar) {
>> + efa_err(&pdev->dev, "Failed to remap register bar\n");
>> + err = -EFAULT;
>> + goto err_release_bars;
>> + }
>> +
>> + err = efa_com_mmio_reg_read_init(edev);
>> + if (err) {
>> + efa_err(&pdev->dev, "Failed to init readless MMIO\n");
>> + goto err_iounmap;
>> + }
>> +
>> + err = efa_device_init(edev, pdev);
>> + if (err) {
>> + efa_err(&pdev->dev, "EFA device init failed\n");
>> + if (err == -ETIME)
>> + err = -EPROBE_DEFER;
>> + goto err_reg_read_destroy;
>> + }
>> +
>> + err = efa_enable_msix(dev);
>> + if (err)
>> + goto err_reg_read_destroy;
>> +
>> + edev->aq.msix_vector_idx = dev->admin_msix_vector_idx;
>> + edev->aenq.msix_vector_idx = dev->admin_msix_vector_idx;
>> +
>> + err = efa_set_mgmnt_irq(dev);
>> + if (err)
>> + goto err_disable_msix;
>> +
>> + err = efa_com_admin_init(edev, &aenq_handlers);
>> + if (err)
>> + goto err_free_mgmnt_irq;
>> +
>> + return 0;
>> +
>> +err_free_mgmnt_irq:
>> + efa_free_mgmnt_irq(dev);
>> +err_disable_msix:
>> + efa_disable_msix(dev);
>> +err_reg_read_destroy:
>> + efa_com_mmio_reg_read_destroy(edev);
>> +err_iounmap:
>> + devm_iounmap(&pdev->dev, edev->reg_bar);
>> +err_release_bars:
>> + efa_release_bars(dev, EFA_BASE_BAR_MASK);
>> +err_free_efa_dev:
>> + kfree(edev);
>> +err_ibdev_destroy:
>> + ib_dealloc_device(&dev->ibdev);
>> +err_disable_device:
>> + pci_disable_device(pdev);
>> + return err;
>> +}
>> +
>> +static void efa_remove_device(struct pci_dev *pdev)
>> +{
>> + struct efa_dev *dev = pci_get_drvdata(pdev);
>> + struct efa_com_dev *edev;
>> +
>> + if (!dev)
>> + /*
>> + * This device didn't load properly and its resources
>> + * already released, nothing to do
>> + */
>
> How can efa_remove_device() be called if it didn't success efa_add_device?
It can't, removing.
>
>> + return;
>> +
>> + edev = dev->edev;
>> +
>> + efa_com_admin_destroy(edev);
>> + efa_free_mgmnt_irq(dev);
>> + efa_disable_msix(dev);
>> + efa_com_mmio_reg_read_destroy(edev);
>> + devm_iounmap(&pdev->dev, edev->reg_bar);
>> + efa_release_bars(dev, EFA_BASE_BAR_MASK);
>> + kfree(edev);
>> + ib_dealloc_device(&dev->ibdev);
>> + pci_disable_device(pdev);
>> +}
>> +
>> +static int efa_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
>> +{
>> + struct efa_dev *dev;
>> + int err;
>> +
>> + err = efa_probe_device(pdev);
>> + if (err)
>> + return err;
>> +
>> + dev = pci_get_drvdata(pdev);
>> + err = efa_ib_device_add(dev);
>> + if (err)
>> + goto err_remove_device;
>> +
>> + return 0;
>> +
>> +err_remove_device:
>> + efa_remove_device(pdev);
>> + return err;
>> +}
>> +
>> +static void efa_remove(struct pci_dev *pdev)
>> +{
>> + struct efa_dev *dev = (struct efa_dev *)pci_get_drvdata(pdev);
>> +
>> + efa_ib_device_remove(dev);
>> + efa_remove_device(pdev);
>> +}
>> +
>> +static struct pci_driver efa_pci_driver = {
>> + .name = DRV_MODULE_NAME,
>> + .id_table = efa_pci_tbl,
>> + .probe = efa_probe,
>> + .remove = efa_remove,
>> +};
>> +
>> +static int __init efa_init(void)
>> +{
>> + int err;
>> +
>> + err = pci_register_driver(&efa_pci_driver);
>> + if (err) {
>> + pr_err("Couldn't register efa driver\n");
>> + goto err_register;
>> + }
>> +
>> + return 0;
>> +
>> +err_register:
>> + return err;
>> +}
>> +
>> +static void __exit efa_exit(void)
>> +{
>> + pci_unregister_driver(&efa_pci_driver);
>> +}
>> +
>> +module_init(efa_init);
>> +module_exit(efa_exit);
>> --
>> 2.7.4
>>
