On Tue, 26 Oct 2021 12:06:05 +0300
Yishai Hadas <yishaih@xxxxxxxxxx> wrote:
> Register its own handler for pci_error_handlers.reset_done and update
> state accordingly.
>
> Signed-off-by: Yishai Hadas <yishaih@xxxxxxxxxx>
> Signed-off-by: Leon Romanovsky <leonro@xxxxxxxxxx>
> ---
> drivers/vfio/pci/mlx5/main.c | 54 ++++++++++++++++++++++++++++++++++--
> 1 file changed, 52 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/vfio/pci/mlx5/main.c b/drivers/vfio/pci/mlx5/main.c
> index 4b21b388dcc5..c157f540d384 100644
> --- a/drivers/vfio/pci/mlx5/main.c
> +++ b/drivers/vfio/pci/mlx5/main.c
> @@ -55,8 +55,11 @@ struct mlx5vf_pci_migration_info {
> struct mlx5vf_pci_core_device {
> struct vfio_pci_core_device core_device;
> u8 migrate_cap:1;
> + u8 defered_reset:1;
s/defered/deferred/ throughout
> /* protect migration state */
> struct mutex state_mutex;
> + /* protect the reset_done flow */
> + spinlock_t reset_lock;
> struct mlx5vf_pci_migration_info vmig;
> };
>
> @@ -471,6 +474,47 @@ mlx5vf_pci_migration_data_rw(struct mlx5vf_pci_core_device *mvdev,
> return count;
> }
>
> +/* This function is called in all state_mutex unlock cases to
> + * handle a 'defered_reset' if exists.
> + */
I refrained from noting it elsewhere, but we're not in net/ or
drivers/net/ here, but we're using their multi-line comment style. Are
we using the strong relation to a driver that does belong there as
justification for the style here?
> +static void mlx5vf_state_mutex_unlock(struct mlx5vf_pci_core_device *mvdev)
> +{
> +again:
> + spin_lock(&mvdev->reset_lock);
> + if (mvdev->defered_reset) {
> + mvdev->defered_reset = false;
> + spin_unlock(&mvdev->reset_lock);
> + mlx5vf_reset_mig_state(mvdev);
> + mvdev->vmig.vfio_dev_state = VFIO_DEVICE_STATE_RUNNING;
> + goto again;
> + }
> + mutex_unlock(&mvdev->state_mutex);
> + spin_unlock(&mvdev->reset_lock);
> +}
> +
> +static void mlx5vf_pci_aer_reset_done(struct pci_dev *pdev)
> +{
> + struct mlx5vf_pci_core_device *mvdev = dev_get_drvdata(&pdev->dev);
> +
> + if (!mvdev->migrate_cap)
> + return;
> +
> + /* As the higher VFIO layers are holding locks across reset and using
> + * those same locks with the mm_lock we need to prevent ABBA deadlock
> + * with the state_mutex and mm_lock.
> + * In case the state_mutex was taken alreday we differ the cleanup work
s/alreday/already/ s/differ/defer/
> + * to the unlock flow of the other running context.
> + */
> + spin_lock(&mvdev->reset_lock);
> + mvdev->defered_reset = true;
> + if (!mutex_trylock(&mvdev->state_mutex)) {
> + spin_unlock(&mvdev->reset_lock);
> + return;
> + }
> + spin_unlock(&mvdev->reset_lock);
> + mlx5vf_state_mutex_unlock(mvdev);
> +}
> +
> static ssize_t mlx5vf_pci_mig_rw(struct vfio_pci_core_device *vdev,
> char __user *buf, size_t count, loff_t *ppos,
> bool iswrite)
> @@ -539,7 +583,7 @@ static ssize_t mlx5vf_pci_mig_rw(struct vfio_pci_core_device *vdev,
> }
>
> end:
> - mutex_unlock(&mvdev->state_mutex);
> + mlx5vf_state_mutex_unlock(mvdev);
I'm a little lost here, if the operation was to read the device_state
and mvdev->vmig.vfio_dev_state was error, that's already been copied to
the user buffer, so the user continues to see the error state for the
first read of device_state after reset if they encounter this race?
Thanks,
Alex
> return ret;
> }
>
> @@ -634,6 +678,7 @@ static int mlx5vf_pci_probe(struct pci_dev *pdev,
> if (MLX5_CAP_GEN(mdev, migration)) {
> mvdev->migrate_cap = 1;
> mutex_init(&mvdev->state_mutex);
> + spin_lock_init(&mvdev->reset_lock);
> }
> mlx5_vf_put_core_dev(mdev);
> }
> @@ -668,12 +713,17 @@ static const struct pci_device_id mlx5vf_pci_table[] = {
>
> MODULE_DEVICE_TABLE(pci, mlx5vf_pci_table);
>
> +const struct pci_error_handlers mlx5vf_err_handlers = {
> + .reset_done = mlx5vf_pci_aer_reset_done,
> + .error_detected = vfio_pci_aer_err_detected,
> +};
> +
> static struct pci_driver mlx5vf_pci_driver = {
> .name = KBUILD_MODNAME,
> .id_table = mlx5vf_pci_table,
> .probe = mlx5vf_pci_probe,
> .remove = mlx5vf_pci_remove,
> - .err_handler = &vfio_pci_core_err_handlers,
> + .err_handler = &mlx5vf_err_handlers,
> };
>
> static void __exit mlx5vf_pci_cleanup(void)
