On Mon, Sep 25, 2023 at 3:21 PM Xiao Ni <xni@xxxxxxxxxx> wrote:
>
> On Mon, Aug 28, 2023 at 10:04 AM Yu Kuai <yukuai1@xxxxxxxxxxxxxxx> wrote:
> >
> > From: Yu Kuai <yukuai3@xxxxxxxxxx>
> >
> > Advantages for new apis:
> > - reconfig_mutex is not required;
> > - the weird logical that suspend array hold 'reconfig_mutex' for
> > mddev_check_recovery() to update superblock is not needed;
> > - the specail handling, 'pers->prepare_suspend', for raid456 is not
> > needed;
> > - It's safe to be called at any time once mddev is allocated, and it's
> > designed to be used from slow path where array configuration is changed;
> >
> > Signed-off-by: Yu Kuai <yukuai3@xxxxxxxxxx>
> > ---
> > drivers/md/md.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++--
> > drivers/md/md.h | 3 ++
> > 2 files changed, 86 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/md/md.c b/drivers/md/md.c
> > index 7fa311a14317..6236e2e395c1 100644
> > --- a/drivers/md/md.c
> > +++ b/drivers/md/md.c
> > @@ -443,12 +443,22 @@ void mddev_suspend(struct mddev *mddev)
> > lockdep_is_held(&mddev->reconfig_mutex));
> >
> > WARN_ON_ONCE(thread && current == thread->tsk);
> > - if (mddev->suspended++)
> > +
> > + /* can't concurrent with __mddev_suspend() and __mddev_resume() */
> > + mutex_lock(&mddev->suspend_mutex);
> > + if (mddev->suspended++) {
> > + mutex_unlock(&mddev->suspend_mutex);
> > return;
> > + }
> > +
> > wake_up(&mddev->sb_wait);
> > set_bit(MD_ALLOW_SB_UPDATE, &mddev->flags);
> > percpu_ref_kill(&mddev->active_io);
> >
> > + /*
> > + * TODO: cleanup 'pers->prepare_suspend after all callers are replaced
> > + * by __mddev_suspend().
> > + */
> > if (mddev->pers && mddev->pers->prepare_suspend)
> > mddev->pers->prepare_suspend(mddev);
> >
> > @@ -459,14 +469,21 @@ void mddev_suspend(struct mddev *mddev)
> > del_timer_sync(&mddev->safemode_timer);
> > /* restrict memory reclaim I/O during raid array is suspend */
> > mddev->noio_flag = memalloc_noio_save();
> > +
> > + mutex_unlock(&mddev->suspend_mutex);
> > }
> > EXPORT_SYMBOL_GPL(mddev_suspend);
> >
> > void mddev_resume(struct mddev *mddev)
> > {
> > lockdep_assert_held(&mddev->reconfig_mutex);
> > - if (--mddev->suspended)
> > +
> > + /* can't concurrent with __mddev_suspend() and __mddev_resume() */
> > + mutex_lock(&mddev->suspend_mutex);
> > + if (--mddev->suspended) {
> > + mutex_unlock(&mddev->suspend_mutex);
> > return;
> > + }
> >
> > /* entred the memalloc scope from mddev_suspend() */
> > memalloc_noio_restore(mddev->noio_flag);
> > @@ -477,9 +494,72 @@ void mddev_resume(struct mddev *mddev)
> > set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
> > md_wakeup_thread(mddev->thread);
> > md_wakeup_thread(mddev->sync_thread); /* possibly kick off a reshape */
> > +
> > + mutex_unlock(&mddev->suspend_mutex);
> > }
> > EXPORT_SYMBOL_GPL(mddev_resume);
> >
> > +void __mddev_suspend(struct mddev *mddev)
> > +{
> > +
> > + /*
> > + * hold reconfig_mutex to wait for normal io will deadlock, because
> > + * other context can't update super_block, and normal io can rely on
> > + * updating super_block.
> > + */
> > + lockdep_assert_not_held(&mddev->reconfig_mutex);
> > +
> > + mutex_lock(&mddev->suspend_mutex);
> > +
> > + if (mddev->suspended) {
> > + WRITE_ONCE(mddev->suspended, mddev->suspended + 1);
> > + mutex_unlock(&mddev->suspend_mutex);
> > + return;
> > + }
> > +
> > + percpu_ref_kill(&mddev->active_io);
> > + wait_event(mddev->sb_wait, percpu_ref_is_zero(&mddev->active_io));
> > +
> > + /*
> > + * For raid456, io might be waiting for reshape to make progress,
> > + * allow new reshape to start while waiting for io to be done to
> > + * prevent deadlock.
> > + */
> > + WRITE_ONCE(mddev->suspended, mddev->suspended + 1);
>
> It changes the order of setting suspended and checking active_io.
> suspended is used to stop I/O. Now it checks active_io first and then
> adds suspended, if the i/o doesn't stop, it looks like active_io can't
> be 0. So it will stuck at waiting active_io to be 0?
Ah, I c, you add the state of active_io to judge if a raid is suspended.
Regards
Xiao
>
> Best Regards
> Xiao
>
> > +
> > + del_timer_sync(&mddev->safemode_timer);
> > + /* restrict memory reclaim I/O during raid array is suspend */
> > + mddev->noio_flag = memalloc_noio_save();
> > +
> > + mutex_unlock(&mddev->suspend_mutex);
> > +}
> > +EXPORT_SYMBOL_GPL(__mddev_suspend);
> > +
> > +void __mddev_resume(struct mddev *mddev)
> > +{
> > + lockdep_assert_not_held(&mddev->reconfig_mutex);
> > +
> > + mutex_lock(&mddev->suspend_mutex);
> > + WRITE_ONCE(mddev->suspended, mddev->suspended - 1);
> > + if (mddev->suspended) {
> > + mutex_unlock(&mddev->suspend_mutex);
> > + return;
> > + }
> > +
> > + /* entred the memalloc scope from __mddev_suspend() */
> > + memalloc_noio_restore(mddev->noio_flag);
> > +
> > + percpu_ref_resurrect(&mddev->active_io);
> > + wake_up(&mddev->sb_wait);
> > +
> > + set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
> > + md_wakeup_thread(mddev->thread);
> > + md_wakeup_thread(mddev->sync_thread); /* possibly kick off a reshape */
> > +
> > + mutex_unlock(&mddev->suspend_mutex);
> > +}
> > +EXPORT_SYMBOL_GPL(__mddev_resume);
> > +
> > /*
> > * Generic flush handling for md
> > */
> > @@ -667,6 +747,7 @@ int mddev_init(struct mddev *mddev)
> > mutex_init(&mddev->open_mutex);
> > mutex_init(&mddev->reconfig_mutex);
> > mutex_init(&mddev->sync_mutex);
> > + mutex_init(&mddev->suspend_mutex);
> > mutex_init(&mddev->bitmap_info.mutex);
> > INIT_LIST_HEAD(&mddev->disks);
> > INIT_LIST_HEAD(&mddev->all_mddevs);
> > diff --git a/drivers/md/md.h b/drivers/md/md.h
> > index fb3b123f16dd..1103e6b08ad9 100644
> > --- a/drivers/md/md.h
> > +++ b/drivers/md/md.h
> > @@ -316,6 +316,7 @@ struct mddev {
> > unsigned long sb_flags;
> >
> > int suspended;
> > + struct mutex suspend_mutex;
> > struct percpu_ref active_io;
> > int ro;
> > int sysfs_active; /* set when sysfs deletes
> > @@ -811,6 +812,8 @@ extern void md_rdev_clear(struct md_rdev *rdev);
> > extern void md_handle_request(struct mddev *mddev, struct bio *bio);
> > extern void mddev_suspend(struct mddev *mddev);
> > extern void mddev_resume(struct mddev *mddev);
> > +extern void __mddev_suspend(struct mddev *mddev);
> > +extern void __mddev_resume(struct mddev *mddev);
> >
> > extern void md_reload_sb(struct mddev *mddev, int raid_disk);
> > extern void md_update_sb(struct mddev *mddev, int force);
> > --
> > 2.39.2
> >
