As of this moment ->swap_slot_free_notify is called from atomic section (under spin-lock) which makes it impossible to make zsmalloc fully preemptible. Deffer slot-free to a non-atomic context. Signed-off-by: Sergey Senozhatsky <senozhatsky@xxxxxxxxxxxx> --- drivers/block/zram/zram_drv.c | 66 +++++++++++++++++++++++++++++++++-- drivers/block/zram/zram_drv.h | 4 +++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index ad3e8885b0d2..9c72beb86ab0 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -2315,9 +2315,52 @@ static void zram_submit_bio(struct bio *bio) } } +static void async_slot_free(struct work_struct *work) +{ + struct zram *zram = container_of(work, struct zram, slot_free_work); + + spin_lock(&zram->slot_free_lock); + while (!list_empty(&zram->slot_free_list)) { + struct zram_pp_slot *pps; + + pps = list_first_entry(&zram->slot_free_list, + struct zram_pp_slot, + entry); + list_del_init(&pps->entry); + spin_unlock(&zram->slot_free_lock); + + zram_slot_write_lock(zram, pps->index); + if (zram_test_flag(zram, pps->index, ZRAM_PP_SLOT)) + zram_free_page(zram, pps->index); + zram_slot_write_unlock(zram, pps->index); + + kfree(pps); + spin_lock(&zram->slot_free_lock); + } + spin_unlock(&zram->slot_free_lock); +}; + +static void zram_kick_slot_free(struct zram *zram) +{ + schedule_work(&zram->slot_free_work); +} + +static void zram_flush_slot_free(struct zram *zram) +{ + flush_work(&zram->slot_free_work); +} + +static void zram_init_slot_free(struct zram *zram) +{ + spin_lock_init(&zram->slot_free_lock); + INIT_LIST_HEAD(&zram->slot_free_list); + INIT_WORK(&zram->slot_free_work, async_slot_free); +} + static void zram_slot_free_notify(struct block_device *bdev, - unsigned long index) + unsigned long index) { + struct zram_pp_slot *pps; struct zram *zram; zram = bdev->bd_disk->private_data; @@ -2328,7 +2371,24 @@ static void zram_slot_free_notify(struct block_device *bdev, return; } - zram_free_page(zram, index); + if (zram_test_flag(zram, index, ZRAM_PP_SLOT)) + goto out; + + pps = kzalloc(sizeof(*pps), GFP_ATOMIC); + if (!pps) { + atomic64_inc(&zram->stats.miss_free); + goto out; + } + + INIT_LIST_HEAD(&pps->entry); + pps->index = index; + zram_set_flag(zram, index, ZRAM_PP_SLOT); + spin_lock(&zram->slot_free_lock); + list_add(&pps->entry, &zram->slot_free_list); + spin_unlock(&zram->slot_free_lock); + + zram_kick_slot_free(zram); +out: zram_slot_write_unlock(zram, index); } @@ -2473,6 +2533,7 @@ static ssize_t reset_store(struct device *dev, /* Make sure all the pending I/O are finished */ sync_blockdev(disk->part0); + zram_flush_slot_free(zram); zram_reset_device(zram); mutex_lock(&disk->open_mutex); @@ -2618,6 +2679,7 @@ static int zram_add(void) atomic_set(&zram->pp_in_progress, 0); zram_comp_params_reset(zram); comp_algorithm_set(zram, ZRAM_PRIMARY_COMP, default_compressor); + zram_init_slot_free(zram); /* Actual capacity set using sysfs (/sys/block/zram<id>/disksize */ set_capacity(zram->disk, 0); diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h index b7e250d6fa02..27ca269f4a4e 100644 --- a/drivers/block/zram/zram_drv.h +++ b/drivers/block/zram/zram_drv.h @@ -134,5 +134,9 @@ struct zram { struct dentry *debugfs_dir; #endif atomic_t pp_in_progress; + + spinlock_t slot_free_lock; + struct list_head slot_free_list; + struct work_struct slot_free_work; }; #endif -- 2.48.1.262.g85cc9f2d1e-goog