Add a shrinker to reclaim several pages used by bounce buffer
in order to avoid memory pressures. We will do reclaiming
chunk by chunk. And only reclaim the iova chunk that no one used.
Signed-off-by: Xie Yongji <xieyongji@xxxxxxxxxxxxx>
---
drivers/vdpa/vdpa_user/iova_domain.c | 83 ++++++++++++++++++++++++++--
drivers/vdpa/vdpa_user/iova_domain.h | 10 ++++
drivers/vdpa/vdpa_user/vduse_dev.c | 51 +++++++++++++++++
3 files changed, 140 insertions(+), 4 deletions(-)
diff --git a/drivers/vdpa/vdpa_user/iova_domain.c b/drivers/vdpa/vdpa_user/iova_domain.c
index a274f78f00d2..9e3d4686de4f 100644
--- a/drivers/vdpa/vdpa_user/iova_domain.c
+++ b/drivers/vdpa/vdpa_user/iova_domain.c
@@ -30,6 +30,8 @@ struct vduse_mmap_vma {
struct list_head list;
};
+struct percpu_counter vduse_total_bounce_pages;
+
static inline struct page *
vduse_domain_get_bounce_page(struct vduse_iova_domain *domain,
unsigned long iova)
@@ -49,6 +51,13 @@ vduse_domain_set_bounce_page(struct vduse_iova_domain *domain,
unsigned long chunkoff = iova & ~IOVA_CHUNK_MASK;
unsigned long pgindex = chunkoff >> PAGE_SHIFT;
+ if (page) {
+ domain->chunks[index].used_bounce_pages++;
+ percpu_counter_inc(&vduse_total_bounce_pages);
+ } else {
+ domain->chunks[index].used_bounce_pages--;
+ percpu_counter_dec(&vduse_total_bounce_pages);
+ }
domain->chunks[index].bounce_pages[pgindex] = page;
}
@@ -159,6 +168,29 @@ void vduse_domain_remove_mapping(struct vduse_iova_domain *domain,
spin_unlock(&domain->map_lock);
}
+static bool vduse_domain_try_unmap(struct vduse_iova_domain *domain,
+ unsigned long iova, size_t size)
+{
+ struct vduse_mmap_vma *mmap_vma;
+ unsigned long uaddr;
+ bool unmap = true;
+
+ mutex_lock(&domain->vma_lock);
+ list_for_each_entry(mmap_vma, &domain->vma_list, list) {
+ if (!mmap_read_trylock(mmap_vma->vma->vm_mm)) {
+ unmap = false;
+ break;
+ }
+
+ uaddr = iova + mmap_vma->vma->vm_start;
+ zap_page_range(mmap_vma->vma, uaddr, size);
+ mmap_read_unlock(mmap_vma->vma->vm_mm);
+ }
+ mutex_unlock(&domain->vma_lock);
+
+ return unmap;
+}
+
void vduse_domain_unmap(struct vduse_iova_domain *domain,
unsigned long iova, size_t size)
{
@@ -284,6 +316,32 @@ bool vduse_domain_is_direct_map(struct vduse_iova_domain *domain,
return atomic_read(&chunk->map_type) == TYPE_DIRECT_MAP;
}
+int vduse_domain_reclaim(struct vduse_iova_domain *domain)
+{
+ struct vduse_iova_chunk *chunk;
+ int i, freed = 0;
+
+ for (i = domain->chunk_num - 1; i >= 0; i--) {
+ chunk = &domain->chunks[i];
+ if (!chunk->used_bounce_pages)
+ continue;
+
+ if (atomic_cmpxchg(&chunk->state, 0, INT_MIN) != 0)
+ continue;
+
+ if (!vduse_domain_try_unmap(domain,
+ chunk->start, IOVA_CHUNK_SIZE)) {
+ atomic_sub(INT_MIN, &chunk->state);
+ break;
+ }
+ freed += vduse_domain_free_bounce_pages(domain,
+ chunk->start, IOVA_CHUNK_SIZE);
+ atomic_sub(INT_MIN, &chunk->state);
+ }
+
+ return freed;
+}
+
unsigned long vduse_domain_alloc_iova(struct vduse_iova_domain *domain,
size_t size, enum iova_map_type type)
{
@@ -301,10 +359,13 @@ unsigned long vduse_domain_alloc_iova(struct vduse_iova_domain *domain,
if (atomic_read(&chunk->map_type) != type)
continue;
- iova = gen_pool_alloc_algo(chunk->pool, size,
+ if (atomic_fetch_inc(&chunk->state) >= 0) {
+ iova = gen_pool_alloc_algo(chunk->pool, size,
gen_pool_first_fit_align, &data);
- if (iova)
- break;
+ if (iova)
+ break;
+ }
+ atomic_dec(&chunk->state);
}
return iova;
@@ -317,6 +378,7 @@ void vduse_domain_free_iova(struct vduse_iova_domain *domain,
struct vduse_iova_chunk *chunk = &domain->chunks[index];
gen_pool_free(chunk->pool, iova, size);
+ atomic_dec(&chunk->state);
}
static void vduse_iova_chunk_cleanup(struct vduse_iova_chunk *chunk)
@@ -332,7 +394,8 @@ void vduse_iova_domain_destroy(struct vduse_iova_domain *domain)
for (i = 0; i < domain->chunk_num; i++) {
chunk = &domain->chunks[i];
- vduse_domain_free_bounce_pages(domain,
+ if (chunk->used_bounce_pages)
+ vduse_domain_free_bounce_pages(domain,
chunk->start, IOVA_CHUNK_SIZE);
vduse_iova_chunk_cleanup(chunk);
}
@@ -365,8 +428,10 @@ static int vduse_iova_chunk_init(struct vduse_iova_chunk *chunk,
if (!chunk->bounce_pages)
goto err;
+ chunk->used_bounce_pages = 0;
chunk->start = addr;
atomic_set(&chunk->map_type, TYPE_NONE);
+ atomic_set(&chunk->state, 0);
return 0;
err:
@@ -411,3 +476,13 @@ struct vduse_iova_domain *vduse_iova_domain_create(size_t size)
return NULL;
}
+
+int vduse_domain_init(void)
+{
+ return percpu_counter_init(&vduse_total_bounce_pages, 0, GFP_KERNEL);
+}
+
+void vduse_domain_exit(void)
+{
+ percpu_counter_destroy(&vduse_total_bounce_pages);
+}
diff --git a/drivers/vdpa/vdpa_user/iova_domain.h b/drivers/vdpa/vdpa_user/iova_domain.h
index 7ae60c0e50ec..016f84d4bef2 100644
--- a/drivers/vdpa/vdpa_user/iova_domain.h
+++ b/drivers/vdpa/vdpa_user/iova_domain.h
@@ -24,8 +24,10 @@ enum iova_map_type {
struct vduse_iova_chunk {
struct gen_pool *pool;
struct page **bounce_pages;
+ int used_bounce_pages;
unsigned long start;
atomic_t map_type;
+ atomic_t state;
};
struct vduse_iova_domain {
@@ -45,6 +47,8 @@ struct vduse_iova_map {
enum dma_data_direction dir;
};
+extern struct percpu_counter vduse_total_bounce_pages;
+
int vduse_domain_add_vma(struct vduse_iova_domain *domain,
struct vm_area_struct *vma);
@@ -78,6 +82,8 @@ int vduse_domain_bounce_map(struct vduse_iova_domain *domain,
bool vduse_domain_is_direct_map(struct vduse_iova_domain *domain,
unsigned long iova);
+int vduse_domain_reclaim(struct vduse_iova_domain *domain);
+
unsigned long vduse_domain_alloc_iova(struct vduse_iova_domain *domain,
size_t size, enum iova_map_type type);
@@ -91,4 +97,8 @@ void vduse_iova_domain_destroy(struct vduse_iova_domain *domain);
struct vduse_iova_domain *vduse_iova_domain_create(size_t size);
+int vduse_domain_init(void);
+
+void vduse_domain_exit(void);
+
#endif /* _VDUSE_IOVA_DOMAIN_H */
diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c
index f04aa02de8c1..1163209ffff3 100644
--- a/drivers/vdpa/vdpa_user/vduse_dev.c
+++ b/drivers/vdpa/vdpa_user/vduse_dev.c
@@ -977,6 +977,43 @@ static long vduse_ioctl(struct file *file, unsigned int cmd,
return ret;
}
+static unsigned long vduse_shrink_scan(struct shrinker *shrinker,
+ struct shrink_control *sc)
+{
+ unsigned long freed = 0;
+ struct vduse_dev *dev;
+
+ if (!mutex_trylock(&vduse_lock))
+ return SHRINK_STOP;
+
+ list_for_each_entry(dev, &vduse_devs, list) {
+ if (!dev->domain)
+ continue;
+
+ freed = vduse_domain_reclaim(dev->domain);
+ if (!freed)
+ continue;
+
+ list_move_tail(&dev->list, &vduse_devs);
+ break;
+ }
+ mutex_unlock(&vduse_lock);
+
+ return freed ? freed : SHRINK_STOP;
+}
+
+static unsigned long vduse_shrink_count(struct shrinker *shrink,
+ struct shrink_control *sc)
+{
+ return percpu_counter_read_positive(&vduse_total_bounce_pages);
+}
+
+static struct shrinker vduse_bounce_pages_shrinker = {
+ .count_objects = vduse_shrink_count,
+ .scan_objects = vduse_shrink_scan,
+ .seeks = DEFAULT_SEEKS,
+};
+
static const struct file_operations vduse_fops = {
.owner = THIS_MODULE,
.unlocked_ioctl = vduse_ioctl,
@@ -1007,7 +1044,19 @@ static int vduse_init(void)
if (ret)
goto err_irqfd;
+ ret = vduse_domain_init();
+ if (ret)
+ goto err_domain;
+
+ ret = register_shrinker(&vduse_bounce_pages_shrinker);
+ if (ret)
+ goto err_shrinker;
+
return 0;
+err_shrinker:
+ vduse_domain_exit();
+err_domain:
+ vduse_virqfd_exit();
err_irqfd:
destroy_workqueue(vduse_vdpa_wq);
err_vdpa_wq:
@@ -1018,8 +1067,10 @@ module_init(vduse_init);
static void vduse_exit(void)
{
+ unregister_shrinker(&vduse_bounce_pages_shrinker);
misc_deregister(&vduse_misc);
destroy_workqueue(vduse_vdpa_wq);
+ vduse_domain_exit();
vduse_virqfd_exit();
}
module_exit(vduse_exit);
--
2.25.1
