Currently whole lightnvm and pblk uses single DMA pool,
for which entry size is always equal to PAGE_SIZE.
PPA list always needs 8b*64, so there is only 56b*64
space for OOB meta. Since NVMe OOB meta can be bigger,
such as 128b, this solution is not robustness.
This patch add the possiblity to support OOB meta above
56b by creating separate DMA pool for PBLK with entry
size which is big enough to store both PPA list and such
a OOB metadata.
Signed-off-by: Igor Konopko <igor.j.konopko@xxxxxxxxx>
---
drivers/lightnvm/core.c | 33 ++++++++++++++++++++++++---------
drivers/lightnvm/pblk-core.c | 24 +++++++++++++-----------
drivers/lightnvm/pblk-init.c | 9 +++++++++
drivers/lightnvm/pblk-read.c | 40 +++++++++++++++++++++++++++-------------
drivers/lightnvm/pblk-recovery.c | 18 ++++++++++--------
drivers/lightnvm/pblk-write.c | 8 ++++----
drivers/lightnvm/pblk.h | 11 ++++++++++-
drivers/nvme/host/lightnvm.c | 6 ++++--
include/linux/lightnvm.h | 8 +++++---
9 files changed, 106 insertions(+), 51 deletions(-)
diff --git a/drivers/lightnvm/core.c b/drivers/lightnvm/core.c
index 60aa7bc5a630..bc8e6ecea083 100644
--- a/drivers/lightnvm/core.c
+++ b/drivers/lightnvm/core.c
@@ -642,20 +642,33 @@ void nvm_unregister_tgt_type(struct nvm_tgt_type *tt)
}
EXPORT_SYMBOL(nvm_unregister_tgt_type);
-void *nvm_dev_dma_alloc(struct nvm_dev *dev, gfp_t mem_flags,
- dma_addr_t *dma_handler)
+void *nvm_dev_dma_alloc(struct nvm_dev *dev, void *pool,
+ gfp_t mem_flags, dma_addr_t *dma_handler)
{
- return dev->ops->dev_dma_alloc(dev, dev->dma_pool, mem_flags,
- dma_handler);
+ return dev->ops->dev_dma_alloc(dev, pool ?: dev->dma_pool,
+ mem_flags, dma_handler);
}
EXPORT_SYMBOL(nvm_dev_dma_alloc);
-void nvm_dev_dma_free(struct nvm_dev *dev, void *addr, dma_addr_t dma_handler)
+void nvm_dev_dma_free(struct nvm_dev *dev, void *pool,
+ void *addr, dma_addr_t dma_handler)
{
- dev->ops->dev_dma_free(dev->dma_pool, addr, dma_handler);
+ dev->ops->dev_dma_free(pool ?: dev->dma_pool, addr, dma_handler);
}
EXPORT_SYMBOL(nvm_dev_dma_free);
+void *nvm_dev_dma_create(struct nvm_dev *dev, int size, char *name)
+{
+ return dev->ops->create_dma_pool(dev, name, size);
+}
+EXPORT_SYMBOL(nvm_dev_dma_create);
+
+void nvm_dev_dma_destroy(struct nvm_dev *dev, void *pool)
+{
+ dev->ops->destroy_dma_pool(pool);
+}
+EXPORT_SYMBOL(nvm_dev_dma_destroy);
+
static struct nvm_dev *nvm_find_nvm_dev(const char *name)
{
struct nvm_dev *dev;
@@ -683,7 +696,8 @@ static int nvm_set_rqd_ppalist(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd,
}
rqd->nr_ppas = nr_ppas;
- rqd->ppa_list = nvm_dev_dma_alloc(dev, GFP_KERNEL, &rqd->dma_ppa_list);
+ rqd->ppa_list = nvm_dev_dma_alloc(dev, NULL, GFP_KERNEL,
+ &rqd->dma_ppa_list);
if (!rqd->ppa_list) {
pr_err("nvm: failed to allocate dma memory\n");
return -ENOMEM;
@@ -709,7 +723,8 @@ static void nvm_free_rqd_ppalist(struct nvm_tgt_dev *tgt_dev,
if (!rqd->ppa_list)
return;
- nvm_dev_dma_free(tgt_dev->parent, rqd->ppa_list, rqd->dma_ppa_list);
+ nvm_dev_dma_free(tgt_dev->parent, NULL, rqd->ppa_list,
+ rqd->dma_ppa_list);
}
int nvm_get_chunk_meta(struct nvm_tgt_dev *tgt_dev, struct nvm_chk_meta *meta,
@@ -933,7 +948,7 @@ int nvm_register(struct nvm_dev *dev)
if (!dev->q || !dev->ops)
return -EINVAL;
- dev->dma_pool = dev->ops->create_dma_pool(dev, "ppalist");
+ dev->dma_pool = dev->ops->create_dma_pool(dev, "ppalist", PAGE_SIZE);
if (!dev->dma_pool) {
pr_err("nvm: could not create dma pool\n");
return -ENOMEM;
diff --git a/drivers/lightnvm/pblk-core.c b/drivers/lightnvm/pblk-core.c
index 8a0ac466872f..c092ee93a18d 100644
--- a/drivers/lightnvm/pblk-core.c
+++ b/drivers/lightnvm/pblk-core.c
@@ -279,7 +279,7 @@ void pblk_free_rqd(struct pblk *pblk, struct nvm_rq *rqd, int type)
}
if (rqd->meta_list)
- nvm_dev_dma_free(dev->parent, rqd->meta_list,
+ nvm_dev_dma_free(dev->parent, pblk->dma_pool, rqd->meta_list,
rqd->dma_meta_list);
mempool_free(rqd, pool);
}
@@ -652,13 +652,13 @@ static int pblk_line_submit_emeta_io(struct pblk *pblk, struct pblk_line *line,
} else
return -EINVAL;
- meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL,
- &dma_meta_list);
+ meta_list = nvm_dev_dma_alloc(dev->parent, pblk->dma_pool,
+ GFP_KERNEL, &dma_meta_list);
if (!meta_list)
return -ENOMEM;
- ppa_list = meta_list + pblk_dma_meta_size;
- dma_ppa_list = dma_meta_list + pblk_dma_meta_size;
+ ppa_list = meta_list + pblk_dma_meta_size(pblk);
+ dma_ppa_list = dma_meta_list + pblk_dma_meta_size(pblk);
next_rq:
memset(&rqd, 0, sizeof(struct nvm_rq));
@@ -758,7 +758,8 @@ static int pblk_line_submit_emeta_io(struct pblk *pblk, struct pblk_line *line,
if (left_ppas)
goto next_rq;
free_rqd_dma:
- nvm_dev_dma_free(dev->parent, rqd.meta_list, rqd.dma_meta_list);
+ nvm_dev_dma_free(dev->parent, pblk->dma_pool, rqd.meta_list,
+ rqd.dma_meta_list);
return ret;
}
@@ -803,13 +804,13 @@ static int pblk_line_submit_smeta_io(struct pblk *pblk, struct pblk_line *line,
memset(&rqd, 0, sizeof(struct nvm_rq));
- rqd.meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL,
- &rqd.dma_meta_list);
+ rqd.meta_list = nvm_dev_dma_alloc(dev->parent, pblk->dma_pool,
+ GFP_KERNEL, &rqd.dma_meta_list);
if (!rqd.meta_list)
return -ENOMEM;
- rqd.ppa_list = rqd.meta_list + pblk_dma_meta_size;
- rqd.dma_ppa_list = rqd.dma_meta_list + pblk_dma_meta_size;
+ rqd.ppa_list = rqd.meta_list + pblk_dma_meta_size(pblk);
+ rqd.dma_ppa_list = rqd.dma_meta_list + pblk_dma_meta_size(pblk);
bio = bio_map_kern(dev->q, line->smeta, lm->smeta_len, GFP_KERNEL);
if (IS_ERR(bio)) {
@@ -861,7 +862,8 @@ static int pblk_line_submit_smeta_io(struct pblk *pblk, struct pblk_line *line,
}
free_ppa_list:
- nvm_dev_dma_free(dev->parent, rqd.meta_list, rqd.dma_meta_list);
+ nvm_dev_dma_free(dev->parent, pblk->dma_pool, rqd.meta_list,
+ rqd.dma_meta_list);
return ret;
}
diff --git a/drivers/lightnvm/pblk-init.c b/drivers/lightnvm/pblk-init.c
index aa2426403171..f05112230a52 100644
--- a/drivers/lightnvm/pblk-init.c
+++ b/drivers/lightnvm/pblk-init.c
@@ -1142,6 +1142,7 @@ static void pblk_free(struct pblk *pblk)
pblk_l2p_free(pblk);
pblk_rwb_free(pblk);
pblk_core_free(pblk);
+ nvm_dev_dma_destroy(pblk->dev->parent, pblk->dma_pool);
kfree(pblk);
}
@@ -1212,6 +1213,13 @@ static void *pblk_init(struct nvm_tgt_dev *dev, struct gendisk *tdisk,
pblk->disk = tdisk;
pblk->state = PBLK_STATE_RUNNING;
pblk->gc.gc_enabled = 0;
+ pblk->dma_pool = nvm_dev_dma_create(dev->parent, (pblk_dma_ppa_size +
+ pblk_dma_meta_size(pblk)),
+ tdisk->disk_name);
+ if (!pblk->dma_pool) {
+ kfree(pblk);
+ return ERR_PTR(-ENOMEM);
+ }
spin_lock_init(&pblk->resubmit_lock);
spin_lock_init(&pblk->trans_lock);
@@ -1312,6 +1320,7 @@ static void *pblk_init(struct nvm_tgt_dev *dev, struct gendisk *tdisk,
fail_free_core:
pblk_core_free(pblk);
fail:
+ nvm_dev_dma_destroy(dev->parent, pblk->dma_pool);
kfree(pblk);
return ERR_PTR(ret);
}
diff --git a/drivers/lightnvm/pblk-read.c b/drivers/lightnvm/pblk-read.c
index 81cf79ea2dc6..9ff4f48c4168 100644
--- a/drivers/lightnvm/pblk-read.c
+++ b/drivers/lightnvm/pblk-read.c
@@ -255,9 +255,13 @@ static int pblk_partial_read(struct pblk *pblk, struct nvm_rq *rqd,
int nr_holes = nr_secs - bitmap_weight(read_bitmap, nr_secs);
int i, ret, hole;
- /* Re-use allocated memory for intermediate lbas */
- lba_list_mem = (((void *)rqd->ppa_list) + pblk_dma_ppa_size);
- lba_list_media = (((void *)rqd->ppa_list) + 2 * pblk_dma_ppa_size);
+ lba_list_mem = kcalloc(nr_secs, sizeof(__le64), GFP_KERNEL);
+ if (!lba_list_mem)
+ goto err_alloc_mem;
+
+ lba_list_media = kcalloc(nr_secs, sizeof(__le64), GFP_KERNEL);
+ if (!lba_list_media)
+ goto err_alloc_media;
new_bio = bio_alloc(GFP_KERNEL, nr_holes);
@@ -349,6 +353,8 @@ static int pblk_partial_read(struct pblk *pblk, struct nvm_rq *rqd,
rqd->bio = NULL;
rqd->nr_ppas = nr_secs;
+ kfree(lba_list_media);
+ kfree(lba_list_mem);
__pblk_end_io_read(pblk, rqd, false);
return NVM_IO_DONE;
@@ -356,6 +362,10 @@ static int pblk_partial_read(struct pblk *pblk, struct nvm_rq *rqd,
/* Free allocated pages in new bio */
pblk_bio_free_pages(pblk, new_bio, 0, new_bio->bi_vcnt);
fail_add_pages:
+ kfree(lba_list_media);
+err_alloc_media:
+ kfree(lba_list_mem);
+err_alloc_mem:
pr_err("pblk: failed to perform partial read\n");
__pblk_end_io_read(pblk, rqd, false);
return NVM_IO_ERR;
@@ -444,16 +454,17 @@ int pblk_submit_read(struct pblk *pblk, struct bio *bio)
*/
bio_init_idx = pblk_get_bi_idx(bio);
- rqd->meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL,
- &rqd->dma_meta_list);
+ rqd->meta_list = nvm_dev_dma_alloc(dev->parent, pblk->dma_pool,
+ GFP_KERNEL, &rqd->dma_meta_list);
if (!rqd->meta_list) {
pr_err("pblk: not able to allocate ppa list\n");
goto fail_rqd_free;
}
if (nr_secs > 1) {
- rqd->ppa_list = rqd->meta_list + pblk_dma_meta_size;
- rqd->dma_ppa_list = rqd->dma_meta_list + pblk_dma_meta_size;
+ rqd->ppa_list = rqd->meta_list + pblk_dma_meta_size(pblk);
+ rqd->dma_ppa_list = rqd->dma_meta_list +
+ pblk_dma_meta_size(pblk);
pblk_read_ppalist_rq(pblk, rqd, bio, blba, &read_bitmap);
} else {
@@ -578,14 +589,15 @@ int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq)
memset(&rqd, 0, sizeof(struct nvm_rq));
- rqd.meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL,
- &rqd.dma_meta_list);
+ rqd.meta_list = nvm_dev_dma_alloc(dev->parent, pblk->dma_pool,
+ GFP_KERNEL, &rqd.dma_meta_list);
if (!rqd.meta_list)
return -ENOMEM;
if (gc_rq->nr_secs > 1) {
- rqd.ppa_list = rqd.meta_list + pblk_dma_meta_size;
- rqd.dma_ppa_list = rqd.dma_meta_list + pblk_dma_meta_size;
+ rqd.ppa_list = rqd.meta_list + pblk_dma_meta_size(pblk);
+ rqd.dma_ppa_list = rqd.dma_meta_list +
+ pblk_dma_meta_size(pblk);
gc_rq->secs_to_gc = read_ppalist_rq_gc(pblk, &rqd, gc_rq->line,
gc_rq->lba_list,
@@ -642,12 +654,14 @@ int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq)
#endif
out:
- nvm_dev_dma_free(dev->parent, rqd.meta_list, rqd.dma_meta_list);
+ nvm_dev_dma_free(dev->parent, pblk->dma_pool, rqd.meta_list,
+ rqd.dma_meta_list);
return ret;
err_free_bio:
bio_put(bio);
err_free_dma:
- nvm_dev_dma_free(dev->parent, rqd.meta_list, rqd.dma_meta_list);
+ nvm_dev_dma_free(dev->parent, pblk->dma_pool, rqd.meta_list,
+ rqd.dma_meta_list);
return ret;
}
diff --git a/drivers/lightnvm/pblk-recovery.c b/drivers/lightnvm/pblk-recovery.c
index 0007e8011476..f5853fc77a0c 100644
--- a/drivers/lightnvm/pblk-recovery.c
+++ b/drivers/lightnvm/pblk-recovery.c
@@ -280,14 +280,15 @@ static int pblk_recov_pad_oob(struct pblk *pblk, struct pblk_line *line,
rq_len = rq_ppas * geo->csecs;
- meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL, &dma_meta_list);
+ meta_list = nvm_dev_dma_alloc(dev->parent, pblk->dma_pool,
+ GFP_KERNEL, &dma_meta_list);
if (!meta_list) {
ret = -ENOMEM;
goto fail_free_pad;
}
- ppa_list = (void *)(meta_list) + pblk_dma_meta_size;
- dma_ppa_list = dma_meta_list + pblk_dma_meta_size;
+ ppa_list = (void *)(meta_list) + pblk_dma_meta_size(pblk);
+ dma_ppa_list = dma_meta_list + pblk_dma_meta_size(pblk);
bio = pblk_bio_map_addr(pblk, data, rq_ppas, rq_len,
PBLK_VMALLOC_META, GFP_KERNEL);
@@ -373,7 +374,7 @@ static int pblk_recov_pad_oob(struct pblk *pblk, struct pblk_line *line,
fail_free_bio:
bio_put(bio);
fail_free_meta:
- nvm_dev_dma_free(dev->parent, meta_list, dma_meta_list);
+ nvm_dev_dma_free(dev->parent, pblk->dma_pool, meta_list, dma_meta_list);
fail_free_pad:
kfree(pad_rq);
vfree(data);
@@ -651,12 +652,13 @@ static int pblk_recov_l2p_from_oob(struct pblk *pblk, struct pblk_line *line)
dma_addr_t dma_ppa_list, dma_meta_list;
int done, ret = 0;
- meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL, &dma_meta_list);
+ meta_list = nvm_dev_dma_alloc(dev->parent, pblk->dma_pool,
+ GFP_KERNEL, &dma_meta_list);
if (!meta_list)
return -ENOMEM;
- ppa_list = (void *)(meta_list) + pblk_dma_meta_size;
- dma_ppa_list = dma_meta_list + pblk_dma_meta_size;
+ ppa_list = (void *)(meta_list) + pblk_dma_meta_size(pblk);
+ dma_ppa_list = dma_meta_list + pblk_dma_meta_size(pblk);
data = kcalloc(pblk->max_write_pgs, geo->csecs, GFP_KERNEL);
if (!data) {
@@ -693,7 +695,7 @@ static int pblk_recov_l2p_from_oob(struct pblk *pblk, struct pblk_line *line)
out:
kfree(data);
free_meta_list:
- nvm_dev_dma_free(dev->parent, meta_list, dma_meta_list);
+ nvm_dev_dma_free(dev->parent, pblk->dma_pool, meta_list, dma_meta_list);
return ret;
}
diff --git a/drivers/lightnvm/pblk-write.c b/drivers/lightnvm/pblk-write.c
index 5f44df999aed..6552db35f916 100644
--- a/drivers/lightnvm/pblk-write.c
+++ b/drivers/lightnvm/pblk-write.c
@@ -306,13 +306,13 @@ static int pblk_alloc_w_rq(struct pblk *pblk, struct nvm_rq *rqd,
rqd->private = pblk;
rqd->end_io = end_io;
- rqd->meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL,
- &rqd->dma_meta_list);
+ rqd->meta_list = nvm_dev_dma_alloc(dev->parent, pblk->dma_pool,
+ GFP_KERNEL, &rqd->dma_meta_list);
if (!rqd->meta_list)
return -ENOMEM;
- rqd->ppa_list = rqd->meta_list + pblk_dma_meta_size;
- rqd->dma_ppa_list = rqd->dma_meta_list + pblk_dma_meta_size;
+ rqd->ppa_list = rqd->meta_list + pblk_dma_meta_size(pblk);
+ rqd->dma_ppa_list = rqd->dma_meta_list + pblk_dma_meta_size(pblk);
return 0;
}
diff --git a/drivers/lightnvm/pblk.h b/drivers/lightnvm/pblk.h
index 27658dc6fc1a..4c61ede5b207 100644
--- a/drivers/lightnvm/pblk.h
+++ b/drivers/lightnvm/pblk.h
@@ -98,7 +98,6 @@ enum {
PBLK_RL_LOW = 4
};
-#define pblk_dma_meta_size (sizeof(struct pblk_sec_meta) * PBLK_MAX_REQ_ADDRS)
#define pblk_dma_ppa_size (sizeof(u64) * PBLK_MAX_REQ_ADDRS)
/* write buffer completion context */
@@ -690,6 +689,7 @@ struct pblk {
struct timer_list wtimer;
struct pblk_gc gc;
+ void *dma_pool;
};
struct pblk_line_ws {
@@ -1448,4 +1448,13 @@ static inline struct pblk_sec_meta *pblk_get_meta_at(struct pblk *pblk,
*/
return meta_ptr + sizeof(struct pblk_sec_meta) * index;
}
+
+static inline int pblk_dma_meta_size(struct pblk *pblk)
+{
+ struct nvm_tgt_dev *dev = pblk->dev;
+ struct nvm_geo *geo = &dev->geo;
+
+ return max(((int)sizeof(struct pblk_sec_meta)), ((int)geo->sos))
+ * PBLK_MAX_REQ_ADDRS;
+}
#endif /* PBLK_H_ */
diff --git a/drivers/nvme/host/lightnvm.c b/drivers/nvme/host/lightnvm.c
index 006d09e0af74..670478abc754 100644
--- a/drivers/nvme/host/lightnvm.c
+++ b/drivers/nvme/host/lightnvm.c
@@ -729,11 +729,13 @@ static int nvme_nvm_submit_io_sync(struct nvm_dev *dev, struct nvm_rq *rqd)
return ret;
}
-static void *nvme_nvm_create_dma_pool(struct nvm_dev *nvmdev, char *name)
+static void *nvme_nvm_create_dma_pool(struct nvm_dev *nvmdev, char *name,
+ int size)
{
struct nvme_ns *ns = nvmdev->q->queuedata;
- return dma_pool_create(name, ns->ctrl->dev, PAGE_SIZE, PAGE_SIZE, 0);
+ size = round_up(size, PAGE_SIZE);
+ return dma_pool_create(name, ns->ctrl->dev, size, PAGE_SIZE, 0);
}
static void nvme_nvm_destroy_dma_pool(void *pool)
diff --git a/include/linux/lightnvm.h b/include/linux/lightnvm.h
index e9e0d1c7eaf5..72a55d71917e 100644
--- a/include/linux/lightnvm.h
+++ b/include/linux/lightnvm.h
@@ -90,7 +90,7 @@ typedef int (nvm_get_chk_meta_fn)(struct nvm_dev *, struct nvm_chk_meta *,
sector_t, int);
typedef int (nvm_submit_io_fn)(struct nvm_dev *, struct nvm_rq *);
typedef int (nvm_submit_io_sync_fn)(struct nvm_dev *, struct nvm_rq *);
-typedef void *(nvm_create_dma_pool_fn)(struct nvm_dev *, char *);
+typedef void *(nvm_create_dma_pool_fn)(struct nvm_dev *, char *, int);
typedef void (nvm_destroy_dma_pool_fn)(void *);
typedef void *(nvm_dev_dma_alloc_fn)(struct nvm_dev *, void *, gfp_t,
dma_addr_t *);
@@ -517,8 +517,10 @@ struct nvm_tgt_type {
extern int nvm_register_tgt_type(struct nvm_tgt_type *);
extern void nvm_unregister_tgt_type(struct nvm_tgt_type *);
-extern void *nvm_dev_dma_alloc(struct nvm_dev *, gfp_t, dma_addr_t *);
-extern void nvm_dev_dma_free(struct nvm_dev *, void *, dma_addr_t);
+extern void *nvm_dev_dma_alloc(struct nvm_dev *, void *, gfp_t, dma_addr_t *);
+extern void nvm_dev_dma_free(struct nvm_dev *, void *, void *, dma_addr_t);
+extern void *nvm_dev_dma_create(struct nvm_dev *, int, char *);
+extern void nvm_dev_dma_destroy(struct nvm_dev *, void *);
extern struct nvm_dev *nvm_alloc_dev(int);
extern int nvm_register(struct nvm_dev *);
--
2.14.3
