This patch supports packed command of eMMC4.5 device.
Several reads(or writes) can be grouped in packed command
and all data of the individual commands can be sent in a
single transfer on the bus.
Signed-off-by: Seungwon Jeon <tgih.jun@xxxxxxxxxxx>
---
drivers/mmc/card/block.c | 355 ++++++++++++++++++++++++++++++++++++++++++++--
drivers/mmc/card/queue.c | 48 ++++++-
drivers/mmc/card/queue.h | 12 ++
include/linux/mmc/core.h | 3 +
4 files changed, 404 insertions(+), 14 deletions(-)
diff --git a/drivers/mmc/card/block.c b/drivers/mmc/card/block.c
index a1cb21f..6c49656 100644
--- a/drivers/mmc/card/block.c
+++ b/drivers/mmc/card/block.c
@@ -59,6 +59,13 @@ MODULE_ALIAS("mmc:block");
#define INAND_CMD38_ARG_SECTRIM1 0x81
#define INAND_CMD38_ARG_SECTRIM2 0x88
+#define mmc_req_rel_wr(req) (((req->cmd_flags & REQ_FUA) || \
+ (req->cmd_flags & REQ_META)) && \
+ (rq_data_dir(req) == WRITE))
+#define PACKED_CMD_VER 0x01
+#define PACKED_CMD_RD 0x01
+#define PACKED_CMD_WR 0x02
+
static DEFINE_MUTEX(block_mutex);
/*
@@ -943,7 +950,8 @@ static int mmc_blk_err_check(struct mmc_card *card,
* kind. If it was a write, we may have transitioned to
* program mode, which we have to wait for it to complete.
*/
- if (!mmc_host_is_spi(card->host) && rq_data_dir(req) != READ) {
+ if ((!mmc_host_is_spi(card->host) && rq_data_dir(req) != READ) ||
+ (mq_mrq->packed_cmd == MMC_PACKED_WR_HDR)) {
u32 status;
do {
int err = get_card_status(card, &status, 5);
@@ -980,12 +988,67 @@ static int mmc_blk_err_check(struct mmc_card *card,
if (!brq->data.bytes_xfered)
return MMC_BLK_RETRY;
+ if (mq_mrq->packed_cmd != MMC_PACKED_NONE) {
+ if (unlikely(brq->data.blocks << 9 != brq->data.bytes_xfered))
+ return MMC_BLK_PARTIAL;
+ else
+ return MMC_BLK_SUCCESS;
+ }
+
if (blk_rq_bytes(req) != brq->data.bytes_xfered)
return MMC_BLK_PARTIAL;
return MMC_BLK_SUCCESS;
}
+static int mmc_blk_packed_err_check(struct mmc_card *card,
+ struct mmc_async_req *areq)
+{
+ struct mmc_queue_req *mq_mrq = container_of(areq, struct mmc_queue_req,
+ mmc_active);
+ int err, check, status;
+ u8 ext_csd[512];
+
+ check = mmc_blk_err_check(card, areq);
+
+ if (check == MMC_BLK_SUCCESS)
+ return check;
+
+ if (check == MMC_BLK_PARTIAL) {
+ err = get_card_status(card, &status, 0);
+ if (err)
+ return MMC_BLK_ABORT;
+
+ if (status & R1_EXP_EVENT) {
+ err = mmc_send_ext_csd(card, ext_csd);
+ if (err)
+ return MMC_BLK_ABORT;
+
+ if ((ext_csd[EXT_CSD_EXP_EVENTS_STATUS + 0] &
+ EXT_CSD_PACKED_FAILURE) &&
+ (ext_csd[EXT_CSD_PACKED_CMD_STATUS] &
+ EXT_CSD_PACKED_GENERIC_ERROR)) {
+ if (ext_csd[EXT_CSD_PACKED_CMD_STATUS] &
+ EXT_CSD_PACKED_INDEXED_ERROR) {
+ /* Make be 0-based */
+ mq_mrq->packed_fail_idx =
+ ext_csd[EXT_CSD_PACKED_FAILURE_INDEX] - 1;
+ return MMC_BLK_PARTIAL;
+ } else {
+ return MMC_BLK_RETRY;
+ }
+ }
+ } else {
+ return MMC_BLK_RETRY;
+ }
+ }
+
+ if (check != MMC_BLK_ABORT)
+ return MMC_BLK_RETRY;
+ else
+ return MMC_BLK_ABORT;
+}
+
static void mmc_blk_rw_rq_prep(struct mmc_queue_req *mqrq,
struct mmc_card *card,
int disable_multi,
@@ -1129,6 +1192,211 @@ static void mmc_blk_rw_rq_prep(struct mmc_queue_req *mqrq,
mmc_queue_bounce_pre(mqrq);
}
+static u8 mmc_blk_chk_packable(struct mmc_queue *mq, struct request *req)
+{
+ struct request_queue *q = mq->queue;
+ struct mmc_card *card = mq->card;
+ struct request *cur = req, *next = NULL;
+ struct mmc_blk_data *md = mq->data;
+ bool en_rel_wr = card->ext_csd.rel_param & EXT_CSD_WR_REL_PARAM_EN;
+ unsigned int req_sectors = 0, phys_segments = 0;
+ unsigned int max_blk_count, max_phys_segs;
+ u8 max_packed_rw = 0;
+ u8 reqs = 0;
+
+ if (!(md->flags & MMC_BLK_CMD23) &&
+ !card->ext_csd.packed_event_en)
+ goto no_packed;
+
+ if (rq_data_dir(cur) == READ)
+ max_packed_rw = card->ext_csd.max_packed_reads;
+ else
+ max_packed_rw = card->ext_csd.max_packed_writes;
+
+ if (max_packed_rw == 0)
+ goto no_packed;
+
+ if (mmc_req_rel_wr(cur) &&
+ (md->flags & MMC_BLK_REL_WR) &&
+ !en_rel_wr) {
+ goto no_packed;
+ }
+
+ max_blk_count = min(card->host->max_blk_count,
+ card->host->max_req_size >> 9);
+ if (unlikely(max_blk_count > 0xffff))
+ max_blk_count = 0xffff;
+
+ max_phys_segs = queue_max_segments(q);
+ req_sectors += blk_rq_sectors(cur);
+ phys_segments += req->nr_phys_segments;
+
+ if (rq_data_dir(cur) == WRITE) {
+ req_sectors++;
+ phys_segments++;
+ }
+
+ while (reqs < max_packed_rw - 1) {
+ next = blk_fetch_request(q);
+ if (!next)
+ break;
+
+ if (rq_data_dir(cur) != rq_data_dir(next)) {
+ blk_requeue_request(q, next);
+ break;
+ }
+
+ if (mmc_req_rel_wr(next) &&
+ (md->flags & MMC_BLK_REL_WR) &&
+ !en_rel_wr) {
+ blk_requeue_request(q, next);
+ break;
+ }
+
+ req_sectors += blk_rq_sectors(next);
+ if (req_sectors > max_blk_count) {
+ blk_requeue_request(q, next);
+ break;
+ }
+
+ phys_segments += next->nr_phys_segments;
+ if (phys_segments > max_phys_segs) {
+ blk_requeue_request(q, next);
+ break;
+ }
+
+ list_add_tail(&next->queuelist, &mq->mqrq_cur->packed_list);
+ cur = next;
+ reqs++;
+ }
+
+ if (reqs > 0) {
+ list_add(&req->queuelist, &mq->mqrq_cur->packed_list);
+ return (reqs + 1);
+ }
+
+no_packed:
+ mq->mqrq_cur->packed_cmd = MMC_PACKED_NONE;
+ return reqs;
+}
+
+static void mmc_blk_packed_hdr_wrq_prep(struct mmc_queue_req *mqrq,
+ struct mmc_card *card,
+ struct mmc_queue *mq,
+ u8 reqs)
+{
+ struct mmc_blk_request *brq = &mqrq->brq;
+ struct request *req = mqrq->req;
+ struct request *prq;
+ struct mmc_blk_data *md = mq->data;
+ bool do_rel_wr;
+ u32 *packed_cmd_hdr = mqrq->packed_cmd_hdr;
+ u8 i =1;
+
+ mqrq->packed_cmd = (rq_data_dir(req) == READ) ?
+ MMC_PACKED_WR_HDR : MMC_PACKED_WRITE;
+ mqrq->packed_blocks = 0;
+ mqrq->packed_fail_idx = -1;
+
+ memset(packed_cmd_hdr, 0, sizeof(mqrq->packed_cmd_hdr));
+ packed_cmd_hdr[0] = (reqs << 16) |
+ (((rq_data_dir(req) == READ) ? PACKED_CMD_RD: PACKED_CMD_WR) << 8) |
+ PACKED_CMD_VER;
+
+ /*
+ * Argument for each entry of packed group
+ */
+ list_for_each_entry(prq, &mqrq->packed_list, queuelist) {
+ do_rel_wr = mmc_req_rel_wr(prq) && (md->flags & MMC_BLK_REL_WR);
+ /* Argument of CMD23*/
+ packed_cmd_hdr[(i * 2)] = (do_rel_wr ? MMC_CMD23_ARG_REL_WR : 0) |
+ blk_rq_sectors(prq);
+ /* Argument of CMD18 or CMD25 */
+ packed_cmd_hdr[((i * 2)) + 1] = mmc_card_blockaddr(card) ?
+ blk_rq_pos(prq) : blk_rq_pos(prq) << 9;
+ mqrq->packed_blocks += blk_rq_sectors(prq);
+ i++;
+ }
+
+ memset(brq, 0, sizeof(struct mmc_blk_request));
+ brq->mrq.cmd = &brq->cmd;
+ brq->mrq.data = &brq->data;
+ brq->mrq.sbc = &brq->sbc;
+ brq->mrq.stop = &brq->stop;
+
+ brq->sbc.opcode = MMC_SET_BLOCK_COUNT;
+ brq->sbc.arg = MMC_CMD23_ARG_PACKED |
+ ((rq_data_dir(req) == READ) ? 1 : mqrq->packed_blocks + 1);
+ brq->sbc.flags = MMC_RSP_R1 | MMC_CMD_AC;
+
+ brq->cmd.opcode = MMC_WRITE_MULTIPLE_BLOCK;
+ brq->cmd.arg = blk_rq_pos(req);
+ if (!mmc_card_blockaddr(card))
+ brq->cmd.arg <<= 9;
+ brq->cmd.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_ADTC;
+
+ brq->data.blksz = 512;
+ /*
+ * Write separately the packd command header only for packed read.
+ * In case of packed write, header is sent with blocks of data.
+ */
+ brq->data.blocks = (rq_data_dir(req) == READ) ?
+ 1 : mqrq->packed_blocks + 1;
+ brq->data.flags |= MMC_DATA_WRITE;
+
+ brq->stop.opcode = MMC_STOP_TRANSMISSION;
+ brq->stop.arg = 0;
+ brq->stop.flags = MMC_RSP_SPI_R1B | MMC_RSP_R1B | MMC_CMD_AC;
+
+ mmc_set_data_timeout(&brq->data, card);
+
+ brq->data.sg = mqrq->sg;
+ brq->data.sg_len = mmc_queue_map_sg(mq, mqrq);
+
+ mqrq->mmc_active.mrq = &brq->mrq;
+ mqrq->mmc_active.err_check = mmc_blk_packed_err_check;
+
+ mmc_queue_bounce_pre(mqrq);
+}
+
+static void mmc_blk_packed_rrq_prep(struct mmc_queue_req *mqrq,
+ struct mmc_card *card,
+ struct mmc_queue *mq)
+{
+ struct mmc_blk_request *brq = &mqrq->brq;
+ struct request *req = mqrq->req;
+
+ mqrq->packed_cmd = MMC_PACKED_READ;
+
+ memset(brq, 0, sizeof(struct mmc_blk_request));
+ brq->mrq.cmd = &brq->cmd;
+ brq->mrq.data = &brq->data;
+ brq->mrq.stop = &brq->stop;
+
+ brq->cmd.opcode = MMC_READ_MULTIPLE_BLOCK;
+ brq->cmd.arg = blk_rq_pos(req);
+ if (!mmc_card_blockaddr(card))
+ brq->cmd.arg <<= 9;
+ brq->cmd.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_ADTC;
+ brq->data.blksz = 512;
+ brq->data.blocks = mqrq->packed_blocks;
+ brq->data.flags |= MMC_DATA_READ;
+
+ brq->stop.opcode = MMC_STOP_TRANSMISSION;
+ brq->stop.arg = 0;
+ brq->stop.flags = MMC_RSP_SPI_R1B | MMC_RSP_R1B | MMC_CMD_AC;
+
+ mmc_set_data_timeout(&brq->data, card);
+
+ brq->data.sg = mqrq->sg;
+ brq->data.sg_len = mmc_queue_map_sg(mq, mqrq);
+
+ mqrq->mmc_active.mrq = &brq->mrq;
+ mqrq->mmc_active.err_check = mmc_blk_packed_err_check;
+
+ mmc_queue_bounce_pre(mqrq);
+}
+
static int mmc_blk_cmd_err(struct mmc_blk_data *md, struct mmc_card *card,
struct mmc_blk_request *brq, struct request *req,
int ret)
@@ -1166,15 +1434,33 @@ static int mmc_blk_issue_rw_rq(struct mmc_queue *mq, struct request *rqc)
int ret = 1, disable_multi = 0, retry = 0, type;
enum mmc_blk_status status;
struct mmc_queue_req *mq_rq;
- struct request *req;
+ struct request *req, *prq;
struct mmc_async_req *areq;
+ u8 reqs = 0;
if (!rqc && !mq->mqrq_prev->req)
return 0;
+ if (rqc)
+ reqs = mmc_blk_chk_packable(mq, rqc);
+
do {
if (rqc) {
- mmc_blk_rw_rq_prep(mq->mqrq_cur, card, 0, mq);
+ if (reqs >= 2) {
+ mmc_blk_packed_hdr_wrq_prep(mq->mqrq_cur, card, mq, reqs);
+ if (rq_data_dir(rqc) == READ) {
+ areq = &mq->mqrq_cur->mmc_active;
+ mmc_wait_for_req(card->host, areq->mrq);
+ status = mmc_blk_packed_err_check(card, areq);
+ if (status == MMC_BLK_SUCCESS) {
+ mmc_blk_packed_rrq_prep(mq->mqrq_cur, card, mq);
+ } else {
+ goto check_status;
+ }
+ }
+ } else {
+ mmc_blk_rw_rq_prep(mq->mqrq_cur, card, 0, mq);
+ }
areq = &mq->mqrq_cur->mmc_active;
} else
areq = NULL;
@@ -1182,6 +1468,7 @@ static int mmc_blk_issue_rw_rq(struct mmc_queue *mq, struct request *rqc)
if (!areq)
return 0;
+check_status:
mq_rq = container_of(areq, struct mmc_queue_req, mmc_active);
brq = &mq_rq->brq;
req = mq_rq->req;
@@ -1195,10 +1482,32 @@ static int mmc_blk_issue_rw_rq(struct mmc_queue *mq, struct request *rqc)
* A block was successfully transferred.
*/
mmc_blk_reset_success(md, type);
- spin_lock_irq(&md->lock);
- ret = __blk_end_request(req, 0,
+
+ if (mq_rq->packed_cmd != MMC_PACKED_NONE) {
+ int idx = mq_rq->packed_fail_idx, i = 0;
+ while (!list_empty(&mq_rq->packed_list)) {
+ prq = list_entry_rq(mq_rq->packed_list.next);
+ if (idx == i) {
+ /* retry from error index */
+ reqs -= idx;
+ mq_rq->req = prq;
+ ret = 1;
+ break;
+ }
+ list_del_init(&prq->queuelist);
+ spin_lock_irq(&md->lock);
+ ret = __blk_end_request(prq, 0, blk_rq_bytes(prq));
+ spin_unlock_irq(&md->lock);
+ i++;
+ }
+ break;
+ } else {
+ spin_lock_irq(&md->lock);
+ ret = __blk_end_request(req, 0,
brq->data.bytes_xfered);
- spin_unlock_irq(&md->lock);
+ spin_unlock_irq(&md->lock);
+ }
+
/*
* If the blk_end_request function returns non-zero even
* though all data has been transferred and no errors
@@ -1257,7 +1566,7 @@ static int mmc_blk_issue_rw_rq(struct mmc_queue *mq, struct request *rqc)
break;
}
- if (ret) {
+ if (ret && (mq_rq->packed_cmd == MMC_PACKED_NONE)) {
/*
* In case of a incomplete request
* prepare it again and resend.
@@ -1270,13 +1579,37 @@ static int mmc_blk_issue_rw_rq(struct mmc_queue *mq, struct request *rqc)
return 1;
cmd_abort:
- spin_lock_irq(&md->lock);
- while (ret)
- ret = __blk_end_request(req, -EIO, blk_rq_cur_bytes(req));
- spin_unlock_irq(&md->lock);
+ if (mq_rq->packed_cmd != MMC_PACKED_NONE) {
+ spin_lock_irq(&md->lock);
+ while (ret)
+ ret = __blk_end_request(req, -EIO, blk_rq_cur_bytes(req));
+ spin_unlock_irq(&md->lock);
+ } else {
+ while (!list_empty(&mq_rq->packed_list)) {
+ prq = list_entry_rq(mq_rq->packed_list.next);
+ list_del_init(&prq->queuelist);
+ spin_lock_irq(&md->lock);
+ __blk_end_request(prq, -EIO, blk_rq_bytes(prq));
+ spin_unlock_irq(&md->lock);
+ }
+ }
start_new_req:
if (rqc) {
+ /*
+ * If current request is packed, it need to put back.
+ */
+ if (mq->mqrq_cur->packed_cmd != MMC_PACKED_NONE) {
+ while (!list_empty(&mq->mqrq_cur->packed_list)) {
+ prq = list_entry_rq(mq->mqrq_cur->packed_list.prev);
+ if (prq->queuelist.prev != &mq->mqrq_cur->packed_list) {
+ list_del_init(&prq->queuelist);
+ blk_requeue_request(mq->queue, prq);
+ } else {
+ list_del_init(&prq->queuelist);
+ }
+ }
+ }
mmc_blk_rw_rq_prep(mq->mqrq_cur, card, 0, mq);
mmc_start_req(card->host, &mq->mqrq_cur->mmc_active, NULL);
}
diff --git a/drivers/mmc/card/queue.c b/drivers/mmc/card/queue.c
index dcad59c..3a4542e 100644
--- a/drivers/mmc/card/queue.c
+++ b/drivers/mmc/card/queue.c
@@ -172,6 +172,8 @@ int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card,
memset(&mq->mqrq_cur, 0, sizeof(mq->mqrq_cur));
memset(&mq->mqrq_prev, 0, sizeof(mq->mqrq_prev));
+ INIT_LIST_HEAD(&mqrq_cur->packed_list);
+ INIT_LIST_HEAD(&mqrq_prev->packed_list);
mq->mqrq_cur = mqrq_cur;
mq->mqrq_prev = mqrq_prev;
mq->queue->queuedata = mq;
@@ -372,6 +374,39 @@ void mmc_queue_resume(struct mmc_queue *mq)
}
}
+static unsigned int mmc_queue_packed_map_sg(struct mmc_queue *mq,
+ struct mmc_queue_req *mqrq,
+ struct scatterlist *sg)
+{
+ struct scatterlist *__sg;
+ unsigned int sg_len = 0;
+ struct request *req;
+ enum mmc_packed_cmd cmd;
+
+ cmd = mqrq->packed_cmd;
+
+ if (cmd == MMC_PACKED_WR_HDR || cmd == MMC_PACKED_WRITE) {
+ __sg =sg;
+ sg_set_buf(__sg, mqrq->packed_cmd_hdr,
+ sizeof(mqrq->packed_cmd_hdr));
+ sg_len++;
+ if (cmd == MMC_PACKED_WR_HDR) {
+ sg_mark_end(__sg);
+ return sg_len;
+ }
+ __sg->page_link &= ~0x02;
+ }
+
+ __sg = sg + sg_len;
+ list_for_each_entry(req, &mqrq->packed_list, queuelist) {
+ sg_len += blk_rq_map_sg(mq->queue, req, __sg);
+ __sg = sg + (sg_len - 1);
+ (__sg++)->page_link &= ~0x02;
+ }
+ sg_mark_end(sg + (sg_len - 1));
+ return sg_len;
+}
+
/*
* Prepare the sg list(s) to be handed of to the host driver
*/
@@ -382,12 +417,19 @@ unsigned int mmc_queue_map_sg(struct mmc_queue *mq, struct mmc_queue_req *mqrq)
struct scatterlist *sg;
int i;
- if (!mqrq->bounce_buf)
- return blk_rq_map_sg(mq->queue, mqrq->req, mqrq->sg);
+ if (!mqrq->bounce_buf) {
+ if (!list_empty(&mqrq->packed_list))
+ return mmc_queue_packed_map_sg(mq, mqrq, mqrq->sg);
+ else
+ return blk_rq_map_sg(mq->queue, mqrq->req, mqrq->sg);
+ }
BUG_ON(!mqrq->bounce_sg);
- sg_len = blk_rq_map_sg(mq->queue, mqrq->req, mqrq->bounce_sg);
+ if (!list_empty(&mqrq->packed_list))
+ sg_len = mmc_queue_packed_map_sg(mq, mqrq, mqrq->bounce_sg);
+ else
+ sg_len = blk_rq_map_sg(mq->queue, mqrq->req, mqrq->bounce_sg);
mqrq->bounce_sg_len = sg_len;
diff --git a/drivers/mmc/card/queue.h b/drivers/mmc/card/queue.h
index d2a1eb4..5d0131e 100644
--- a/drivers/mmc/card/queue.h
+++ b/drivers/mmc/card/queue.h
@@ -12,6 +12,13 @@ struct mmc_blk_request {
struct mmc_data data;
};
+enum mmc_packed_cmd {
+ MMC_PACKED_NONE = 0,
+ MMC_PACKED_WR_HDR,
+ MMC_PACKED_WRITE,
+ MMC_PACKED_READ,
+};
+
struct mmc_queue_req {
struct request *req;
struct mmc_blk_request brq;
@@ -20,6 +27,11 @@ struct mmc_queue_req {
struct scatterlist *bounce_sg;
unsigned int bounce_sg_len;
struct mmc_async_req mmc_active;
+ struct list_head packed_list;
+ u32 packed_cmd_hdr[128];
+ unsigned int packed_blocks;
+ enum mmc_packed_cmd packed_cmd;
+ int packed_fail_idx;
};
struct mmc_queue {
diff --git a/include/linux/mmc/core.h b/include/linux/mmc/core.h
index 174a844..3c61d5a 100644
--- a/include/linux/mmc/core.h
+++ b/include/linux/mmc/core.h
@@ -18,6 +18,8 @@ struct mmc_request;
struct mmc_command {
u32 opcode;
u32 arg;
+#define MMC_CMD23_ARG_REL_WR (1 << 31)
+#define MMC_CMD23_ARG_PACKED ((0 << 31) | (1 << 30))
u32 resp[4];
unsigned int flags; /* expected response type */
#define MMC_RSP_PRESENT (1 << 0)
@@ -143,6 +145,7 @@ extern int mmc_app_cmd(struct mmc_host *, struct mmc_card *);
extern int mmc_wait_for_app_cmd(struct mmc_host *, struct mmc_card *,
struct mmc_command *, int);
extern int mmc_switch(struct mmc_card *, u8, u8, u8, unsigned int);
+extern int mmc_send_ext_csd(struct mmc_card *card, u8 *ext_csd);
#define MMC_ERASE_ARG 0x00000000
#define MMC_SECURE_ERASE_ARG 0x80000000
--
1.7.0.4
--
To unsubscribe from this list: send the line "unsubscribe linux-mmc" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
