This extends the PL011 UART driver with generic DMA engine support
using the PrimeCell DMA engine interface.
Tested-by: Jerzy Kasenberg <jerzy.kasenberg@xxxxxxxxx>
Tested-by: Grzegorz Sygieda <grzegorz.sygieda@xxxxxxxxx>
Tested-by: Marcin Mielczarczyk <marcin.mielczarczyk@xxxxxxxxx>
Signed-off-by: Linus Walleij <linus.walleij@xxxxxxxxxxxxxx>
---
Changes v5->v6:
- Fixes from Jerzy, Grzegorz and Marcin to enable DMA for RX/TX
in a more fine-grained manner and use the correct FIFO size for
controlling DMA behaviour.
- Rename state struct member use_dma to enable_dma for
consistency with the other PrimeCell DMA drivers
- Rewritten to fall back to interrupt mode when DMA
channel aquisition fails, and retry DMA again at first
possible instant.
- Tested with U300 DMA engine and fault injection (every or
every second DMA transaction fail) without problems.
---
drivers/serial/amba-pl011.c | 810 ++++++++++++++++++++++++++++++++++++++++++-
include/linux/amba/serial.h | 6 +
2 files changed, 807 insertions(+), 9 deletions(-)
diff --git a/drivers/serial/amba-pl011.c b/drivers/serial/amba-pl011.c
index eb4cb48..a651c74 100644
--- a/drivers/serial/amba-pl011.c
+++ b/drivers/serial/amba-pl011.c
@@ -7,6 +7,7 @@
*
* Copyright 1999 ARM Limited
* Copyright (C) 2000 Deep Blue Solutions Ltd.
+ * Copyright (C) 2010 ST-Ericsson SA
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -48,6 +49,12 @@
#include <linux/amba/serial.h>
#include <linux/clk.h>
#include <linux/slab.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+#include <linux/scatterlist.h>
+#include <linux/completion.h>
+#include <linux/amba/dma.h>
+#include <linux/delay.h>
#include <asm/io.h>
#include <asm/sizes.h>
@@ -63,6 +70,24 @@
#define UART_DR_ERROR (UART011_DR_OE|UART011_DR_BE|UART011_DR_PE|UART011_DR_FE)
#define UART_DUMMY_DR_RX (1 << 16)
+/* Deals with DMA transactions */
+struct pl011_dma_rx_transaction {
+ struct completion complete;
+ bool use_buffer_b;
+ struct scatterlist scatter_a;
+ struct scatterlist scatter_b;
+ char *rx_dma_buf_a;
+ char *rx_dma_buf_b;
+ dma_cookie_t cookie;
+};
+
+struct pl011_dma_tx_transaction {
+ struct completion complete;
+ struct scatterlist scatter;
+ char *tx_dma_buf;
+ dma_cookie_t cookie;
+};
+
/*
* We wrap our port structure around the generic uart_port.
*/
@@ -73,6 +98,16 @@ struct uart_amba_port {
unsigned int old_status;
unsigned int ifls; /* vendor-specific */
bool autorts;
+ unsigned int fifosize;
+ /* DMA stuff */
+ bool enable_dma;
+ bool rx_dma_running;
+#ifdef CONFIG_DMADEVICES
+ struct dma_chan *dma_rx_channel;
+ struct dma_chan *dma_tx_channel;
+ struct pl011_dma_rx_transaction dmarx;
+ struct pl011_dma_tx_transaction dmatx;
+#endif
};
/* There is by now at least one vendor with differing details, so handle it */
@@ -91,6 +126,683 @@ static struct vendor_data vendor_st = {
.fifosize = 64,
};
+/*
+ * All the DMA operation mode stuff goes inside this ifdef.
+ * This assumes that you have a generic DMA device interface,
+ * no custom DMA interfaces are supported.
+ */
+#ifdef CONFIG_DMADEVICES
+
+#define PL011_DMA_BUFFER_SIZE PAGE_SIZE
+
+static void __init pl011_dma_probe_initcall(struct uart_amba_port *uap)
+{
+ /* DMA is the sole user of the platform data right now */
+ struct amba_pl011_data *plat = uap->port.dev->platform_data;
+ struct pl011_dma_rx_transaction *dmarx = &uap->dmarx;
+ struct pl011_dma_tx_transaction *dmatx = &uap->dmatx;
+ struct amba_dma_channel_config rx_conf = {
+ .addr = uap->port.mapbase + UART01x_DR,
+ .addr_width = 1,
+ .direction = DMA_FROM_DEVICE,
+ .maxburst = uap->fifosize >> 1,
+ };
+ struct amba_dma_channel_config tx_conf = {
+ .addr = uap->port.mapbase + UART01x_DR,
+ .addr_width = 1,
+ .direction = DMA_TO_DEVICE,
+ .maxburst = uap->fifosize >> 1,
+ };
+ dma_cap_mask_t mask;
+ int sglen;
+
+ /* We need platform data */
+ if (!plat) {
+ dev_err(uap->port.dev, "no DMA platform data!\n");
+ return;
+ }
+
+ /* Try to acquire a generic DMA engine slave channel */
+ dma_cap_zero(mask);
+ dma_cap_set(DMA_SLAVE, mask);
+ /*
+ * We need both RX and TX channels to do DMA, else do none
+ * of them.
+ */
+ uap->dma_rx_channel = dma_request_channel(mask,
+ plat->dma_filter,
+ plat->dma_rx_param);
+ if (!uap->dma_rx_channel) {
+ dev_err(uap->port.dev, "no RX DMA channel!\n");
+ return;
+ }
+ dma_set_ambaconfig(uap->dma_rx_channel, &rx_conf);
+
+ uap->dma_tx_channel = dma_request_channel(mask,
+ plat->dma_filter,
+ plat->dma_tx_param);
+ if (!uap->dma_tx_channel) {
+ dev_err(uap->port.dev, "no TX DMA channel!\n");
+ goto err_no_txchan;
+ }
+ dma_set_ambaconfig(uap->dma_tx_channel, &tx_conf);
+
+ /* Allocate DMA RX and TX buffers */
+ dmarx->rx_dma_buf_a = kmalloc(PL011_DMA_BUFFER_SIZE, GFP_KERNEL);
+ if (!dmarx->rx_dma_buf_a) {
+ dev_err(uap->port.dev, "failed to allocate DMA RX buffer A\n");
+ goto err_no_rxbuf_a;
+ }
+
+ dmarx->rx_dma_buf_b = kmalloc(PL011_DMA_BUFFER_SIZE, GFP_KERNEL);
+ if (!dmarx->rx_dma_buf_b) {
+ dev_err(uap->port.dev, "failed to allocate DMA RX buffer B\n");
+ goto err_no_rxbuf_b;
+ }
+
+ dmatx->tx_dma_buf = kmalloc(PL011_DMA_BUFFER_SIZE, GFP_KERNEL);
+ if (!dmatx->tx_dma_buf) {
+ dev_err(uap->port.dev, "failed to allocate DMA TX buffer\n");
+ goto err_no_txbuf;
+ }
+
+ /* Provide single SG list with one item to the buffers */
+ sg_init_one(&dmarx->scatter_a, dmarx->rx_dma_buf_a,
+ PL011_DMA_BUFFER_SIZE);
+ sg_init_one(&dmarx->scatter_b, dmarx->rx_dma_buf_b,
+ PL011_DMA_BUFFER_SIZE);
+ sg_init_one(&dmatx->scatter, dmatx->tx_dma_buf, PL011_DMA_BUFFER_SIZE);
+
+ /* Map DMA buffers */
+ sglen = dma_map_sg(uap->port.dev, &dmarx->scatter_a,
+ 1, DMA_FROM_DEVICE);
+ if (sglen != 1)
+ goto err_rx_sgmap_a;
+
+ sglen = dma_map_sg(uap->port.dev, &dmarx->scatter_b,
+ 1, DMA_FROM_DEVICE);
+ if (sglen != 1)
+ goto err_rx_sgmap_b;
+
+ sglen = dma_map_sg(uap->port.dev, &dmatx->scatter,
+ 1, DMA_TO_DEVICE);
+ if (sglen != 1)
+ goto err_tx_sgmap;
+
+ /* Initially we say the transfers are incomplete */
+ init_completion(&uap->dmatx.complete);
+ complete(&uap->dmatx.complete);
+
+ /* The DMA buffer is now the FIFO the TTY subsystem can use */
+ uap->port.fifosize = PL011_DMA_BUFFER_SIZE;
+
+ uap->enable_dma = true;
+ dev_info(uap->port.dev, "setup for DMA on RX %s, TX %s\n",
+ dma_chan_name(uap->dma_rx_channel),
+ dma_chan_name(uap->dma_tx_channel));
+ return;
+
+err_tx_sgmap:
+ dma_unmap_sg(uap->port.dev, &dmarx->scatter_b,
+ 1, DMA_FROM_DEVICE);
+err_rx_sgmap_b:
+ dma_unmap_sg(uap->port.dev, &dmarx->scatter_a,
+ 1, DMA_FROM_DEVICE);
+err_rx_sgmap_a:
+ kfree(dmatx->tx_dma_buf);
+err_no_txbuf:
+ kfree(dmarx->rx_dma_buf_b);
+err_no_rxbuf_b:
+ kfree(dmarx->rx_dma_buf_a);
+err_no_rxbuf_a:
+ dma_release_channel(uap->dma_tx_channel);
+ uap->dma_tx_channel = NULL;
+err_no_txchan:
+ dma_release_channel(uap->dma_rx_channel);
+ uap->dma_rx_channel = NULL;
+ return;
+}
+
+/*
+ * Stack up the UARTs and let the above initcall be done at
+ * device initcall time, because the serial driver is called as
+ * an arch initcall, and at this time the DMA subsystem is not yet
+ * registered. At this point the driver will switch over to using
+ * DMA where desired.
+ */
+
+struct dma_uap {
+ struct list_head node;
+ struct uart_amba_port *uap;
+};
+
+struct list_head __initdata pl011_dma_uarts = LIST_HEAD_INIT(pl011_dma_uarts);
+
+static int __init pl011_dma_initcall(void)
+{
+ struct list_head *node, *tmp;
+
+ list_for_each_safe(node, tmp, &pl011_dma_uarts) {
+ struct dma_uap *dmau = list_entry(node, struct dma_uap, node);
+ pl011_dma_probe_initcall(dmau->uap);
+ list_del(node);
+ kfree(dmau);
+ }
+ return 0;
+}
+
+device_initcall(pl011_dma_initcall);
+
+static void pl011_dma_probe(struct uart_amba_port *uap)
+{
+ struct dma_uap *dmau = kzalloc(sizeof(struct dma_uap), GFP_KERNEL);
+
+ if (dmau == NULL)
+ return;
+ dmau->uap = uap;
+ list_add_tail(&dmau->node, &pl011_dma_uarts);
+}
+
+static void pl011_dma_remove(struct uart_amba_port *uap)
+{
+ struct pl011_dma_rx_transaction *dmarx = &uap->dmarx;
+ struct pl011_dma_tx_transaction *dmatx = &uap->dmatx;
+
+ /* TODO: remove the initcall if it has not yet executed */
+ /* Unmap and free DMA buffers */
+ if (uap->dma_rx_channel)
+ dma_release_channel(uap->dma_rx_channel);
+ if (uap->dma_tx_channel)
+ dma_release_channel(uap->dma_tx_channel);
+ if (dmatx->tx_dma_buf) {
+ dma_unmap_sg(uap->port.dev, &dmatx->scatter,
+ 1, DMA_TO_DEVICE);
+ kfree(dmatx->tx_dma_buf);
+ }
+ if (dmarx->rx_dma_buf_b) {
+ dma_unmap_sg(uap->port.dev, &dmarx->scatter_b,
+ 1, DMA_FROM_DEVICE);
+ kfree(dmarx->rx_dma_buf_b);
+ }
+ if (dmarx->rx_dma_buf_a) {
+ dma_unmap_sg(uap->port.dev, &dmarx->scatter_a,
+ 1, DMA_FROM_DEVICE);
+ kfree(dmarx->rx_dma_buf_a);
+ }
+}
+
+/* Forward declare this for the refill routine */
+static int pl011_dma_tx_refill(struct uart_amba_port *uap);
+
+/*
+ * Move the tail when this IRQ occurs, if not empty refill and
+ * fire another transaction
+ */
+static void pl011_dma_tx_callback(void *data)
+{
+ struct uart_amba_port *uap = data;
+ struct pl011_dma_tx_transaction *dmatx = &uap->dmatx;
+ struct circ_buf *xmit = &uap->port.state->xmit;
+ u16 val;
+ int ret;
+
+ /* Temporarily disable TX DMA */
+ val = readw(uap->port.membase + UART011_DMACR);
+ val &= ~(UART011_TXDMAE);
+ writew(val, uap->port.membase + UART011_DMACR);
+
+ /* Refill the TX if the buffer is not empty */
+ if (!uart_circ_empty(xmit)) {
+ ret = pl011_dma_tx_refill(uap);
+ if (ret == -EBUSY)
+ /*
+ * If DMA cannot be used right now, we complete this
+ * transaction and let the TTY layer retry. If the
+ * firs following xfer fails to set up for DMA, it
+ * will fall through to interrupt mode.
+ */
+ dev_dbg(uap->port.dev, "DMA busy\n");
+ } else {
+ complete(&dmatx->complete);
+ }
+}
+
+static int pl011_dma_tx_refill(struct uart_amba_port *uap)
+{
+ struct pl011_dma_tx_transaction *dmatx = &uap->dmatx;
+ struct dma_chan *chan = uap->dma_tx_channel;
+ struct dma_async_tx_descriptor *desc;
+ struct circ_buf *xmit = &uap->port.state->xmit;
+ unsigned int count;
+ unsigned long flags;
+ u16 val;
+
+ /* Don't bother about using DMA on XON/XOFF */
+ if (uap->port.x_char) {
+ /* If we can't get it into the FIFO, retry later */
+ if (readw(uap->port.membase + UART01x_FR) &
+ UART01x_FR_TXFF) {
+ complete(&dmatx->complete);
+ return 0;
+ }
+ writew(uap->port.x_char, uap->port.membase + UART01x_DR);
+ uap->port.icount.tx++;
+ uap->port.x_char = 0;
+ complete(&dmatx->complete);
+ return 0;
+ }
+
+ /*
+ * Try to avoid the overhead involved in using DMA if the
+ * transaction fits in the first half of the FIFO and it's not
+ * full. Unfortunately there is only one single bit in the
+ * hardware to tell whether the FIFO is full or not, so
+ * we don't know exactly how many chars we can fit in.
+ */
+ if (uart_circ_chars_pending(xmit) < (uap->fifosize >> 1)) {
+ while (uart_circ_chars_pending(xmit)) {
+ if (readw(uap->port.membase + UART01x_FR) &
+ UART01x_FR_TXFF) {
+ /*
+ * Ooops TX FIFO is full, we'd better stop
+ * this. Let's enable TX interrupt here to get
+ * informed when there is again some space in
+ * the TX FIFO so we can continue the transfer.
+ * This interrupt will be cleared just before
+ * setting up DMA, as it could interfere with
+ * TX interrupt handling routine.
+ */
+ uap->im |= UART011_TXIM;
+ writew(uap->im,
+ uap->port.membase + UART011_IMSC);
+ break;
+ }
+ writew(xmit->buf[xmit->tail],
+ uap->port.membase + UART01x_DR);
+ uap->port.icount.tx++;
+ xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1);
+ }
+ complete(&dmatx->complete);
+ return 0;
+ }
+
+ /*
+ * Clear TX interrupt to be sure that DMA will not interfere with
+ * TX ISR
+ */
+ local_irq_save(flags);
+ uap->im &= ~UART011_TXIM;
+ writew(uap->im, uap->port.membase + UART011_IMSC);
+ local_irq_restore(flags);
+
+ /* Else proceed to copy the TX chars to the DMA buffer and fire DMA */
+ count = uart_circ_chars_pending(xmit);
+ if (count > PL011_DMA_BUFFER_SIZE)
+ count = PL011_DMA_BUFFER_SIZE;
+
+ if (xmit->tail < xmit->head)
+ memcpy(&dmatx->tx_dma_buf[0], &xmit->buf[xmit->tail], count);
+ else {
+ size_t first = UART_XMIT_SIZE - xmit->tail;
+ size_t second = xmit->head;
+
+ memcpy(&dmatx->tx_dma_buf[0], &xmit->buf[xmit->tail], first);
+ memcpy(&dmatx->tx_dma_buf[first], &xmit->buf[0], second);
+ }
+
+ dmatx->scatter.length = count;
+
+ /* Synchronize the scatterlist, invalidate buffers, caches etc */
+ dma_sync_sg_for_device(uap->port.dev,
+ &dmatx->scatter,
+ 1,
+ DMA_TO_DEVICE);
+
+ /* Send the scatterlist */
+ desc = chan->device->device_prep_slave_sg(chan,
+ &dmatx->scatter,
+ 1,
+ DMA_TO_DEVICE,
+ DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+ if (!desc) {
+ /* "Complete" DMA (errorpath) */
+ complete(&dmatx->complete);
+ return -EBUSY;
+ }
+
+ /*
+ * Now we know that DMA will fire, so advance the ring buffer
+ * with the stuff we just dispatched
+ */
+ xmit->tail = (xmit->tail + count) & (UART_XMIT_SIZE - 1);
+ uap->port.icount.tx += count;
+ if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
+ uart_write_wakeup(&uap->port);
+
+ /* Some data to go along to the callback */
+ desc->callback = pl011_dma_tx_callback;
+ desc->callback_param = uap;
+ dmatx->cookie = desc->tx_submit(desc);
+ chan->device->device_issue_pending(chan);
+
+ val = readw(uap->port.membase + UART011_DMACR);
+ val |= UART011_TXDMAE;
+ writew(val, uap->port.membase + UART011_DMACR);
+ return 0;
+}
+
+static void pl011_dma_rx_callback(void *data);
+
+static int pl011_dma_rx_trigger_dma(struct uart_amba_port *uap)
+{
+ struct dma_chan *rxchan = uap->dma_rx_channel;
+ struct pl011_dma_rx_transaction *dmarx = &uap->dmarx;
+ struct dma_async_tx_descriptor *desc;
+ struct scatterlist *scatter = dmarx->use_buffer_b ?
+ &dmarx->scatter_b : &dmarx->scatter_a;
+ u16 val;
+
+ /* Start the RX DMA job */
+ desc = rxchan->device->device_prep_slave_sg(rxchan,
+ scatter,
+ 1,
+ DMA_FROM_DEVICE,
+ DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+ /*
+ * If the DMA engine is busy and cannot prepare a
+ * channel, no big deal, the driver will fall back
+ * to interrupt mode as a result of this error code.
+ */
+ if (!desc) {
+ uap->rx_dma_running = false;
+ return -EBUSY;
+ }
+
+ /* Some data to go along to the callback */
+ desc->callback = pl011_dma_rx_callback;
+ desc->callback_param = uap;
+ dmarx->cookie = desc->tx_submit(desc);
+ rxchan->device->device_issue_pending(rxchan);
+
+ val = readw(uap->port.membase + UART011_DMACR);
+ val |= UART011_RXDMAE;
+ writew(val, uap->port.membase + UART011_DMACR);
+ uap->rx_dma_running = true;
+
+ return 0;
+}
+
+/*
+ * This is called when either the DMA job is complete, or
+ * the FIFO timeout interrupt occurred. This must be called
+ * with the port spinlock uap->port.lock held.
+ */
+static void pl011_dma_rx_chars(struct uart_amba_port *uap,
+ u32 pending, bool use_buffer_b,
+ bool readfifo)
+{
+ struct pl011_dma_rx_transaction *dmarx = &uap->dmarx;
+ struct tty_struct *tty = uap->port.state->port.tty;
+ char *buf = use_buffer_b ? dmarx->rx_dma_buf_b : dmarx->rx_dma_buf_a;
+ struct scatterlist *scatter = use_buffer_b ?
+ &dmarx->scatter_b : &dmarx->scatter_a;
+ unsigned int status, ch, flag;
+ u32 count = pending;
+ u32 bufp = 0;
+ u32 fifotaken = 0; /* only used for vdbg() */
+
+ /* Sync in buffer */
+ dma_sync_sg_for_cpu(uap->port.dev,
+ scatter,
+ 1,
+ DMA_FROM_DEVICE);
+
+ status = readw(uap->port.membase + UART01x_FR);
+
+ /*
+ * First take all chars in the DMA pipe, then look
+ * in the FIFO. So loop while we have chars in the
+ * DMA buffer or the FIFO. If we came here from a
+ * DMA buffer full interrupt, there is already another
+ * DMA job triggered to read the FIFO, so don't look
+ * at it.
+ */
+ while (count ||
+ (readfifo && (status & UART01x_FR_RXFE) == 0)) {
+
+ flag = TTY_NORMAL;
+ uap->port.icount.rx++;
+
+ if (count) {
+ /* Take chars from the DMA buffer */
+ ch = (unsigned int) buf[bufp];
+ bufp++;
+ count--;
+ } else {
+ /* Take chars from the FIFO and update status */
+ ch = readw(uap->port.membase + UART01x_DR);
+ status = readw(uap->port.membase + UART01x_FR);
+ fifotaken++;
+
+ /*
+ * Error conditions will only occur in the FIFO,
+ * these will trigger an immediate interrupt and
+ * stop the DMA job, so we will always find the
+ * error in the FIFO, never in the DMA buffer.
+ */
+ if (unlikely(ch & UART_DR_ERROR)) {
+ if (ch & UART011_DR_BE) {
+ ch &= ~(UART011_DR_FE | UART011_DR_PE);
+ uap->port.icount.brk++;
+ if (uart_handle_break(&uap->port))
+ continue;
+ } else if (ch & UART011_DR_PE)
+ uap->port.icount.parity++;
+ else if (ch & UART011_DR_FE)
+ uap->port.icount.frame++;
+ if (ch & UART011_DR_OE)
+ uap->port.icount.overrun++;
+
+ ch &= uap->port.read_status_mask;
+
+ if (ch & UART011_DR_BE)
+ flag = TTY_BREAK;
+ else if (ch & UART011_DR_PE)
+ flag = TTY_PARITY;
+ else if (ch & UART011_DR_FE)
+ flag = TTY_FRAME;
+ }
+ }
+
+ if (uart_handle_sysrq_char(&uap->port, ch & 255))
+ continue;
+
+ uart_insert_char(&uap->port, ch, UART011_DR_OE, ch, flag);
+ }
+
+ spin_unlock(&uap->port.lock);
+ dev_vdbg(uap->port.dev,
+ "Took %d chars from DMA buffer and %d chars from the FIFO\n",
+ bufp, fifotaken);
+ tty_flip_buffer_push(tty);
+ spin_lock(&uap->port.lock);
+}
+
+static void pl011_dma_rx_irq(struct uart_amba_port *uap)
+{
+ struct dma_chan *rxchan = uap->dma_rx_channel;
+ struct pl011_dma_rx_transaction *dmarx = &uap->dmarx;
+ struct scatterlist *scatter = dmarx->use_buffer_b ?
+ &dmarx->scatter_b : &dmarx->scatter_a;
+ u32 pending;
+ int ret;
+ struct dma_tx_state state;
+ enum dma_status dmastat;
+ u16 val;
+
+ /* Use PrimeCell DMA extensions to stop the transfer */
+ ret = rxchan->device->device_control(rxchan, DMA_PAUSE);
+ if (ret)
+ dev_err(uap->port.dev, "unable to pause DMA transfer\n");
+ dmastat = rxchan->device->device_tx_status(rxchan,
+ dmarx->cookie, &state);
+
+ /* Disable RX DMA temporarily */
+ val = readw(uap->port.membase + UART011_DMACR);
+ val &= ~(UART011_RXDMAE);
+ writew(val, uap->port.membase + UART011_DMACR);
+ uap->rx_dma_running = false;
+
+ if (dmastat != DMA_PAUSED)
+ dev_err(uap->port.dev, "unable to pause DMA transfer\n");
+ pending = scatter->length - state.residue;
+
+ BUG_ON(pending > PL011_DMA_BUFFER_SIZE);
+
+ ret = rxchan->device->device_control(rxchan, DMA_TERMINATE_ALL);
+ if (ret)
+ dev_err(uap->port.dev, "unable to terminate DMA transfer\n");
+
+ /*
+ * This will take the chars we have so far and insert
+ * into the framework.
+ */
+ pl011_dma_rx_chars(uap, pending, dmarx->use_buffer_b, true);
+
+ /* Switch buffer & re-trigger DMA job */
+ dmarx->use_buffer_b = !dmarx->use_buffer_b;
+ ret = pl011_dma_rx_trigger_dma(uap);
+ if (ret) {
+ dev_dbg(uap->port.dev, "could not retrigger RX DMA job "
+ "fall back to interrupt mode\n");
+ uap->im |= UART011_RXIM;
+ writew(uap->im, uap->port.membase + UART011_IMSC);
+ }
+}
+
+static void pl011_dma_rx_callback(void *data)
+{
+ struct uart_amba_port *uap = data;
+ struct pl011_dma_rx_transaction *dmarx = &uap->dmarx;
+ bool lastbuf = dmarx->use_buffer_b;
+ int ret;
+
+ /*
+ * This completion interrupt occurs typically when the
+ * RX buffer is totally stuffed but no timeout has yet
+ * occurred. When that happens, we just want the RX
+ * routine to flush out the secondary DMA buffer while
+ * we immediately trigger the next DMA job.
+ */
+ uap->rx_dma_running = false;
+ dmarx->use_buffer_b = !lastbuf;
+ ret = pl011_dma_rx_trigger_dma(uap);
+
+ spin_lock_irq(&uap->port.lock);
+ pl011_dma_rx_chars(uap, PL011_DMA_BUFFER_SIZE, lastbuf, false);
+ spin_unlock_irq(&uap->port.lock);
+ /*
+ * Do this check after we picked the DMA chars so we don't
+ * get some IRQ immediately from RX.
+ */
+ if (ret) {
+ dev_dbg(uap->port.dev, "could not retrigger RX DMA job "
+ "fall back to interrupt mode\n");
+ uap->im |= UART011_RXIM;
+ writew(uap->im, uap->port.membase + UART011_IMSC);
+ }
+}
+
+static void pl011_dma_startup(struct uart_amba_port *uap)
+{
+ u16 val;
+ int ret = 0;
+
+ if (!uap->enable_dma)
+ return;
+
+ /* Turn on DMA error (RX/TX will be enabled on demand) */
+ val = readw(uap->port.membase + UART011_DMACR);
+ val |= UART011_DMAONERR;
+ writew(val, uap->port.membase + UART011_DMACR);
+
+ ret = pl011_dma_rx_trigger_dma(uap);
+ if (ret)
+ dev_dbg(uap->port.dev, "could not trigger initial "
+ "RX DMA job, fall back to interrupt mode\n");
+}
+
+static void pl011_dma_shutdown(struct uart_amba_port *uap)
+{
+ struct dma_chan *rxchan = uap->dma_rx_channel;
+ struct dma_chan *txchan = uap->dma_tx_channel;
+ u16 val;
+
+ if (!uap->enable_dma)
+ return;
+
+ /* Disable RX and TX DMA */
+ while (readw(uap->port.membase + UART01x_FR) & UART01x_FR_BUSY)
+ barrier();
+ val = readw(uap->port.membase + UART011_DMACR);
+ val &= ~(UART011_DMAONERR | UART011_RXDMAE | UART011_TXDMAE);
+ writew(val, uap->port.membase + UART011_DMACR);
+ /* Terminate any RX and TX DMA jobs */
+ rxchan->device->device_control(rxchan, DMA_TERMINATE_ALL);
+ txchan->device->device_control(txchan, DMA_TERMINATE_ALL);
+}
+
+static int pl011_dma_tx_chars(struct uart_amba_port *uap)
+{
+ struct pl011_dma_tx_transaction *dmatx = &uap->dmatx;
+
+ /* Try to wait for completion, return if something is in progress */
+ if (!try_wait_for_completion(&dmatx->complete))
+ return -EINPROGRESS;
+
+ /* Set up and fire the DMA job */
+ init_completion(&dmatx->complete);
+ return pl011_dma_tx_refill(uap);
+}
+
+#else
+/* Blank functions if the DMA engine is not available */
+static inline void pl011_dma_probe(struct uart_amba_port *uap)
+{
+}
+
+static inline void pl011_dma_remove(struct uart_amba_port *uap)
+{
+}
+
+static inline void pl011_dma_rx_irq(struct uart_amba_port *uap)
+{
+}
+
+static inline int pl011_dma_rx_trigger_dma(struct uart_amba_port *uap)
+{
+ return -EIO;
+}
+
+static inline void pl011_dma_startup(struct uart_amba_port *uap)
+{
+}
+
+static inline void pl011_dma_shutdown(struct uart_amba_port *uap)
+{
+}
+
+static inline int pl011_dma_tx_chars(struct uart_amba_port *uap)
+{
+ return -EIO;
+}
+
+static inline void pl011_dma_rx_timeout(struct uart_amba_port *uap)
+{
+}
+#endif
+
+
static void pl011_stop_tx(struct uart_port *port)
{
struct uart_amba_port *uap = (struct uart_amba_port *)port;
@@ -99,10 +811,18 @@ static void pl011_stop_tx(struct uart_port *port)
writew(uap->im, uap->port.membase + UART011_IMSC);
}
+static void pl011_tx_chars(struct uart_amba_port *uap);
+
static void pl011_start_tx(struct uart_port *port)
{
struct uart_amba_port *uap = (struct uart_amba_port *)port;
+ if (uap->enable_dma) {
+ /* Immediately push out chars in DMA mode */
+ pl011_tx_chars(uap);
+ return;
+ }
+ /* In interrupt mode, let the interrupt pull chars */
uap->im |= UART011_TXIM;
writew(uap->im, uap->port.membase + UART011_IMSC);
}
@@ -128,6 +848,7 @@ static void pl011_rx_chars(struct uart_amba_port *uap)
{
struct tty_struct *tty = uap->port.state->port.tty;
unsigned int status, ch, flag, max_count = 256;
+ int ret;
status = readw(uap->port.membase + UART01x_FR);
while ((status & UART01x_FR_RXFE) == 0 && max_count--) {
@@ -172,6 +893,21 @@ static void pl011_rx_chars(struct uart_amba_port *uap)
}
spin_unlock(&uap->port.lock);
tty_flip_buffer_push(tty);
+ /*
+ * If we were temporarily out of DMA mode for a while,
+ * attempt to switch back to DMA mode again.
+ */
+ if (uap->enable_dma) {
+ uap->im &= ~UART011_RXIM;
+ writew(uap->im, uap->port.membase + UART011_IMSC);
+ ret = pl011_dma_rx_trigger_dma(uap);
+ if (ret) {
+ dev_dbg(uap->port.dev, "could not trigger RX DMA job "
+ "fall back to interrupt mode again\n");
+ uap->im |= UART011_RXIM;
+ writew(uap->im, uap->port.membase + UART011_IMSC);
+ }
+ }
spin_lock(&uap->port.lock);
}
@@ -180,6 +916,25 @@ static void pl011_tx_chars(struct uart_amba_port *uap)
struct circ_buf *xmit = &uap->port.state->xmit;
int count;
+ if (uap->enable_dma) {
+ int ret;
+
+ ret = pl011_dma_tx_chars(uap);
+ if (!ret)
+ return;
+ if (ret == -EINPROGRESS)
+ return;
+
+ /*
+ * On any other error (including -EBUSY which is emitted
+ * in case the DMA engine is out of physical channels
+ * for example) we fall through to interrupt mode
+ */
+ dev_dbg(uap->port.dev, "DMA unavailable for TX\n");
+ uap->im |= UART011_TXIM;
+ writew(uap->im, uap->port.membase + UART011_IMSC);
+ }
+
if (uap->port.x_char) {
writew(uap->port.x_char, uap->port.membase + UART01x_DR);
uap->port.icount.tx++;
@@ -191,8 +946,10 @@ static void pl011_tx_chars(struct uart_amba_port *uap)
return;
}
- count = uap->port.fifosize >> 1;
+ count = uap->fifosize >> 1;
do {
+ if (readw(uap->port.membase + UART01x_FR) & UART01x_FR_TXFF)
+ break;
writew(xmit->buf[xmit->tail], uap->port.membase + UART01x_DR);
xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1);
uap->port.icount.tx++;
@@ -237,7 +994,7 @@ static irqreturn_t pl011_int(int irq, void *dev_id)
unsigned int status, pass_counter = AMBA_ISR_PASS_LIMIT;
int handled = 0;
- spin_lock(&uap->port.lock);
+ spin_lock_irq(&uap->port.lock);
status = readw(uap->port.membase + UART011_MIS);
if (status) {
@@ -246,13 +1003,30 @@ static irqreturn_t pl011_int(int irq, void *dev_id)
UART011_RXIS),
uap->port.membase + UART011_ICR);
- if (status & (UART011_RTIS|UART011_RXIS))
- pl011_rx_chars(uap);
+ if (status & (UART011_RTIS|UART011_RXIS)) {
+ if (uap->enable_dma && uap->rx_dma_running)
+ pl011_dma_rx_irq(uap);
+ else
+ pl011_rx_chars(uap);
+ }
if (status & (UART011_DSRMIS|UART011_DCDMIS|
UART011_CTSMIS|UART011_RIMIS))
pl011_modem_status(uap);
- if (status & UART011_TXIS)
+ if (status & UART011_TXIS) {
+ /*
+ * When DMA is enabled we still use TX
+ * interrupt to send small amounts of data,
+ * and as a fallback when the DMA channel is
+ * not available. This interrupt is cleared
+ * here and will be enabled when it's needed.
+ */
+ if (uap->enable_dma) {
+ uap->im &= ~UART011_TXIM;
+ writew(uap->im,
+ uap->port.membase + UART011_IMSC);
+ }
pl011_tx_chars(uap);
+ }
if (pass_counter-- == 0)
break;
@@ -262,7 +1036,7 @@ static irqreturn_t pl011_int(int irq, void *dev_id)
handled = 1;
}
- spin_unlock(&uap->port.lock);
+ spin_unlock_irq(&uap->port.lock);
return IRQ_RETVAL(handled);
}
@@ -401,16 +1175,28 @@ static int pl011_startup(struct uart_port *port)
cr = UART01x_CR_UARTEN | UART011_CR_RXE | UART011_CR_TXE;
writew(cr, uap->port.membase + UART011_CR);
+ /* Clear pending error interrupts*/
+ writew(0xFFFF & ~(UART011_TXIS | UART011_RTIS | UART011_RXIS),
+ uap->port.membase + UART011_ICR);
+
/*
* initialise the old status of the modem signals
*/
uap->old_status = readw(uap->port.membase + UART01x_FR) & UART01x_FR_MODEM_ANY;
+ /* Startup DMA */
+ pl011_dma_startup(uap);
+
/*
- * Finally, enable interrupts
+ * Finally, enable interrupts, only timeouts when using DMA
+ * if initial RX DMA job failed, start in interrupt mode
+ * as well.
*/
spin_lock_irq(&uap->port.lock);
- uap->im = UART011_RXIM | UART011_RTIM;
+ if (uap->enable_dma && uap->rx_dma_running)
+ uap->im = UART011_RTIM;
+ else
+ uap->im = UART011_RXIM | UART011_RTIM;
writew(uap->im, uap->port.membase + UART011_IMSC);
spin_unlock_irq(&uap->port.lock);
@@ -427,6 +1213,8 @@ static void pl011_shutdown(struct uart_port *port)
struct uart_amba_port *uap = (struct uart_amba_port *)port;
unsigned long val;
+ pl011_dma_shutdown(uap);
+
/*
* disable all interrupts
*/
@@ -496,7 +1284,7 @@ pl011_set_termios(struct uart_port *port, struct ktermios *termios,
if (!(termios->c_cflag & PARODD))
lcr_h |= UART01x_LCRH_EPS;
}
- if (port->fifosize > 1)
+ if (uap->fifosize > 1)
lcr_h |= UART01x_LCRH_FEN;
spin_lock_irqsave(&port->lock, flags);
@@ -809,6 +1597,8 @@ static int pl011_probe(struct amba_device *dev, struct amba_id *id)
uap->port.ops = &amba_pl011_pops;
uap->port.flags = UPF_BOOT_AUTOCONF;
uap->port.line = i;
+ uap->fifosize = vendor->fifosize;
+ pl011_dma_probe(uap);
amba_ports[i] = uap;
@@ -817,6 +1607,7 @@ static int pl011_probe(struct amba_device *dev, struct amba_id *id)
if (ret) {
amba_set_drvdata(dev, NULL);
amba_ports[i] = NULL;
+ pl011_dma_remove(uap);
clk_put(uap->clk);
unmap:
iounmap(base);
@@ -840,6 +1631,7 @@ static int pl011_remove(struct amba_device *dev)
if (amba_ports[i] == uap)
amba_ports[i] = NULL;
+ pl011_dma_remove(uap);
iounmap(uap->port.membase);
clk_put(uap->clk);
kfree(uap);
diff --git a/include/linux/amba/serial.h b/include/linux/amba/serial.h
index 5a5a7fd..2ce6a75 100644
--- a/include/linux/amba/serial.h
+++ b/include/linux/amba/serial.h
@@ -166,6 +166,12 @@ struct amba_device; /* in uncompress this is included but amba/bus.h is not */
struct amba_pl010_data {
void (*set_mctrl)(struct amba_device *dev, void __iomem *base, unsigned int mctrl);
};
+struct dma_chan;
+struct amba_pl011_data {
+ bool (*dma_filter)(struct dma_chan *chan, void *filter_param);
+ void *dma_rx_param;
+ void *dma_tx_param;
+};
#endif
#endif
--
1.6.3.3
--
To unsubscribe from this list: send the line "unsubscribe linux-mmc" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
