2 * Copyright (c) 2006 ARM Ltd.
3 * Copyright (c) 2010 ST-Ericsson SA
5 * Author: Peter Pearse <peter.pearse@arm.com>
6 * Author: Linus Walleij <linus.walleij@stericsson.com>
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
18 * You should have received a copy of the GNU General Public License along with
19 * this program; if not, write to the Free Software Foundation, Inc., 59
20 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
22 * The full GNU General Public License is in this distribution in the file
25 * Documentation: ARM DDI 0196G == PL080
26 * Documentation: ARM DDI 0218E == PL081
28 * PL080 & PL081 both have 16 sets of DMA signals that can be routed to any
31 * The PL080 has 8 channels available for simultaneous use, and the PL081
32 * has only two channels. So on these DMA controllers the number of channels
33 * and the number of incoming DMA signals are two totally different things.
34 * It is usually not possible to theoretically handle all physical signals,
35 * so a multiplexing scheme with possible denial of use is necessary.
37 * The PL080 has a dual bus master, PL081 has a single master.
39 * Memory to peripheral transfer may be visualized as
40 * Get data from memory to DMAC
42 * On burst request from peripheral
43 * Destination burst from DMAC to peripheral
45 * Raise terminal count interrupt
47 * For peripherals with a FIFO:
48 * Source burst size == half the depth of the peripheral FIFO
49 * Destination burst size == the depth of the peripheral FIFO
51 * (Bursts are irrelevant for mem to mem transfers - there are no burst
52 * signals, the DMA controller will simply facilitate its AHB master.)
54 * ASSUMES default (little) endianness for DMA transfers
56 * The PL08x has two flow control settings:
57 * - DMAC flow control: the transfer size defines the number of transfers
58 * which occur for the current LLI entry, and the DMAC raises TC at the
59 * end of every LLI entry. Observed behaviour shows the DMAC listening
60 * to both the BREQ and SREQ signals (contrary to documented),
61 * transferring data if either is active. The LBREQ and LSREQ signals
64 * - Peripheral flow control: the transfer size is ignored (and should be
65 * zero). The data is transferred from the current LLI entry, until
66 * after the final transfer signalled by LBREQ or LSREQ. The DMAC
67 * will then move to the next LLI entry.
70 * - Break out common code from arch/arm/mach-s3c64xx and share
72 #include <linux/amba/bus.h>
73 #include <linux/amba/pl08x.h>
74 #include <linux/debugfs.h>
75 #include <linux/delay.h>
76 #include <linux/device.h>
77 #include <linux/dmaengine.h>
78 #include <linux/dmapool.h>
79 #include <linux/dma-mapping.h>
80 #include <linux/init.h>
81 #include <linux/interrupt.h>
82 #include <linux/module.h>
83 #include <linux/pm_runtime.h>
84 #include <linux/seq_file.h>
85 #include <linux/slab.h>
86 #include <asm/hardware/pl080.h>
88 #include "dmaengine.h"
90 #define DRIVER_NAME "pl08xdmac"
92 static struct amba_driver pl08x_amba_driver;
93 struct pl08x_driver_data;
96 * struct vendor_data - vendor-specific config parameters for PL08x derivatives
97 * @channels: the number of channels available in this variant
98 * @dualmaster: whether this version supports dual AHB masters or not.
99 * @nomadik: whether the channels have Nomadik security extension bits
100 * that need to be checked for permission before use and some registers are
110 * PL08X private data structures
111 * An LLI struct - see PL08x TRM. Note that next uses bit[0] as a bus bit,
112 * start & end do not - their bus bit info is in cctl. Also note that these
113 * are fixed 32-bit quantities.
123 * struct pl08x_bus_data - information of source or destination
124 * busses for a transfer
125 * @addr: current address
126 * @maxwidth: the maximum width of a transfer on this bus
127 * @buswidth: the width of this bus in bytes: 1, 2 or 4
129 struct pl08x_bus_data {
136 * struct pl08x_phy_chan - holder for the physical channels
137 * @id: physical index to this channel
138 * @lock: a lock to use when altering an instance of this struct
139 * @signal: the physical signal (aka channel) serving this physical channel
141 * @serving: the virtual channel currently being served by this physical
144 struct pl08x_phy_chan {
149 struct pl08x_dma_chan *serving;
153 * struct pl08x_sg - structure containing data per sg
154 * @src_addr: src address of sg
155 * @dst_addr: dst address of sg
156 * @len: transfer len in bytes
157 * @node: node for txd's dsg_list
163 struct list_head node;
167 * struct pl08x_txd - wrapper for struct dma_async_tx_descriptor
168 * @tx: async tx descriptor
169 * @node: node for txd list for channels
170 * @dsg_list: list of children sg's
171 * @direction: direction of transfer
172 * @llis_bus: DMA memory address (physical) start for the LLIs
173 * @llis_va: virtual memory address start for the LLIs
174 * @cctl: control reg values for current txd
175 * @ccfg: config reg values for current txd
178 struct dma_async_tx_descriptor tx;
179 struct list_head node;
180 struct list_head dsg_list;
181 enum dma_transfer_direction direction;
183 struct pl08x_lli *llis_va;
184 /* Default cctl value for LLIs */
187 * Settings to be put into the physical channel when we
188 * trigger this txd. Other registers are in llis_va[0].
194 * struct pl08x_dma_chan_state - holds the PL08x specific virtual channel
196 * @PL08X_CHAN_IDLE: the channel is idle
197 * @PL08X_CHAN_RUNNING: the channel has allocated a physical transport
198 * channel and is running a transfer on it
199 * @PL08X_CHAN_PAUSED: the channel has allocated a physical transport
200 * channel, but the transfer is currently paused
201 * @PL08X_CHAN_WAITING: the channel is waiting for a physical transport
202 * channel to become available (only pertains to memcpy channels)
204 enum pl08x_dma_chan_state {
212 * struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel
213 * @chan: wrappped abstract channel
214 * @phychan: the physical channel utilized by this channel, if there is one
215 * @phychan_hold: if non-zero, hold on to the physical channel even if we
216 * have no pending entries
217 * @tasklet: tasklet scheduled by the IRQ to handle actual work etc
218 * @name: name of channel
219 * @cd: channel platform data
220 * @runtime_addr: address for RX/TX according to the runtime config
221 * @pend_list: queued transactions pending on this channel
222 * @at: active transaction on this channel
223 * @lock: a lock for this channel data
224 * @host: a pointer to the host (internal use)
225 * @state: whether the channel is idle, paused, running etc
226 * @slave: whether this channel is a device (slave) or for memcpy
227 * @waiting: a TX descriptor on this channel which is waiting for a physical
228 * channel to become available
230 struct pl08x_dma_chan {
231 struct dma_chan chan;
232 struct pl08x_phy_chan *phychan;
234 struct tasklet_struct tasklet;
236 const struct pl08x_channel_data *cd;
237 struct dma_slave_config cfg;
240 struct list_head pend_list;
241 struct pl08x_txd *at;
243 struct pl08x_driver_data *host;
244 enum pl08x_dma_chan_state state;
246 struct pl08x_txd *waiting;
250 * struct pl08x_driver_data - the local state holder for the PL08x
251 * @slave: slave engine for this instance
252 * @memcpy: memcpy engine for this instance
253 * @base: virtual memory base (remapped) for the PL08x
254 * @adev: the corresponding AMBA (PrimeCell) bus entry
255 * @vd: vendor data for this PL08x variant
256 * @pd: platform data passed in from the platform/machine
257 * @phy_chans: array of data for the physical channels
258 * @pool: a pool for the LLI descriptors
259 * @pool_ctr: counter of LLIs in the pool
260 * @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI
262 * @mem_buses: set to indicate memory transfers on AHB2.
263 * @lock: a spinlock for this struct
265 struct pl08x_driver_data {
266 struct dma_device slave;
267 struct dma_device memcpy;
269 struct amba_device *adev;
270 const struct vendor_data *vd;
271 struct pl08x_platform_data *pd;
272 struct pl08x_phy_chan *phy_chans;
273 struct dma_pool *pool;
280 * PL08X specific defines
283 /* Size (bytes) of each LLI buffer allocated for one transfer */
284 # define PL08X_LLI_TSFR_SIZE 0x2000
286 /* Maximum times we call dma_pool_alloc on this pool without freeing */
287 #define MAX_NUM_TSFR_LLIS (PL08X_LLI_TSFR_SIZE/sizeof(struct pl08x_lli))
288 #define PL08X_ALIGN 8
290 static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan)
292 return container_of(chan, struct pl08x_dma_chan, chan);
295 static inline struct pl08x_txd *to_pl08x_txd(struct dma_async_tx_descriptor *tx)
297 return container_of(tx, struct pl08x_txd, tx);
301 * Physical channel handling
304 /* Whether a certain channel is busy or not */
305 static int pl08x_phy_channel_busy(struct pl08x_phy_chan *ch)
309 val = readl(ch->base + PL080_CH_CONFIG);
310 return val & PL080_CONFIG_ACTIVE;
314 * Set the initial DMA register values i.e. those for the first LLI
315 * The next LLI pointer and the configuration interrupt bit have
316 * been set when the LLIs were constructed. Poke them into the hardware
317 * and start the transfer.
319 static void pl08x_start_txd(struct pl08x_dma_chan *plchan,
320 struct pl08x_txd *txd)
322 struct pl08x_driver_data *pl08x = plchan->host;
323 struct pl08x_phy_chan *phychan = plchan->phychan;
324 struct pl08x_lli *lli = &txd->llis_va[0];
329 /* Wait for channel inactive */
330 while (pl08x_phy_channel_busy(phychan))
333 dev_vdbg(&pl08x->adev->dev,
334 "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "
335 "clli=0x%08x, cctl=0x%08x, ccfg=0x%08x\n",
336 phychan->id, lli->src, lli->dst, lli->lli, lli->cctl,
339 writel(lli->src, phychan->base + PL080_CH_SRC_ADDR);
340 writel(lli->dst, phychan->base + PL080_CH_DST_ADDR);
341 writel(lli->lli, phychan->base + PL080_CH_LLI);
342 writel(lli->cctl, phychan->base + PL080_CH_CONTROL);
343 writel(txd->ccfg, phychan->base + PL080_CH_CONFIG);
345 /* Enable the DMA channel */
346 /* Do not access config register until channel shows as disabled */
347 while (readl(pl08x->base + PL080_EN_CHAN) & (1 << phychan->id))
350 /* Do not access config register until channel shows as inactive */
351 val = readl(phychan->base + PL080_CH_CONFIG);
352 while ((val & PL080_CONFIG_ACTIVE) || (val & PL080_CONFIG_ENABLE))
353 val = readl(phychan->base + PL080_CH_CONFIG);
355 writel(val | PL080_CONFIG_ENABLE, phychan->base + PL080_CH_CONFIG);
359 * Pause the channel by setting the HALT bit.
361 * For M->P transfers, pause the DMAC first and then stop the peripheral -
362 * the FIFO can only drain if the peripheral is still requesting data.
363 * (note: this can still timeout if the DMAC FIFO never drains of data.)
365 * For P->M transfers, disable the peripheral first to stop it filling
366 * the DMAC FIFO, and then pause the DMAC.
368 static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch)
373 /* Set the HALT bit and wait for the FIFO to drain */
374 val = readl(ch->base + PL080_CH_CONFIG);
375 val |= PL080_CONFIG_HALT;
376 writel(val, ch->base + PL080_CH_CONFIG);
378 /* Wait for channel inactive */
379 for (timeout = 1000; timeout; timeout--) {
380 if (!pl08x_phy_channel_busy(ch))
384 if (pl08x_phy_channel_busy(ch))
385 pr_err("pl08x: channel%u timeout waiting for pause\n", ch->id);
388 static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch)
392 /* Clear the HALT bit */
393 val = readl(ch->base + PL080_CH_CONFIG);
394 val &= ~PL080_CONFIG_HALT;
395 writel(val, ch->base + PL080_CH_CONFIG);
399 * pl08x_terminate_phy_chan() stops the channel, clears the FIFO and
400 * clears any pending interrupt status. This should not be used for
401 * an on-going transfer, but as a method of shutting down a channel
402 * (eg, when it's no longer used) or terminating a transfer.
404 static void pl08x_terminate_phy_chan(struct pl08x_driver_data *pl08x,
405 struct pl08x_phy_chan *ch)
407 u32 val = readl(ch->base + PL080_CH_CONFIG);
409 val &= ~(PL080_CONFIG_ENABLE | PL080_CONFIG_ERR_IRQ_MASK |
410 PL080_CONFIG_TC_IRQ_MASK);
412 writel(val, ch->base + PL080_CH_CONFIG);
414 writel(1 << ch->id, pl08x->base + PL080_ERR_CLEAR);
415 writel(1 << ch->id, pl08x->base + PL080_TC_CLEAR);
418 static inline u32 get_bytes_in_cctl(u32 cctl)
420 /* The source width defines the number of bytes */
421 u32 bytes = cctl & PL080_CONTROL_TRANSFER_SIZE_MASK;
423 switch (cctl >> PL080_CONTROL_SWIDTH_SHIFT) {
424 case PL080_WIDTH_8BIT:
426 case PL080_WIDTH_16BIT:
429 case PL080_WIDTH_32BIT:
436 /* The channel should be paused when calling this */
437 static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan)
439 struct pl08x_phy_chan *ch;
440 struct pl08x_txd *txd;
444 spin_lock_irqsave(&plchan->lock, flags);
445 ch = plchan->phychan;
449 * Follow the LLIs to get the number of remaining
450 * bytes in the currently active transaction.
453 u32 clli = readl(ch->base + PL080_CH_LLI) & ~PL080_LLI_LM_AHB2;
455 /* First get the remaining bytes in the active transfer */
456 bytes = get_bytes_in_cctl(readl(ch->base + PL080_CH_CONTROL));
459 struct pl08x_lli *llis_va = txd->llis_va;
460 dma_addr_t llis_bus = txd->llis_bus;
463 BUG_ON(clli < llis_bus || clli >= llis_bus +
464 sizeof(struct pl08x_lli) * MAX_NUM_TSFR_LLIS);
467 * Locate the next LLI - as this is an array,
468 * it's simple maths to find.
470 index = (clli - llis_bus) / sizeof(struct pl08x_lli);
472 for (; index < MAX_NUM_TSFR_LLIS; index++) {
473 bytes += get_bytes_in_cctl(llis_va[index].cctl);
476 * A LLI pointer of 0 terminates the LLI list
478 if (!llis_va[index].lli)
484 /* Sum up all queued transactions */
485 if (!list_empty(&plchan->pend_list)) {
486 struct pl08x_txd *txdi;
487 list_for_each_entry(txdi, &plchan->pend_list, node) {
488 struct pl08x_sg *dsg;
489 list_for_each_entry(dsg, &txd->dsg_list, node)
494 spin_unlock_irqrestore(&plchan->lock, flags);
500 * Allocate a physical channel for a virtual channel
502 * Try to locate a physical channel to be used for this transfer. If all
503 * are taken return NULL and the requester will have to cope by using
504 * some fallback PIO mode or retrying later.
506 static struct pl08x_phy_chan *
507 pl08x_get_phy_channel(struct pl08x_driver_data *pl08x,
508 struct pl08x_dma_chan *virt_chan)
510 struct pl08x_phy_chan *ch = NULL;
514 for (i = 0; i < pl08x->vd->channels; i++) {
515 ch = &pl08x->phy_chans[i];
517 spin_lock_irqsave(&ch->lock, flags);
519 if (!ch->locked && !ch->serving) {
520 ch->serving = virt_chan;
522 spin_unlock_irqrestore(&ch->lock, flags);
526 spin_unlock_irqrestore(&ch->lock, flags);
529 if (i == pl08x->vd->channels) {
530 /* No physical channel available, cope with it */
537 static inline void pl08x_put_phy_channel(struct pl08x_driver_data *pl08x,
538 struct pl08x_phy_chan *ch)
542 spin_lock_irqsave(&ch->lock, flags);
544 /* Stop the channel and clear its interrupts */
545 pl08x_terminate_phy_chan(pl08x, ch);
547 /* Mark it as free */
549 spin_unlock_irqrestore(&ch->lock, flags);
556 static inline unsigned int pl08x_get_bytes_for_cctl(unsigned int coded)
559 case PL080_WIDTH_8BIT:
561 case PL080_WIDTH_16BIT:
563 case PL080_WIDTH_32BIT:
572 static inline u32 pl08x_cctl_bits(u32 cctl, u8 srcwidth, u8 dstwidth,
577 /* Remove all src, dst and transfer size bits */
578 retbits &= ~PL080_CONTROL_DWIDTH_MASK;
579 retbits &= ~PL080_CONTROL_SWIDTH_MASK;
580 retbits &= ~PL080_CONTROL_TRANSFER_SIZE_MASK;
582 /* Then set the bits according to the parameters */
585 retbits |= PL080_WIDTH_8BIT << PL080_CONTROL_SWIDTH_SHIFT;
588 retbits |= PL080_WIDTH_16BIT << PL080_CONTROL_SWIDTH_SHIFT;
591 retbits |= PL080_WIDTH_32BIT << PL080_CONTROL_SWIDTH_SHIFT;
600 retbits |= PL080_WIDTH_8BIT << PL080_CONTROL_DWIDTH_SHIFT;
603 retbits |= PL080_WIDTH_16BIT << PL080_CONTROL_DWIDTH_SHIFT;
606 retbits |= PL080_WIDTH_32BIT << PL080_CONTROL_DWIDTH_SHIFT;
613 retbits |= tsize << PL080_CONTROL_TRANSFER_SIZE_SHIFT;
617 struct pl08x_lli_build_data {
618 struct pl08x_txd *txd;
619 struct pl08x_bus_data srcbus;
620 struct pl08x_bus_data dstbus;
626 * Autoselect a master bus to use for the transfer. Slave will be the chosen as
627 * victim in case src & dest are not similarly aligned. i.e. If after aligning
628 * masters address with width requirements of transfer (by sending few byte by
629 * byte data), slave is still not aligned, then its width will be reduced to
631 * - prefers the destination bus if both available
632 * - prefers bus with fixed address (i.e. peripheral)
634 static void pl08x_choose_master_bus(struct pl08x_lli_build_data *bd,
635 struct pl08x_bus_data **mbus, struct pl08x_bus_data **sbus, u32 cctl)
637 if (!(cctl & PL080_CONTROL_DST_INCR)) {
640 } else if (!(cctl & PL080_CONTROL_SRC_INCR)) {
644 if (bd->dstbus.buswidth >= bd->srcbus.buswidth) {
655 * Fills in one LLI for a certain transfer descriptor and advance the counter
657 static void pl08x_fill_lli_for_desc(struct pl08x_lli_build_data *bd,
658 int num_llis, int len, u32 cctl)
660 struct pl08x_lli *llis_va = bd->txd->llis_va;
661 dma_addr_t llis_bus = bd->txd->llis_bus;
663 BUG_ON(num_llis >= MAX_NUM_TSFR_LLIS);
665 llis_va[num_llis].cctl = cctl;
666 llis_va[num_llis].src = bd->srcbus.addr;
667 llis_va[num_llis].dst = bd->dstbus.addr;
668 llis_va[num_llis].lli = llis_bus + (num_llis + 1) *
669 sizeof(struct pl08x_lli);
670 llis_va[num_llis].lli |= bd->lli_bus;
672 if (cctl & PL080_CONTROL_SRC_INCR)
673 bd->srcbus.addr += len;
674 if (cctl & PL080_CONTROL_DST_INCR)
675 bd->dstbus.addr += len;
677 BUG_ON(bd->remainder < len);
679 bd->remainder -= len;
682 static inline void prep_byte_width_lli(struct pl08x_lli_build_data *bd,
683 u32 *cctl, u32 len, int num_llis, size_t *total_bytes)
685 *cctl = pl08x_cctl_bits(*cctl, 1, 1, len);
686 pl08x_fill_lli_for_desc(bd, num_llis, len, *cctl);
687 (*total_bytes) += len;
691 * This fills in the table of LLIs for the transfer descriptor
692 * Note that we assume we never have to change the burst sizes
695 static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
696 struct pl08x_txd *txd)
698 struct pl08x_bus_data *mbus, *sbus;
699 struct pl08x_lli_build_data bd;
701 u32 cctl, early_bytes = 0;
702 size_t max_bytes_per_lli, total_bytes;
703 struct pl08x_lli *llis_va;
704 struct pl08x_sg *dsg;
706 txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, &txd->llis_bus);
708 dev_err(&pl08x->adev->dev, "%s no memory for llis\n", __func__);
715 bd.lli_bus = (pl08x->lli_buses & PL08X_AHB2) ? PL080_LLI_LM_AHB2 : 0;
718 /* Find maximum width of the source bus */
720 pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_SWIDTH_MASK) >>
721 PL080_CONTROL_SWIDTH_SHIFT);
723 /* Find maximum width of the destination bus */
725 pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_DWIDTH_MASK) >>
726 PL080_CONTROL_DWIDTH_SHIFT);
728 list_for_each_entry(dsg, &txd->dsg_list, node) {
732 bd.srcbus.addr = dsg->src_addr;
733 bd.dstbus.addr = dsg->dst_addr;
734 bd.remainder = dsg->len;
735 bd.srcbus.buswidth = bd.srcbus.maxwidth;
736 bd.dstbus.buswidth = bd.dstbus.maxwidth;
738 pl08x_choose_master_bus(&bd, &mbus, &sbus, cctl);
740 dev_vdbg(&pl08x->adev->dev, "src=0x%08x%s/%u dst=0x%08x%s/%u len=%zu\n",
741 bd.srcbus.addr, cctl & PL080_CONTROL_SRC_INCR ? "+" : "",
743 bd.dstbus.addr, cctl & PL080_CONTROL_DST_INCR ? "+" : "",
746 dev_vdbg(&pl08x->adev->dev, "mbus=%s sbus=%s\n",
747 mbus == &bd.srcbus ? "src" : "dst",
748 sbus == &bd.srcbus ? "src" : "dst");
751 * Zero length is only allowed if all these requirements are
753 * - flow controller is peripheral.
754 * - src.addr is aligned to src.width
755 * - dst.addr is aligned to dst.width
757 * sg_len == 1 should be true, as there can be two cases here:
759 * - Memory addresses are contiguous and are not scattered.
760 * Here, Only one sg will be passed by user driver, with
761 * memory address and zero length. We pass this to controller
762 * and after the transfer it will receive the last burst
763 * request from peripheral and so transfer finishes.
765 * - Memory addresses are scattered and are not contiguous.
766 * Here, Obviously as DMA controller doesn't know when a lli's
767 * transfer gets over, it can't load next lli. So in this
768 * case, there has to be an assumption that only one lli is
769 * supported. Thus, we can't have scattered addresses.
772 u32 fc = (txd->ccfg & PL080_CONFIG_FLOW_CONTROL_MASK) >>
773 PL080_CONFIG_FLOW_CONTROL_SHIFT;
774 if (!((fc >= PL080_FLOW_SRC2DST_DST) &&
775 (fc <= PL080_FLOW_SRC2DST_SRC))) {
776 dev_err(&pl08x->adev->dev, "%s sg len can't be zero",
781 if ((bd.srcbus.addr % bd.srcbus.buswidth) ||
782 (bd.dstbus.addr % bd.dstbus.buswidth)) {
783 dev_err(&pl08x->adev->dev,
784 "%s src & dst address must be aligned to src"
785 " & dst width if peripheral is flow controller",
790 cctl = pl08x_cctl_bits(cctl, bd.srcbus.buswidth,
791 bd.dstbus.buswidth, 0);
792 pl08x_fill_lli_for_desc(&bd, num_llis++, 0, cctl);
797 * Send byte by byte for following cases
798 * - Less than a bus width available
799 * - until master bus is aligned
801 if (bd.remainder < mbus->buswidth)
802 early_bytes = bd.remainder;
803 else if ((mbus->addr) % (mbus->buswidth)) {
804 early_bytes = mbus->buswidth - (mbus->addr) %
806 if ((bd.remainder - early_bytes) < mbus->buswidth)
807 early_bytes = bd.remainder;
811 dev_vdbg(&pl08x->adev->dev,
812 "%s byte width LLIs (remain 0x%08x)\n",
813 __func__, bd.remainder);
814 prep_byte_width_lli(&bd, &cctl, early_bytes, num_llis++,
821 * - if slave is not then we must set its width down
823 if (sbus->addr % sbus->buswidth) {
824 dev_dbg(&pl08x->adev->dev,
825 "%s set down bus width to one byte\n",
832 * Bytes transferred = tsize * src width, not
835 max_bytes_per_lli = bd.srcbus.buswidth *
836 PL080_CONTROL_TRANSFER_SIZE_MASK;
837 dev_vdbg(&pl08x->adev->dev,
838 "%s max bytes per lli = %zu\n",
839 __func__, max_bytes_per_lli);
842 * Make largest possible LLIs until less than one bus
845 while (bd.remainder > (mbus->buswidth - 1)) {
846 size_t lli_len, tsize, width;
849 * If enough left try to send max possible,
850 * otherwise try to send the remainder
852 lli_len = min(bd.remainder, max_bytes_per_lli);
855 * Check against maximum bus alignment:
856 * Calculate actual transfer size in relation to
857 * bus width an get a maximum remainder of the
858 * highest bus width - 1
860 width = max(mbus->buswidth, sbus->buswidth);
861 lli_len = (lli_len / width) * width;
862 tsize = lli_len / bd.srcbus.buswidth;
864 dev_vdbg(&pl08x->adev->dev,
865 "%s fill lli with single lli chunk of "
866 "size 0x%08zx (remainder 0x%08zx)\n",
867 __func__, lli_len, bd.remainder);
869 cctl = pl08x_cctl_bits(cctl, bd.srcbus.buswidth,
870 bd.dstbus.buswidth, tsize);
871 pl08x_fill_lli_for_desc(&bd, num_llis++,
873 total_bytes += lli_len;
880 dev_vdbg(&pl08x->adev->dev,
881 "%s align with boundary, send odd bytes (remain %zu)\n",
882 __func__, bd.remainder);
883 prep_byte_width_lli(&bd, &cctl, bd.remainder,
884 num_llis++, &total_bytes);
888 if (total_bytes != dsg->len) {
889 dev_err(&pl08x->adev->dev,
890 "%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n",
891 __func__, total_bytes, dsg->len);
895 if (num_llis >= MAX_NUM_TSFR_LLIS) {
896 dev_err(&pl08x->adev->dev,
897 "%s need to increase MAX_NUM_TSFR_LLIS from 0x%08x\n",
898 __func__, (u32) MAX_NUM_TSFR_LLIS);
903 llis_va = txd->llis_va;
904 /* The final LLI terminates the LLI. */
905 llis_va[num_llis - 1].lli = 0;
906 /* The final LLI element shall also fire an interrupt. */
907 llis_va[num_llis - 1].cctl |= PL080_CONTROL_TC_IRQ_EN;
913 dev_vdbg(&pl08x->adev->dev,
914 "%-3s %-9s %-10s %-10s %-10s %s\n",
915 "lli", "", "csrc", "cdst", "clli", "cctl");
916 for (i = 0; i < num_llis; i++) {
917 dev_vdbg(&pl08x->adev->dev,
918 "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x\n",
919 i, &llis_va[i], llis_va[i].src,
920 llis_va[i].dst, llis_va[i].lli, llis_va[i].cctl
929 /* You should call this with the struct pl08x lock held */
930 static void pl08x_free_txd(struct pl08x_driver_data *pl08x,
931 struct pl08x_txd *txd)
933 struct pl08x_sg *dsg, *_dsg;
937 dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus);
941 list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) {
942 list_del(&dsg->node);
949 static void pl08x_free_txd_list(struct pl08x_driver_data *pl08x,
950 struct pl08x_dma_chan *plchan)
952 struct pl08x_txd *txdi = NULL;
953 struct pl08x_txd *next;
955 if (!list_empty(&plchan->pend_list)) {
956 list_for_each_entry_safe(txdi,
957 next, &plchan->pend_list, node) {
958 list_del(&txdi->node);
959 pl08x_free_txd(pl08x, txdi);
967 static int pl08x_alloc_chan_resources(struct dma_chan *chan)
972 static void pl08x_free_chan_resources(struct dma_chan *chan)
977 * This should be called with the channel plchan->lock held
979 static int prep_phy_channel(struct pl08x_dma_chan *plchan,
980 struct pl08x_txd *txd)
982 struct pl08x_driver_data *pl08x = plchan->host;
983 struct pl08x_phy_chan *ch;
986 /* Check if we already have a channel */
987 if (plchan->phychan) {
988 ch = plchan->phychan;
992 ch = pl08x_get_phy_channel(pl08x, plchan);
994 /* No physical channel available, cope with it */
995 dev_dbg(&pl08x->adev->dev, "no physical channel available for xfer on %s\n", plchan->name);
1000 * OK we have a physical channel: for memcpy() this is all we
1001 * need, but for slaves the physical signals may be muxed!
1002 * Can the platform allow us to use this channel?
1004 if (plchan->slave && pl08x->pd->get_signal) {
1005 ret = pl08x->pd->get_signal(plchan->cd);
1007 dev_dbg(&pl08x->adev->dev,
1008 "unable to use physical channel %d for transfer on %s due to platform restrictions\n",
1009 ch->id, plchan->name);
1010 /* Release physical channel & return */
1011 pl08x_put_phy_channel(pl08x, ch);
1017 plchan->phychan = ch;
1018 dev_dbg(&pl08x->adev->dev, "allocated physical channel %d and signal %d for xfer on %s\n",
1024 /* Assign the flow control signal to this channel */
1025 if (txd->direction == DMA_MEM_TO_DEV)
1026 txd->ccfg |= ch->signal << PL080_CONFIG_DST_SEL_SHIFT;
1027 else if (txd->direction == DMA_DEV_TO_MEM)
1028 txd->ccfg |= ch->signal << PL080_CONFIG_SRC_SEL_SHIFT;
1030 plchan->phychan_hold++;
1035 static void release_phy_channel(struct pl08x_dma_chan *plchan)
1037 struct pl08x_driver_data *pl08x = plchan->host;
1039 if ((plchan->phychan->signal >= 0) && pl08x->pd->put_signal) {
1040 pl08x->pd->put_signal(plchan->cd, plchan->phychan->signal);
1041 plchan->phychan->signal = -1;
1043 pl08x_put_phy_channel(pl08x, plchan->phychan);
1044 plchan->phychan = NULL;
1047 static dma_cookie_t pl08x_tx_submit(struct dma_async_tx_descriptor *tx)
1049 struct pl08x_dma_chan *plchan = to_pl08x_chan(tx->chan);
1050 struct pl08x_txd *txd = to_pl08x_txd(tx);
1051 unsigned long flags;
1052 dma_cookie_t cookie;
1054 spin_lock_irqsave(&plchan->lock, flags);
1055 cookie = dma_cookie_assign(tx);
1057 /* Put this onto the pending list */
1058 list_add_tail(&txd->node, &plchan->pend_list);
1061 * If there was no physical channel available for this memcpy,
1062 * stack the request up and indicate that the channel is waiting
1063 * for a free physical channel.
1065 if (!plchan->slave && !plchan->phychan) {
1066 /* Do this memcpy whenever there is a channel ready */
1067 plchan->state = PL08X_CHAN_WAITING;
1068 plchan->waiting = txd;
1070 plchan->phychan_hold--;
1073 spin_unlock_irqrestore(&plchan->lock, flags);
1078 static struct dma_async_tx_descriptor *pl08x_prep_dma_interrupt(
1079 struct dma_chan *chan, unsigned long flags)
1081 struct dma_async_tx_descriptor *retval = NULL;
1087 * Code accessing dma_async_is_complete() in a tight loop may give problems.
1088 * If slaves are relying on interrupts to signal completion this function
1089 * must not be called with interrupts disabled.
1091 static enum dma_status pl08x_dma_tx_status(struct dma_chan *chan,
1092 dma_cookie_t cookie, struct dma_tx_state *txstate)
1094 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1095 enum dma_status ret;
1097 ret = dma_cookie_status(chan, cookie, txstate);
1098 if (ret == DMA_SUCCESS)
1102 * This cookie not complete yet
1103 * Get number of bytes left in the active transactions and queue
1105 dma_set_residue(txstate, pl08x_getbytes_chan(plchan));
1107 if (plchan->state == PL08X_CHAN_PAUSED)
1110 /* Whether waiting or running, we're in progress */
1111 return DMA_IN_PROGRESS;
1114 /* PrimeCell DMA extension */
1115 struct burst_table {
1120 static const struct burst_table burst_sizes[] = {
1123 .reg = PL080_BSIZE_256,
1127 .reg = PL080_BSIZE_128,
1131 .reg = PL080_BSIZE_64,
1135 .reg = PL080_BSIZE_32,
1139 .reg = PL080_BSIZE_16,
1143 .reg = PL080_BSIZE_8,
1147 .reg = PL080_BSIZE_4,
1151 .reg = PL080_BSIZE_1,
1156 * Given the source and destination available bus masks, select which
1157 * will be routed to each port. We try to have source and destination
1158 * on separate ports, but always respect the allowable settings.
1160 static u32 pl08x_select_bus(u8 src, u8 dst)
1164 if (!(dst & PL08X_AHB1) || ((dst & PL08X_AHB2) && (src & PL08X_AHB1)))
1165 cctl |= PL080_CONTROL_DST_AHB2;
1166 if (!(src & PL08X_AHB1) || ((src & PL08X_AHB2) && !(dst & PL08X_AHB2)))
1167 cctl |= PL080_CONTROL_SRC_AHB2;
1172 static u32 pl08x_cctl(u32 cctl)
1174 cctl &= ~(PL080_CONTROL_SRC_AHB2 | PL080_CONTROL_DST_AHB2 |
1175 PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR |
1176 PL080_CONTROL_PROT_MASK);
1178 /* Access the cell in privileged mode, non-bufferable, non-cacheable */
1179 return cctl | PL080_CONTROL_PROT_SYS;
1182 static u32 pl08x_width(enum dma_slave_buswidth width)
1185 case DMA_SLAVE_BUSWIDTH_1_BYTE:
1186 return PL080_WIDTH_8BIT;
1187 case DMA_SLAVE_BUSWIDTH_2_BYTES:
1188 return PL080_WIDTH_16BIT;
1189 case DMA_SLAVE_BUSWIDTH_4_BYTES:
1190 return PL080_WIDTH_32BIT;
1196 static u32 pl08x_burst(u32 maxburst)
1200 for (i = 0; i < ARRAY_SIZE(burst_sizes); i++)
1201 if (burst_sizes[i].burstwords <= maxburst)
1204 return burst_sizes[i].reg;
1207 static u32 pl08x_get_cctl(struct pl08x_dma_chan *plchan,
1208 enum dma_slave_buswidth addr_width, u32 maxburst)
1210 u32 width, burst, cctl = 0;
1212 width = pl08x_width(addr_width);
1216 cctl |= width << PL080_CONTROL_SWIDTH_SHIFT;
1217 cctl |= width << PL080_CONTROL_DWIDTH_SHIFT;
1220 * If this channel will only request single transfers, set this
1221 * down to ONE element. Also select one element if no maxburst
1224 if (plchan->cd->single)
1227 burst = pl08x_burst(maxburst);
1228 cctl |= burst << PL080_CONTROL_SB_SIZE_SHIFT;
1229 cctl |= burst << PL080_CONTROL_DB_SIZE_SHIFT;
1231 return pl08x_cctl(cctl);
1234 static int dma_set_runtime_config(struct dma_chan *chan,
1235 struct dma_slave_config *config)
1237 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1238 struct pl08x_driver_data *pl08x = plchan->host;
1239 u32 src_cctl, dst_cctl;
1244 dst_cctl = pl08x_get_cctl(plchan, config->dst_addr_width,
1245 config->dst_maxburst);
1246 if (dst_cctl == ~0 && config->direction == DMA_MEM_TO_DEV) {
1247 dev_err(&pl08x->adev->dev,
1248 "bad runtime_config: alien address width (M2D)\n");
1252 src_cctl = pl08x_get_cctl(plchan, config->src_addr_width,
1253 config->src_maxburst);
1254 if (src_cctl == ~0 && config->direction == DMA_DEV_TO_MEM) {
1255 dev_err(&pl08x->adev->dev,
1256 "bad runtime_config: alien address width (D2M)\n");
1260 plchan->dst_cctl = dst_cctl;
1261 plchan->src_cctl = src_cctl;
1262 plchan->cfg = *config;
1268 * Slave transactions callback to the slave device to allow
1269 * synchronization of slave DMA signals with the DMAC enable
1271 static void pl08x_issue_pending(struct dma_chan *chan)
1273 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1274 unsigned long flags;
1276 spin_lock_irqsave(&plchan->lock, flags);
1277 /* Something is already active, or we're waiting for a channel... */
1278 if (plchan->at || plchan->state == PL08X_CHAN_WAITING) {
1279 spin_unlock_irqrestore(&plchan->lock, flags);
1283 /* Take the first element in the queue and execute it */
1284 if (!list_empty(&plchan->pend_list)) {
1285 struct pl08x_txd *next;
1287 next = list_first_entry(&plchan->pend_list,
1290 list_del(&next->node);
1291 plchan->state = PL08X_CHAN_RUNNING;
1293 pl08x_start_txd(plchan, next);
1296 spin_unlock_irqrestore(&plchan->lock, flags);
1299 static int pl08x_prep_channel_resources(struct pl08x_dma_chan *plchan,
1300 struct pl08x_txd *txd)
1302 struct pl08x_driver_data *pl08x = plchan->host;
1303 unsigned long flags;
1306 num_llis = pl08x_fill_llis_for_desc(pl08x, txd);
1308 spin_lock_irqsave(&plchan->lock, flags);
1309 pl08x_free_txd(pl08x, txd);
1310 spin_unlock_irqrestore(&plchan->lock, flags);
1314 spin_lock_irqsave(&plchan->lock, flags);
1317 * See if we already have a physical channel allocated,
1318 * else this is the time to try to get one.
1320 ret = prep_phy_channel(plchan, txd);
1323 * No physical channel was available.
1325 * memcpy transfers can be sorted out at submission time.
1327 * Slave transfers may have been denied due to platform
1328 * channel muxing restrictions. Since there is no guarantee
1329 * that this will ever be resolved, and the signal must be
1330 * acquired AFTER acquiring the physical channel, we will let
1331 * them be NACK:ed with -EBUSY here. The drivers can retry
1332 * the prep() call if they are eager on doing this using DMA.
1334 if (plchan->slave) {
1335 pl08x_free_txd_list(pl08x, plchan);
1336 pl08x_free_txd(pl08x, txd);
1337 spin_unlock_irqrestore(&plchan->lock, flags);
1342 * Else we're all set, paused and ready to roll, status
1343 * will switch to PL08X_CHAN_RUNNING when we call
1344 * issue_pending(). If there is something running on the
1345 * channel already we don't change its state.
1347 if (plchan->state == PL08X_CHAN_IDLE)
1348 plchan->state = PL08X_CHAN_PAUSED;
1350 spin_unlock_irqrestore(&plchan->lock, flags);
1355 static struct pl08x_txd *pl08x_get_txd(struct pl08x_dma_chan *plchan,
1356 unsigned long flags)
1358 struct pl08x_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
1361 dma_async_tx_descriptor_init(&txd->tx, &plchan->chan);
1362 txd->tx.flags = flags;
1363 txd->tx.tx_submit = pl08x_tx_submit;
1364 INIT_LIST_HEAD(&txd->node);
1365 INIT_LIST_HEAD(&txd->dsg_list);
1367 /* Always enable error and terminal interrupts */
1368 txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK |
1369 PL080_CONFIG_TC_IRQ_MASK;
1375 * Initialize a descriptor to be used by memcpy submit
1377 static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy(
1378 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
1379 size_t len, unsigned long flags)
1381 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1382 struct pl08x_driver_data *pl08x = plchan->host;
1383 struct pl08x_txd *txd;
1384 struct pl08x_sg *dsg;
1387 txd = pl08x_get_txd(plchan, flags);
1389 dev_err(&pl08x->adev->dev,
1390 "%s no memory for descriptor\n", __func__);
1394 dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT);
1396 pl08x_free_txd(pl08x, txd);
1397 dev_err(&pl08x->adev->dev, "%s no memory for pl080 sg\n",
1401 list_add_tail(&dsg->node, &txd->dsg_list);
1403 txd->direction = DMA_MEM_TO_MEM;
1404 dsg->src_addr = src;
1405 dsg->dst_addr = dest;
1408 /* Set platform data for m2m */
1409 txd->ccfg |= PL080_FLOW_MEM2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT;
1410 txd->cctl = pl08x->pd->memcpy_channel.cctl &
1411 ~(PL080_CONTROL_DST_AHB2 | PL080_CONTROL_SRC_AHB2);
1413 /* Both to be incremented or the code will break */
1414 txd->cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR;
1416 if (pl08x->vd->dualmaster)
1417 txd->cctl |= pl08x_select_bus(pl08x->mem_buses,
1420 ret = pl08x_prep_channel_resources(plchan, txd);
1427 static struct dma_async_tx_descriptor *pl08x_prep_slave_sg(
1428 struct dma_chan *chan, struct scatterlist *sgl,
1429 unsigned int sg_len, enum dma_transfer_direction direction,
1430 unsigned long flags, void *context)
1432 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1433 struct pl08x_driver_data *pl08x = plchan->host;
1434 struct pl08x_txd *txd;
1435 struct pl08x_sg *dsg;
1436 struct scatterlist *sg;
1437 dma_addr_t slave_addr;
1439 u8 src_buses, dst_buses;
1442 dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n",
1443 __func__, sg_dma_len(sgl), plchan->name);
1445 txd = pl08x_get_txd(plchan, flags);
1447 dev_err(&pl08x->adev->dev, "%s no txd\n", __func__);
1452 * Set up addresses, the PrimeCell configured address
1453 * will take precedence since this may configure the
1454 * channel target address dynamically at runtime.
1456 txd->direction = direction;
1458 if (direction == DMA_MEM_TO_DEV) {
1459 cctl = plchan->dst_cctl | PL080_CONTROL_SRC_INCR;
1460 slave_addr = plchan->cfg.dst_addr;
1461 src_buses = pl08x->mem_buses;
1462 dst_buses = plchan->cd->periph_buses;
1463 } else if (direction == DMA_DEV_TO_MEM) {
1464 cctl = plchan->src_cctl | PL080_CONTROL_DST_INCR;
1465 slave_addr = plchan->cfg.src_addr;
1466 src_buses = plchan->cd->periph_buses;
1467 dst_buses = pl08x->mem_buses;
1469 pl08x_free_txd(pl08x, txd);
1470 dev_err(&pl08x->adev->dev,
1471 "%s direction unsupported\n", __func__);
1476 pl08x_free_txd(pl08x, txd);
1477 dev_err(&pl08x->adev->dev,
1478 "DMA slave configuration botched?\n");
1482 txd->cctl = cctl | pl08x_select_bus(src_buses, dst_buses);
1484 if (plchan->cfg.device_fc)
1485 tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER_PER :
1486 PL080_FLOW_PER2MEM_PER;
1488 tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER :
1491 txd->ccfg |= tmp << PL080_CONFIG_FLOW_CONTROL_SHIFT;
1493 for_each_sg(sgl, sg, sg_len, tmp) {
1494 dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT);
1496 pl08x_free_txd(pl08x, txd);
1497 dev_err(&pl08x->adev->dev, "%s no mem for pl080 sg\n",
1501 list_add_tail(&dsg->node, &txd->dsg_list);
1503 dsg->len = sg_dma_len(sg);
1504 if (direction == DMA_MEM_TO_DEV) {
1505 dsg->src_addr = sg_dma_address(sg);
1506 dsg->dst_addr = slave_addr;
1508 dsg->src_addr = slave_addr;
1509 dsg->dst_addr = sg_dma_address(sg);
1513 ret = pl08x_prep_channel_resources(plchan, txd);
1520 static int pl08x_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
1523 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1524 struct pl08x_driver_data *pl08x = plchan->host;
1525 unsigned long flags;
1528 /* Controls applicable to inactive channels */
1529 if (cmd == DMA_SLAVE_CONFIG) {
1530 return dma_set_runtime_config(chan,
1531 (struct dma_slave_config *)arg);
1535 * Anything succeeds on channels with no physical allocation and
1536 * no queued transfers.
1538 spin_lock_irqsave(&plchan->lock, flags);
1539 if (!plchan->phychan && !plchan->at) {
1540 spin_unlock_irqrestore(&plchan->lock, flags);
1545 case DMA_TERMINATE_ALL:
1546 plchan->state = PL08X_CHAN_IDLE;
1548 if (plchan->phychan) {
1549 pl08x_terminate_phy_chan(pl08x, plchan->phychan);
1552 * Mark physical channel as free and free any slave
1555 release_phy_channel(plchan);
1556 plchan->phychan_hold = 0;
1558 /* Dequeue jobs and free LLIs */
1560 pl08x_free_txd(pl08x, plchan->at);
1563 /* Dequeue jobs not yet fired as well */
1564 pl08x_free_txd_list(pl08x, plchan);
1567 pl08x_pause_phy_chan(plchan->phychan);
1568 plchan->state = PL08X_CHAN_PAUSED;
1571 pl08x_resume_phy_chan(plchan->phychan);
1572 plchan->state = PL08X_CHAN_RUNNING;
1575 /* Unknown command */
1580 spin_unlock_irqrestore(&plchan->lock, flags);
1585 bool pl08x_filter_id(struct dma_chan *chan, void *chan_id)
1587 struct pl08x_dma_chan *plchan;
1588 char *name = chan_id;
1590 /* Reject channels for devices not bound to this driver */
1591 if (chan->device->dev->driver != &pl08x_amba_driver.drv)
1594 plchan = to_pl08x_chan(chan);
1596 /* Check that the channel is not taken! */
1597 if (!strcmp(plchan->name, name))
1604 * Just check that the device is there and active
1605 * TODO: turn this bit on/off depending on the number of physical channels
1606 * actually used, if it is zero... well shut it off. That will save some
1607 * power. Cut the clock at the same time.
1609 static void pl08x_ensure_on(struct pl08x_driver_data *pl08x)
1611 /* The Nomadik variant does not have the config register */
1612 if (pl08x->vd->nomadik)
1614 writel(PL080_CONFIG_ENABLE, pl08x->base + PL080_CONFIG);
1617 static void pl08x_unmap_buffers(struct pl08x_txd *txd)
1619 struct device *dev = txd->tx.chan->device->dev;
1620 struct pl08x_sg *dsg;
1622 if (!(txd->tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
1623 if (txd->tx.flags & DMA_COMPL_SRC_UNMAP_SINGLE)
1624 list_for_each_entry(dsg, &txd->dsg_list, node)
1625 dma_unmap_single(dev, dsg->src_addr, dsg->len,
1628 list_for_each_entry(dsg, &txd->dsg_list, node)
1629 dma_unmap_page(dev, dsg->src_addr, dsg->len,
1633 if (!(txd->tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
1634 if (txd->tx.flags & DMA_COMPL_DEST_UNMAP_SINGLE)
1635 list_for_each_entry(dsg, &txd->dsg_list, node)
1636 dma_unmap_single(dev, dsg->dst_addr, dsg->len,
1639 list_for_each_entry(dsg, &txd->dsg_list, node)
1640 dma_unmap_page(dev, dsg->dst_addr, dsg->len,
1645 static void pl08x_tasklet(unsigned long data)
1647 struct pl08x_dma_chan *plchan = (struct pl08x_dma_chan *) data;
1648 struct pl08x_driver_data *pl08x = plchan->host;
1649 struct pl08x_txd *txd;
1650 unsigned long flags;
1652 spin_lock_irqsave(&plchan->lock, flags);
1658 /* Update last completed */
1659 dma_cookie_complete(&txd->tx);
1662 /* If a new descriptor is queued, set it up plchan->at is NULL here */
1663 if (!list_empty(&plchan->pend_list)) {
1664 struct pl08x_txd *next;
1666 next = list_first_entry(&plchan->pend_list,
1669 list_del(&next->node);
1671 pl08x_start_txd(plchan, next);
1672 } else if (plchan->phychan_hold) {
1674 * This channel is still in use - we have a new txd being
1675 * prepared and will soon be queued. Don't give up the
1679 struct pl08x_dma_chan *waiting = NULL;
1682 * No more jobs, so free up the physical channel
1683 * Free any allocated signal on slave transfers too
1685 release_phy_channel(plchan);
1686 plchan->state = PL08X_CHAN_IDLE;
1689 * And NOW before anyone else can grab that free:d up
1690 * physical channel, see if there is some memcpy pending
1691 * that seriously needs to start because of being stacked
1692 * up while we were choking the physical channels with data.
1694 list_for_each_entry(waiting, &pl08x->memcpy.channels,
1696 if (waiting->state == PL08X_CHAN_WAITING &&
1697 waiting->waiting != NULL) {
1700 /* This should REALLY not fail now */
1701 ret = prep_phy_channel(waiting,
1704 waiting->phychan_hold--;
1705 waiting->state = PL08X_CHAN_RUNNING;
1706 waiting->waiting = NULL;
1707 pl08x_issue_pending(&waiting->chan);
1713 spin_unlock_irqrestore(&plchan->lock, flags);
1716 dma_async_tx_callback callback = txd->tx.callback;
1717 void *callback_param = txd->tx.callback_param;
1719 /* Don't try to unmap buffers on slave channels */
1721 pl08x_unmap_buffers(txd);
1723 /* Free the descriptor */
1724 spin_lock_irqsave(&plchan->lock, flags);
1725 pl08x_free_txd(pl08x, txd);
1726 spin_unlock_irqrestore(&plchan->lock, flags);
1728 /* Callback to signal completion */
1730 callback(callback_param);
1734 static irqreturn_t pl08x_irq(int irq, void *dev)
1736 struct pl08x_driver_data *pl08x = dev;
1737 u32 mask = 0, err, tc, i;
1739 /* check & clear - ERR & TC interrupts */
1740 err = readl(pl08x->base + PL080_ERR_STATUS);
1742 dev_err(&pl08x->adev->dev, "%s error interrupt, register value 0x%08x\n",
1744 writel(err, pl08x->base + PL080_ERR_CLEAR);
1746 tc = readl(pl08x->base + PL080_TC_STATUS);
1748 writel(tc, pl08x->base + PL080_TC_CLEAR);
1753 for (i = 0; i < pl08x->vd->channels; i++) {
1754 if (((1 << i) & err) || ((1 << i) & tc)) {
1755 /* Locate physical channel */
1756 struct pl08x_phy_chan *phychan = &pl08x->phy_chans[i];
1757 struct pl08x_dma_chan *plchan = phychan->serving;
1760 dev_err(&pl08x->adev->dev,
1761 "%s Error TC interrupt on unused channel: 0x%08x\n",
1766 /* Schedule tasklet on this channel */
1767 tasklet_schedule(&plchan->tasklet);
1772 return mask ? IRQ_HANDLED : IRQ_NONE;
1775 static void pl08x_dma_slave_init(struct pl08x_dma_chan *chan)
1777 u32 cctl = pl08x_cctl(chan->cd->cctl);
1780 chan->name = chan->cd->bus_id;
1781 chan->cfg.src_addr = chan->cd->addr;
1782 chan->cfg.dst_addr = chan->cd->addr;
1783 chan->src_cctl = cctl;
1784 chan->dst_cctl = cctl;
1788 * Initialise the DMAC memcpy/slave channels.
1789 * Make a local wrapper to hold required data
1791 static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x,
1792 struct dma_device *dmadev, unsigned int channels, bool slave)
1794 struct pl08x_dma_chan *chan;
1797 INIT_LIST_HEAD(&dmadev->channels);
1800 * Register as many many memcpy as we have physical channels,
1801 * we won't always be able to use all but the code will have
1802 * to cope with that situation.
1804 for (i = 0; i < channels; i++) {
1805 chan = kzalloc(sizeof(*chan), GFP_KERNEL);
1807 dev_err(&pl08x->adev->dev,
1808 "%s no memory for channel\n", __func__);
1813 chan->state = PL08X_CHAN_IDLE;
1816 chan->cd = &pl08x->pd->slave_channels[i];
1817 pl08x_dma_slave_init(chan);
1819 chan->cd = &pl08x->pd->memcpy_channel;
1820 chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i);
1826 dev_dbg(&pl08x->adev->dev,
1827 "initialize virtual channel \"%s\"\n",
1830 chan->chan.device = dmadev;
1831 dma_cookie_init(&chan->chan);
1833 spin_lock_init(&chan->lock);
1834 INIT_LIST_HEAD(&chan->pend_list);
1835 tasklet_init(&chan->tasklet, pl08x_tasklet,
1836 (unsigned long) chan);
1838 list_add_tail(&chan->chan.device_node, &dmadev->channels);
1840 dev_info(&pl08x->adev->dev, "initialized %d virtual %s channels\n",
1841 i, slave ? "slave" : "memcpy");
1845 static void pl08x_free_virtual_channels(struct dma_device *dmadev)
1847 struct pl08x_dma_chan *chan = NULL;
1848 struct pl08x_dma_chan *next;
1850 list_for_each_entry_safe(chan,
1851 next, &dmadev->channels, chan.device_node) {
1852 list_del(&chan->chan.device_node);
1857 #ifdef CONFIG_DEBUG_FS
1858 static const char *pl08x_state_str(enum pl08x_dma_chan_state state)
1861 case PL08X_CHAN_IDLE:
1863 case PL08X_CHAN_RUNNING:
1865 case PL08X_CHAN_PAUSED:
1867 case PL08X_CHAN_WAITING:
1872 return "UNKNOWN STATE";
1875 static int pl08x_debugfs_show(struct seq_file *s, void *data)
1877 struct pl08x_driver_data *pl08x = s->private;
1878 struct pl08x_dma_chan *chan;
1879 struct pl08x_phy_chan *ch;
1880 unsigned long flags;
1883 seq_printf(s, "PL08x physical channels:\n");
1884 seq_printf(s, "CHANNEL:\tUSER:\n");
1885 seq_printf(s, "--------\t-----\n");
1886 for (i = 0; i < pl08x->vd->channels; i++) {
1887 struct pl08x_dma_chan *virt_chan;
1889 ch = &pl08x->phy_chans[i];
1891 spin_lock_irqsave(&ch->lock, flags);
1892 virt_chan = ch->serving;
1894 seq_printf(s, "%d\t\t%s%s\n",
1896 virt_chan ? virt_chan->name : "(none)",
1897 ch->locked ? " LOCKED" : "");
1899 spin_unlock_irqrestore(&ch->lock, flags);
1902 seq_printf(s, "\nPL08x virtual memcpy channels:\n");
1903 seq_printf(s, "CHANNEL:\tSTATE:\n");
1904 seq_printf(s, "--------\t------\n");
1905 list_for_each_entry(chan, &pl08x->memcpy.channels, chan.device_node) {
1906 seq_printf(s, "%s\t\t%s\n", chan->name,
1907 pl08x_state_str(chan->state));
1910 seq_printf(s, "\nPL08x virtual slave channels:\n");
1911 seq_printf(s, "CHANNEL:\tSTATE:\n");
1912 seq_printf(s, "--------\t------\n");
1913 list_for_each_entry(chan, &pl08x->slave.channels, chan.device_node) {
1914 seq_printf(s, "%s\t\t%s\n", chan->name,
1915 pl08x_state_str(chan->state));
1921 static int pl08x_debugfs_open(struct inode *inode, struct file *file)
1923 return single_open(file, pl08x_debugfs_show, inode->i_private);
1926 static const struct file_operations pl08x_debugfs_operations = {
1927 .open = pl08x_debugfs_open,
1929 .llseek = seq_lseek,
1930 .release = single_release,
1933 static void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
1935 /* Expose a simple debugfs interface to view all clocks */
1936 (void) debugfs_create_file(dev_name(&pl08x->adev->dev),
1937 S_IFREG | S_IRUGO, NULL, pl08x,
1938 &pl08x_debugfs_operations);
1942 static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
1947 static int pl08x_probe(struct amba_device *adev, const struct amba_id *id)
1949 struct pl08x_driver_data *pl08x;
1950 const struct vendor_data *vd = id->data;
1954 ret = amba_request_regions(adev, NULL);
1958 /* Create the driver state holder */
1959 pl08x = kzalloc(sizeof(*pl08x), GFP_KERNEL);
1965 /* Initialize memcpy engine */
1966 dma_cap_set(DMA_MEMCPY, pl08x->memcpy.cap_mask);
1967 pl08x->memcpy.dev = &adev->dev;
1968 pl08x->memcpy.device_alloc_chan_resources = pl08x_alloc_chan_resources;
1969 pl08x->memcpy.device_free_chan_resources = pl08x_free_chan_resources;
1970 pl08x->memcpy.device_prep_dma_memcpy = pl08x_prep_dma_memcpy;
1971 pl08x->memcpy.device_prep_dma_interrupt = pl08x_prep_dma_interrupt;
1972 pl08x->memcpy.device_tx_status = pl08x_dma_tx_status;
1973 pl08x->memcpy.device_issue_pending = pl08x_issue_pending;
1974 pl08x->memcpy.device_control = pl08x_control;
1976 /* Initialize slave engine */
1977 dma_cap_set(DMA_SLAVE, pl08x->slave.cap_mask);
1978 pl08x->slave.dev = &adev->dev;
1979 pl08x->slave.device_alloc_chan_resources = pl08x_alloc_chan_resources;
1980 pl08x->slave.device_free_chan_resources = pl08x_free_chan_resources;
1981 pl08x->slave.device_prep_dma_interrupt = pl08x_prep_dma_interrupt;
1982 pl08x->slave.device_tx_status = pl08x_dma_tx_status;
1983 pl08x->slave.device_issue_pending = pl08x_issue_pending;
1984 pl08x->slave.device_prep_slave_sg = pl08x_prep_slave_sg;
1985 pl08x->slave.device_control = pl08x_control;
1987 /* Get the platform data */
1988 pl08x->pd = dev_get_platdata(&adev->dev);
1990 dev_err(&adev->dev, "no platform data supplied\n");
1991 goto out_no_platdata;
1994 /* Assign useful pointers to the driver state */
1998 /* By default, AHB1 only. If dualmaster, from platform */
1999 pl08x->lli_buses = PL08X_AHB1;
2000 pl08x->mem_buses = PL08X_AHB1;
2001 if (pl08x->vd->dualmaster) {
2002 pl08x->lli_buses = pl08x->pd->lli_buses;
2003 pl08x->mem_buses = pl08x->pd->mem_buses;
2006 /* A DMA memory pool for LLIs, align on 1-byte boundary */
2007 pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev,
2008 PL08X_LLI_TSFR_SIZE, PL08X_ALIGN, 0);
2011 goto out_no_lli_pool;
2014 pl08x->base = ioremap(adev->res.start, resource_size(&adev->res));
2017 goto out_no_ioremap;
2020 /* Turn on the PL08x */
2021 pl08x_ensure_on(pl08x);
2023 /* Attach the interrupt handler */
2024 writel(0x000000FF, pl08x->base + PL080_ERR_CLEAR);
2025 writel(0x000000FF, pl08x->base + PL080_TC_CLEAR);
2027 ret = request_irq(adev->irq[0], pl08x_irq, IRQF_DISABLED,
2028 DRIVER_NAME, pl08x);
2030 dev_err(&adev->dev, "%s failed to request interrupt %d\n",
2031 __func__, adev->irq[0]);
2035 /* Initialize physical channels */
2036 pl08x->phy_chans = kzalloc((vd->channels * sizeof(*pl08x->phy_chans)),
2038 if (!pl08x->phy_chans) {
2039 dev_err(&adev->dev, "%s failed to allocate "
2040 "physical channel holders\n",
2042 goto out_no_phychans;
2045 for (i = 0; i < vd->channels; i++) {
2046 struct pl08x_phy_chan *ch = &pl08x->phy_chans[i];
2049 ch->base = pl08x->base + PL080_Cx_BASE(i);
2050 spin_lock_init(&ch->lock);
2054 * Nomadik variants can have channels that are locked
2055 * down for the secure world only. Lock up these channels
2056 * by perpetually serving a dummy virtual channel.
2061 val = readl(ch->base + PL080_CH_CONFIG);
2062 if (val & (PL080N_CONFIG_ITPROT | PL080N_CONFIG_SECPROT)) {
2063 dev_info(&adev->dev, "physical channel %d reserved for secure access only\n", i);
2068 dev_dbg(&adev->dev, "physical channel %d is %s\n",
2069 i, pl08x_phy_channel_busy(ch) ? "BUSY" : "FREE");
2072 /* Register as many memcpy channels as there are physical channels */
2073 ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->memcpy,
2074 pl08x->vd->channels, false);
2076 dev_warn(&pl08x->adev->dev,
2077 "%s failed to enumerate memcpy channels - %d\n",
2081 pl08x->memcpy.chancnt = ret;
2083 /* Register slave channels */
2084 ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->slave,
2085 pl08x->pd->num_slave_channels, true);
2087 dev_warn(&pl08x->adev->dev,
2088 "%s failed to enumerate slave channels - %d\n",
2092 pl08x->slave.chancnt = ret;
2094 ret = dma_async_device_register(&pl08x->memcpy);
2096 dev_warn(&pl08x->adev->dev,
2097 "%s failed to register memcpy as an async device - %d\n",
2099 goto out_no_memcpy_reg;
2102 ret = dma_async_device_register(&pl08x->slave);
2104 dev_warn(&pl08x->adev->dev,
2105 "%s failed to register slave as an async device - %d\n",
2107 goto out_no_slave_reg;
2110 amba_set_drvdata(adev, pl08x);
2111 init_pl08x_debugfs(pl08x);
2112 dev_info(&pl08x->adev->dev, "DMA: PL%03x rev%u at 0x%08llx irq %d\n",
2113 amba_part(adev), amba_rev(adev),
2114 (unsigned long long)adev->res.start, adev->irq[0]);
2119 dma_async_device_unregister(&pl08x->memcpy);
2121 pl08x_free_virtual_channels(&pl08x->slave);
2123 pl08x_free_virtual_channels(&pl08x->memcpy);
2125 kfree(pl08x->phy_chans);
2127 free_irq(adev->irq[0], pl08x);
2129 iounmap(pl08x->base);
2131 dma_pool_destroy(pl08x->pool);
2136 amba_release_regions(adev);
2140 /* PL080 has 8 channels and the PL080 have just 2 */
2141 static struct vendor_data vendor_pl080 = {
2146 static struct vendor_data vendor_nomadik = {
2152 static struct vendor_data vendor_pl081 = {
2154 .dualmaster = false,
2157 static struct amba_id pl08x_ids[] = {
2162 .data = &vendor_pl080,
2168 .data = &vendor_pl081,
2170 /* Nomadik 8815 PL080 variant */
2174 .data = &vendor_nomadik,
2179 MODULE_DEVICE_TABLE(amba, pl08x_ids);
2181 static struct amba_driver pl08x_amba_driver = {
2182 .drv.name = DRIVER_NAME,
2183 .id_table = pl08x_ids,
2184 .probe = pl08x_probe,
2187 static int __init pl08x_init(void)
2190 retval = amba_driver_register(&pl08x_amba_driver);
2192 printk(KERN_WARNING DRIVER_NAME
2193 "failed to register as an AMBA device (%d)\n",
2197 subsys_initcall(pl08x_init);