drivers/dma/sa11x0-dma.c

   1 /*
   2  * SA11x0 DMAengine support
   3  *
   4  * Copyright (C) 2012 Russell King
   5  *   Derived in part from arch/arm/mach-sa1100/dma.c,
   6  *   Copyright (C) 2000, 2001 by Nicolas Pitre
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 as
  10  * published by the Free Software Foundation.
  11  */
  12 #include <linux/sched.h>
  13 #include <linux/device.h>
  14 #include <linux/dmaengine.h>
  15 #include <linux/init.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/kernel.h>
  18 #include <linux/module.h>
  19 #include <linux/platform_device.h>
  20 #include <linux/sa11x0-dma.h>
  21 #include <linux/slab.h>
  22 #include <linux/spinlock.h>
  23
  24 #define NR_PHY_CHAN     6
  25 #define DMA_ALIGN       3
  26 #define DMA_MAX_SIZE    0x1fff
  27 #define DMA_CHUNK_SIZE  0x1000
  28
  29 #define DMA_DDAR        0x00
  30 #define DMA_DCSR_S      0x04
  31 #define DMA_DCSR_C      0x08
  32 #define DMA_DCSR_R      0x0c
  33 #define DMA_DBSA        0x10
  34 #define DMA_DBTA        0x14
  35 #define DMA_DBSB        0x18
  36 #define DMA_DBTB        0x1c
  37 #define DMA_SIZE        0x20
  38
  39 #define DCSR_RUN        (1 << 0)
  40 #define DCSR_IE         (1 << 1)
  41 #define DCSR_ERROR      (1 << 2)
  42 #define DCSR_DONEA      (1 << 3)
  43 #define DCSR_STRTA      (1 << 4)
  44 #define DCSR_DONEB      (1 << 5)
  45 #define DCSR_STRTB      (1 << 6)
  46 #define DCSR_BIU        (1 << 7)
  47
  48 #define DDAR_RW         (1 << 0)        /* 0 = W, 1 = R */
  49 #define DDAR_E          (1 << 1)        /* 0 = LE, 1 = BE */
  50 #define DDAR_BS         (1 << 2)        /* 0 = BS4, 1 = BS8 */
  51 #define DDAR_DW         (1 << 3)        /* 0 = 8b, 1 = 16b */
  52 #define DDAR_Ser0UDCTr  (0x0 << 4)
  53 #define DDAR_Ser0UDCRc  (0x1 << 4)
  54 #define DDAR_Ser1SDLCTr (0x2 << 4)
  55 #define DDAR_Ser1SDLCRc (0x3 << 4)
  56 #define DDAR_Ser1UARTTr (0x4 << 4)
  57 #define DDAR_Ser1UARTRc (0x5 << 4)
  58 #define DDAR_Ser2ICPTr  (0x6 << 4)
  59 #define DDAR_Ser2ICPRc  (0x7 << 4)
  60 #define DDAR_Ser3UARTTr (0x8 << 4)
  61 #define DDAR_Ser3UARTRc (0x9 << 4)
  62 #define DDAR_Ser4MCP0Tr (0xa << 4)
  63 #define DDAR_Ser4MCP0Rc (0xb << 4)
  64 #define DDAR_Ser4MCP1Tr (0xc << 4)
  65 #define DDAR_Ser4MCP1Rc (0xd << 4)
  66 #define DDAR_Ser4SSPTr  (0xe << 4)
  67 #define DDAR_Ser4SSPRc  (0xf << 4)
  68
  69 struct sa11x0_dma_sg {
  70         u32                     addr;
  71         u32                     len;
  72 };
  73
  74 struct sa11x0_dma_desc {
  75         struct dma_async_tx_descriptor tx;
  76         u32                     ddar;
  77         size_t                  size;
  78
  79         /* maybe protected by c->lock */
  80         struct list_head        node;
  81         unsigned                sglen;
  82         struct sa11x0_dma_sg    sg[0];
  83 };
  84
  85 struct sa11x0_dma_phy;
  86
  87 struct sa11x0_dma_chan {
  88         struct dma_chan         chan;
  89         spinlock_t              lock;
  90         dma_cookie_t            lc;
  91
  92         /* protected by c->lock */
  93         struct sa11x0_dma_phy   *phy;
  94         enum dma_status         status;
  95         struct list_head        desc_submitted;
  96         struct list_head        desc_issued;
  97
  98         /* protected by d->lock */
  99         struct list_head        node;
 100
 101         u32                     ddar;
 102         const char              *name;
 103 };
 104
 105 struct sa11x0_dma_phy {
 106         void __iomem            *base;
 107         struct sa11x0_dma_dev   *dev;
 108         unsigned                num;
 109
 110         struct sa11x0_dma_chan  *vchan;
 111
 112         /* Protected by c->lock */
 113         unsigned                sg_load;
 114         struct sa11x0_dma_desc  *txd_load;
 115         unsigned                sg_done;
 116         struct sa11x0_dma_desc  *txd_done;
 117 #ifdef CONFIG_PM_SLEEP
 118         u32                     dbs[2];
 119         u32                     dbt[2];
 120         u32                     dcsr;
 121 #endif
 122 };
 123
 124 struct sa11x0_dma_dev {
 125         struct dma_device       slave;
 126         void __iomem            *base;
 127         spinlock_t              lock;
 128         struct tasklet_struct   task;
 129         struct list_head        chan_pending;
 130         struct list_head        desc_complete;
 131         struct sa11x0_dma_phy   phy[NR_PHY_CHAN];
 132 };
 133
 134 static struct sa11x0_dma_chan *to_sa11x0_dma_chan(struct dma_chan *chan)
 135 {
 136         return container_of(chan, struct sa11x0_dma_chan, chan);
 137 }
 138
 139 static struct sa11x0_dma_dev *to_sa11x0_dma(struct dma_device *dmadev)
 140 {
 141         return container_of(dmadev, struct sa11x0_dma_dev, slave);
 142 }
 143
 144 static struct sa11x0_dma_desc *to_sa11x0_dma_tx(struct dma_async_tx_descriptor *tx)
 145 {
 146         return container_of(tx, struct sa11x0_dma_desc, tx);
 147 }
 148
 149 static struct sa11x0_dma_desc *sa11x0_dma_next_desc(struct sa11x0_dma_chan *c)
 150 {
 151         if (list_empty(&c->desc_issued))
 152                 return NULL;
 153
 154         return list_first_entry(&c->desc_issued, struct sa11x0_dma_desc, node);
 155 }
 156
 157 static void sa11x0_dma_start_desc(struct sa11x0_dma_phy *p, struct sa11x0_dma_desc *txd)
 158 {
 159         list_del(&txd->node);
 160         p->txd_load = txd;
 161         p->sg_load = 0;
 162
 163         dev_vdbg(p->dev->slave.dev, "pchan %u: txd %p[%x]: starting: DDAR:%x\n",
 164                 p->num, txd, txd->tx.cookie, txd->ddar);
 165 }
 166
 167 static void noinline sa11x0_dma_start_sg(struct sa11x0_dma_phy *p,
 168         struct sa11x0_dma_chan *c)
 169 {
 170         struct sa11x0_dma_desc *txd = p->txd_load;
 171         struct sa11x0_dma_sg *sg;
 172         void __iomem *base = p->base;
 173         unsigned dbsx, dbtx;
 174         u32 dcsr;
 175
 176         if (!txd)
 177                 return;
 178
 179         dcsr = readl_relaxed(base + DMA_DCSR_R);
 180
 181         /* Don't try to load the next transfer if both buffers are started */
 182         if ((dcsr & (DCSR_STRTA | DCSR_STRTB)) == (DCSR_STRTA | DCSR_STRTB))
 183                 return;
 184
 185         if (p->sg_load == txd->sglen) {
 186                 struct sa11x0_dma_desc *txn = sa11x0_dma_next_desc(c);
 187
 188                 /*
 189                  * We have reached the end of the current descriptor.
 190                  * Peek at the next descriptor, and if compatible with
 191                  * the current, start processing it.
 192                  */
 193                 if (txn && txn->ddar == txd->ddar) {
 194                         txd = txn;
 195                         sa11x0_dma_start_desc(p, txn);
 196                 } else {
 197                         p->txd_load = NULL;
 198                         return;
 199                 }
 200         }
 201
 202         sg = &txd->sg[p->sg_load++];
 203
 204         /* Select buffer to load according to channel status */
 205         if (((dcsr & (DCSR_BIU | DCSR_STRTB)) == (DCSR_BIU | DCSR_STRTB)) ||
 206             ((dcsr & (DCSR_BIU | DCSR_STRTA)) == 0)) {
 207                 dbsx = DMA_DBSA;
 208                 dbtx = DMA_DBTA;
 209                 dcsr = DCSR_STRTA | DCSR_IE | DCSR_RUN;
 210         } else {
 211                 dbsx = DMA_DBSB;
 212                 dbtx = DMA_DBTB;
 213                 dcsr = DCSR_STRTB | DCSR_IE | DCSR_RUN;
 214         }
 215
 216         writel_relaxed(sg->addr, base + dbsx);
 217         writel_relaxed(sg->len, base + dbtx);
 218         writel(dcsr, base + DMA_DCSR_S);
 219
 220         dev_dbg(p->dev->slave.dev, "pchan %u: load: DCSR:%02x DBS%c:%08x DBT%c:%08x\n",
 221                 p->num, dcsr,
 222                 'A' + (dbsx == DMA_DBSB), sg->addr,
 223                 'A' + (dbtx == DMA_DBTB), sg->len);
 224 }
 225
 226 static void noinline sa11x0_dma_complete(struct sa11x0_dma_phy *p,
 227         struct sa11x0_dma_chan *c)
 228 {
 229         struct sa11x0_dma_desc *txd = p->txd_done;
 230
 231         if (++p->sg_done == txd->sglen) {
 232                 struct sa11x0_dma_dev *d = p->dev;
 233
 234                 dev_vdbg(d->slave.dev, "pchan %u: txd %p[%x]: completed\n",
 235                         p->num, p->txd_done, p->txd_done->tx.cookie);
 236
 237                 c->lc = txd->tx.cookie;
 238
 239                 spin_lock(&d->lock);
 240                 list_add_tail(&txd->node, &d->desc_complete);
 241                 spin_unlock(&d->lock);
 242
 243                 p->sg_done = 0;
 244                 p->txd_done = p->txd_load;
 245
 246                 tasklet_schedule(&d->task);
 247         }
 248
 249         sa11x0_dma_start_sg(p, c);
 250 }
 251
 252 static irqreturn_t sa11x0_dma_irq(int irq, void *dev_id)
 253 {
 254         struct sa11x0_dma_phy *p = dev_id;
 255         struct sa11x0_dma_dev *d = p->dev;
 256         struct sa11x0_dma_chan *c;
 257         u32 dcsr;
 258
 259         dcsr = readl_relaxed(p->base + DMA_DCSR_R);
 260         if (!(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB)))
 261                 return IRQ_NONE;
 262
 263         /* Clear reported status bits */
 264         writel_relaxed(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB),
 265                 p->base + DMA_DCSR_C);
 266
 267         dev_dbg(d->slave.dev, "pchan %u: irq: DCSR:%02x\n", p->num, dcsr);
 268
 269         if (dcsr & DCSR_ERROR) {
 270                 dev_err(d->slave.dev, "pchan %u: error. DCSR:%02x DDAR:%08x DBSA:%08x DBTA:%08x DBSB:%08x DBTB:%08x\n",
 271                         p->num, dcsr,
 272                         readl_relaxed(p->base + DMA_DDAR),
 273                         readl_relaxed(p->base + DMA_DBSA),
 274                         readl_relaxed(p->base + DMA_DBTA),
 275                         readl_relaxed(p->base + DMA_DBSB),
 276                         readl_relaxed(p->base + DMA_DBTB));
 277         }
 278
 279         c = p->vchan;
 280         if (c) {
 281                 unsigned long flags;
 282
 283                 spin_lock_irqsave(&c->lock, flags);
 284                 /*
 285                  * Now that we're holding the lock, check that the vchan
 286                  * really is associated with this pchan before touching the
 287                  * hardware.  This should always succeed, because we won't
 288                  * change p->vchan or c->phy while the channel is actively
 289                  * transferring.
 290                  */
 291                 if (c->phy == p) {
 292                         if (dcsr & DCSR_DONEA)
 293                                 sa11x0_dma_complete(p, c);
 294                         if (dcsr & DCSR_DONEB)
 295                                 sa11x0_dma_complete(p, c);
 296                 }
 297                 spin_unlock_irqrestore(&c->lock, flags);
 298         }
 299
 300         return IRQ_HANDLED;
 301 }
 302
 303 static void sa11x0_dma_start_txd(struct sa11x0_dma_chan *c)
 304 {
 305         struct sa11x0_dma_desc *txd = sa11x0_dma_next_desc(c);
 306
 307         /* If the issued list is empty, we have no further txds to process */
 308         if (txd) {
 309                 struct sa11x0_dma_phy *p = c->phy;
 310
 311                 sa11x0_dma_start_desc(p, txd);
 312                 p->txd_done = txd;
 313                 p->sg_done = 0;
 314
 315                 /* The channel should not have any transfers started */
 316                 WARN_ON(readl_relaxed(p->base + DMA_DCSR_R) &
 317                                       (DCSR_STRTA | DCSR_STRTB));
 318
 319                 /* Clear the run and start bits before changing DDAR */
 320                 writel_relaxed(DCSR_RUN | DCSR_STRTA | DCSR_STRTB,
 321                                p->base + DMA_DCSR_C);
 322                 writel_relaxed(txd->ddar, p->base + DMA_DDAR);
 323
 324                 /* Try to start both buffers */
 325                 sa11x0_dma_start_sg(p, c);
 326                 sa11x0_dma_start_sg(p, c);
 327         }
 328 }
 329
 330 static void sa11x0_dma_tasklet(unsigned long arg)
 331 {
 332         struct sa11x0_dma_dev *d = (struct sa11x0_dma_dev *)arg;
 333         struct sa11x0_dma_phy *p;
 334         struct sa11x0_dma_chan *c;
 335         struct sa11x0_dma_desc *txd, *txn;
 336         LIST_HEAD(head);
 337         unsigned pch, pch_alloc = 0;
 338
 339         dev_dbg(d->slave.dev, "tasklet enter\n");
 340
 341         /* Get the completed tx descriptors */
 342         spin_lock_irq(&d->lock);
 343         list_splice_init(&d->desc_complete, &head);
 344         spin_unlock_irq(&d->lock);
 345
 346         list_for_each_entry(txd, &head, node) {
 347                 c = to_sa11x0_dma_chan(txd->tx.chan);
 348
 349                 dev_dbg(d->slave.dev, "vchan %p: txd %p[%x] completed\n",
 350                         c, txd, txd->tx.cookie);
 351
 352                 spin_lock_irq(&c->lock);
 353                 p = c->phy;
 354                 if (p) {
 355                         if (!p->txd_done)
 356                                 sa11x0_dma_start_txd(c);
 357                         if (!p->txd_done) {
 358                                 /* No current txd associated with this channel */
 359                                 dev_dbg(d->slave.dev, "pchan %u: free\n", p->num);
 360
 361                                 /* Mark this channel free */
 362                                 c->phy = NULL;
 363                                 p->vchan = NULL;
 364                         }
 365                 }
 366                 spin_unlock_irq(&c->lock);
 367         }
 368
 369         spin_lock_irq(&d->lock);
 370         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
 371                 p = &d->phy[pch];
 372
 373                 if (p->vchan == NULL && !list_empty(&d->chan_pending)) {
 374                         c = list_first_entry(&d->chan_pending,
 375                                 struct sa11x0_dma_chan, node);
 376                         list_del_init(&c->node);
 377
 378                         pch_alloc |= 1 << pch;
 379
 380                         /* Mark this channel allocated */
 381                         p->vchan = c;
 382
 383                         dev_dbg(d->slave.dev, "pchan %u: alloc vchan %p\n", pch, c);
 384                 }
 385         }
 386         spin_unlock_irq(&d->lock);
 387
 388         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
 389                 if (pch_alloc & (1 << pch)) {
 390                         p = &d->phy[pch];
 391                         c = p->vchan;
 392
 393                         spin_lock_irq(&c->lock);
 394                         c->phy = p;
 395
 396                         sa11x0_dma_start_txd(c);
 397                         spin_unlock_irq(&c->lock);
 398                 }
 399         }
 400
 401         /* Now free the completed tx descriptor, and call their callbacks */
 402         list_for_each_entry_safe(txd, txn, &head, node) {
 403                 dma_async_tx_callback callback = txd->tx.callback;
 404                 void *callback_param = txd->tx.callback_param;
 405
 406                 dev_dbg(d->slave.dev, "txd %p[%x]: callback and free\n",
 407                         txd, txd->tx.cookie);
 408
 409                 kfree(txd);
 410
 411                 if (callback)
 412                         callback(callback_param);
 413         }
 414
 415         dev_dbg(d->slave.dev, "tasklet exit\n");
 416 }
 417
 418
 419 static void sa11x0_dma_desc_free(struct sa11x0_dma_dev *d, struct list_head *head)
 420 {
 421         struct sa11x0_dma_desc *txd, *txn;
 422
 423         list_for_each_entry_safe(txd, txn, head, node) {
 424                 dev_dbg(d->slave.dev, "txd %p: freeing\n", txd);
 425                 kfree(txd);
 426         }
 427 }
 428
 429 static int sa11x0_dma_alloc_chan_resources(struct dma_chan *chan)
 430 {
 431         return 0;
 432 }
 433
 434 static void sa11x0_dma_free_chan_resources(struct dma_chan *chan)
 435 {
 436         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 437         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 438         unsigned long flags;
 439         LIST_HEAD(head);
 440
 441         spin_lock_irqsave(&c->lock, flags);
 442         spin_lock(&d->lock);
 443         list_del_init(&c->node);
 444         spin_unlock(&d->lock);
 445
 446         list_splice_tail_init(&c->desc_submitted, &head);
 447         list_splice_tail_init(&c->desc_issued, &head);
 448         spin_unlock_irqrestore(&c->lock, flags);
 449
 450         sa11x0_dma_desc_free(d, &head);
 451 }
 452
 453 static dma_addr_t sa11x0_dma_pos(struct sa11x0_dma_phy *p)
 454 {
 455         unsigned reg;
 456         u32 dcsr;
 457
 458         dcsr = readl_relaxed(p->base + DMA_DCSR_R);
 459
 460         if ((dcsr & (DCSR_BIU | DCSR_STRTA)) == DCSR_STRTA ||
 461             (dcsr & (DCSR_BIU | DCSR_STRTB)) == DCSR_BIU)
 462                 reg = DMA_DBSA;
 463         else
 464                 reg = DMA_DBSB;
 465
 466         return readl_relaxed(p->base + reg);
 467 }
 468
 469 static enum dma_status sa11x0_dma_tx_status(struct dma_chan *chan,
 470         dma_cookie_t cookie, struct dma_tx_state *state)
 471 {
 472         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 473         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 474         struct sa11x0_dma_phy *p;
 475         struct sa11x0_dma_desc *txd;
 476         dma_cookie_t last_used, last_complete;
 477         unsigned long flags;
 478         enum dma_status ret;
 479         size_t bytes = 0;
 480
 481         last_used = c->chan.cookie;
 482         last_complete = c->lc;
 483
 484         ret = dma_async_is_complete(cookie, last_complete, last_used);
 485         if (ret == DMA_SUCCESS) {
 486                 dma_set_tx_state(state, last_complete, last_used, 0);
 487                 return ret;
 488         }
 489
 490         spin_lock_irqsave(&c->lock, flags);
 491         p = c->phy;
 492         ret = c->status;
 493         if (p) {
 494                 dma_addr_t addr = sa11x0_dma_pos(p);
 495
 496                 dev_vdbg(d->slave.dev, "tx_status: addr:%x\n", addr);
 497
 498                 txd = p->txd_done;
 499                 if (txd) {
 500                         unsigned i;
 501
 502                         for (i = 0; i < txd->sglen; i++) {
 503                                 dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x\n",
 504                                         i, txd->sg[i].addr, txd->sg[i].len);
 505                                 if (addr >= txd->sg[i].addr &&
 506                                     addr < txd->sg[i].addr + txd->sg[i].len) {
 507                                         unsigned len;
 508
 509                                         len = txd->sg[i].len -
 510                                                 (addr - txd->sg[i].addr);
 511                                         dev_vdbg(d->slave.dev, "tx_status: [%u] +%x\n",
 512                                                 i, len);
 513                                         bytes += len;
 514                                         i++;
 515                                         break;
 516                                 }
 517                         }
 518                         for (; i < txd->sglen; i++) {
 519                                 dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x ++\n",
 520                                         i, txd->sg[i].addr, txd->sg[i].len);
 521                                 bytes += txd->sg[i].len;
 522                         }
 523                 }
 524                 if (txd != p->txd_load && p->txd_load)
 525                         bytes += p->txd_load->size;
 526         }
 527         list_for_each_entry(txd, &c->desc_issued, node) {
 528                 bytes += txd->size;
 529         }
 530         spin_unlock_irqrestore(&c->lock, flags);
 531
 532         dma_set_tx_state(state, last_complete, last_used, bytes);
 533
 534         dev_vdbg(d->slave.dev, "tx_status: bytes 0x%zx\n", bytes);
 535
 536         return ret;
 537 }
 538
 539 /*
 540  * Move pending txds to the issued list, and re-init pending list.
 541  * If not already pending, add this channel to the list of pending
 542  * channels and trigger the tasklet to run.
 543  */
 544 static void sa11x0_dma_issue_pending(struct dma_chan *chan)
 545 {
 546         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 547         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 548         unsigned long flags;
 549
 550         spin_lock_irqsave(&c->lock, flags);
 551         list_splice_tail_init(&c->desc_submitted, &c->desc_issued);
 552         if (!list_empty(&c->desc_issued)) {
 553                 spin_lock(&d->lock);
 554                 if (!c->phy && list_empty(&c->node)) {
 555                         list_add_tail(&c->node, &d->chan_pending);
 556                         tasklet_schedule(&d->task);
 557                         dev_dbg(d->slave.dev, "vchan %p: issued\n", c);
 558                 }
 559                 spin_unlock(&d->lock);
 560         } else
 561                 dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", c);
 562         spin_unlock_irqrestore(&c->lock, flags);
 563 }
 564
 565 static dma_cookie_t sa11x0_dma_tx_submit(struct dma_async_tx_descriptor *tx)
 566 {
 567         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(tx->chan);
 568         struct sa11x0_dma_desc *txd = to_sa11x0_dma_tx(tx);
 569         unsigned long flags;
 570
 571         spin_lock_irqsave(&c->lock, flags);
 572         c->chan.cookie += 1;
 573         if (c->chan.cookie < 0)
 574                 c->chan.cookie = 1;
 575         txd->tx.cookie = c->chan.cookie;
 576
 577         list_add_tail(&txd->node, &c->desc_submitted);
 578         spin_unlock_irqrestore(&c->lock, flags);
 579
 580         dev_dbg(tx->chan->device->dev, "vchan %p: txd %p[%x]: submitted\n",
 581                 c, txd, txd->tx.cookie);
 582
 583         return txd->tx.cookie;
 584 }
 585
 586 static struct dma_async_tx_descriptor *sa11x0_dma_prep_slave_sg(
 587         struct dma_chan *chan, struct scatterlist *sg, unsigned int sglen,
 588         enum dma_transfer_direction dir, unsigned long flags, void *context)
 589 {
 590         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 591         struct sa11x0_dma_desc *txd;
 592         struct scatterlist *sgent;
 593         unsigned i, j = sglen;
 594         size_t size = 0;
 595
 596         /* SA11x0 channels can only operate in their native direction */
 597         if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
 598                 dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
 599                         c, c->ddar, dir);
 600                 return NULL;
 601         }
 602
 603         /* Do not allow zero-sized txds */
 604         if (sglen == 0)
 605                 return NULL;
 606
 607         for_each_sg(sg, sgent, sglen, i) {
 608                 dma_addr_t addr = sg_dma_address(sgent);
 609                 unsigned int len = sg_dma_len(sgent);
 610
 611                 if (len > DMA_MAX_SIZE)
 612                         j += DIV_ROUND_UP(len, DMA_MAX_SIZE & ~DMA_ALIGN) - 1;
 613                 if (addr & DMA_ALIGN) {
 614                         dev_dbg(chan->device->dev, "vchan %p: bad buffer alignment: %08x\n",
 615                                 c, addr);
 616                         return NULL;
 617                 }
 618         }
 619
 620         txd = kzalloc(sizeof(*txd) + j * sizeof(txd->sg[0]), GFP_ATOMIC);
 621         if (!txd) {
 622                 dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", c);
 623                 return NULL;
 624         }
 625
 626         j = 0;
 627         for_each_sg(sg, sgent, sglen, i) {
 628                 dma_addr_t addr = sg_dma_address(sgent);
 629                 unsigned len = sg_dma_len(sgent);
 630
 631                 size += len;
 632
 633                 do {
 634                         unsigned tlen = len;
 635
 636                         /*
 637                          * Check whether the transfer will fit.  If not, try
 638                          * to split the transfer up such that we end up with
 639                          * equal chunks - but make sure that we preserve the
 640                          * alignment.  This avoids small segments.
 641                          */
 642                         if (tlen > DMA_MAX_SIZE) {
 643                                 unsigned mult = DIV_ROUND_UP(tlen,
 644                                         DMA_MAX_SIZE & ~DMA_ALIGN);
 645
 646                                 tlen = (tlen / mult) & ~DMA_ALIGN;
 647                         }
 648
 649                         txd->sg[j].addr = addr;
 650                         txd->sg[j].len = tlen;
 651
 652                         addr += tlen;
 653                         len -= tlen;
 654                         j++;
 655                 } while (len);
 656         }
 657
 658         dma_async_tx_descriptor_init(&txd->tx, &c->chan);
 659         txd->tx.flags = flags;
 660         txd->tx.tx_submit = sa11x0_dma_tx_submit;
 661         txd->ddar = c->ddar;
 662         txd->size = size;
 663         txd->sglen = j;
 664
 665         dev_dbg(chan->device->dev, "vchan %p: txd %p: size %u nr %u\n",
 666                 c, txd, txd->size, txd->sglen);
 667
 668         return &txd->tx;
 669 }
 670
 671 static int sa11x0_dma_slave_config(struct sa11x0_dma_chan *c, struct dma_slave_config *cfg)
 672 {
 673         u32 ddar = c->ddar & ((0xf << 4) | DDAR_RW);
 674         dma_addr_t addr;
 675         enum dma_slave_buswidth width;
 676         u32 maxburst;
 677
 678         if (ddar & DDAR_RW) {
 679                 addr = cfg->src_addr;
 680                 width = cfg->src_addr_width;
 681                 maxburst = cfg->src_maxburst;
 682         } else {
 683                 addr = cfg->dst_addr;
 684                 width = cfg->dst_addr_width;
 685                 maxburst = cfg->dst_maxburst;
 686         }
 687
 688         if ((width != DMA_SLAVE_BUSWIDTH_1_BYTE &&
 689              width != DMA_SLAVE_BUSWIDTH_2_BYTES) ||
 690             (maxburst != 4 && maxburst != 8))
 691                 return -EINVAL;
 692
 693         if (width == DMA_SLAVE_BUSWIDTH_2_BYTES)
 694                 ddar |= DDAR_DW;
 695         if (maxburst == 8)
 696                 ddar |= DDAR_BS;
 697
 698         dev_dbg(c->chan.device->dev, "vchan %p: dma_slave_config addr %x width %u burst %u\n",
 699                 c, addr, width, maxburst);
 700
 701         c->ddar = ddar | (addr & 0xf0000000) | (addr & 0x003ffffc) << 6;
 702
 703         return 0;
 704 }
 705
 706 static int sa11x0_dma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
 707         unsigned long arg)
 708 {
 709         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 710         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 711         struct sa11x0_dma_phy *p;
 712         LIST_HEAD(head);
 713         unsigned long flags;
 714         int ret;
 715
 716         switch (cmd) {
 717         case DMA_SLAVE_CONFIG:
 718                 return sa11x0_dma_slave_config(c, (struct dma_slave_config *)arg);
 719
 720         case DMA_TERMINATE_ALL:
 721                 dev_dbg(d->slave.dev, "vchan %p: terminate all\n", c);
 722                 /* Clear the tx descriptor lists */
 723                 spin_lock_irqsave(&c->lock, flags);
 724                 list_splice_tail_init(&c->desc_submitted, &head);
 725                 list_splice_tail_init(&c->desc_issued, &head);
 726
 727                 p = c->phy;
 728                 if (p) {
 729                         struct sa11x0_dma_desc *txd, *txn;
 730
 731                         dev_dbg(d->slave.dev, "pchan %u: terminating\n", p->num);
 732                         /* vchan is assigned to a pchan - stop the channel */
 733                         writel(DCSR_RUN | DCSR_IE |
 734                                 DCSR_STRTA | DCSR_DONEA |
 735                                 DCSR_STRTB | DCSR_DONEB,
 736                                 p->base + DMA_DCSR_C);
 737
 738                         list_for_each_entry_safe(txd, txn, &d->desc_complete, node)
 739                                 if (txd->tx.chan == &c->chan)
 740                                         list_move(&txd->node, &head);
 741
 742                         if (p->txd_load) {
 743                                 if (p->txd_load != p->txd_done)
 744                                         list_add_tail(&p->txd_load->node, &head);
 745                                 p->txd_load = NULL;
 746                         }
 747                         if (p->txd_done) {
 748                                 list_add_tail(&p->txd_done->node, &head);
 749                                 p->txd_done = NULL;
 750                         }
 751                         c->phy = NULL;
 752                         spin_lock(&d->lock);
 753                         p->vchan = NULL;
 754                         spin_unlock(&d->lock);
 755                         tasklet_schedule(&d->task);
 756                 }
 757                 spin_unlock_irqrestore(&c->lock, flags);
 758                 sa11x0_dma_desc_free(d, &head);
 759                 ret = 0;
 760                 break;
 761
 762         case DMA_PAUSE:
 763                 dev_dbg(d->slave.dev, "vchan %p: pause\n", c);
 764                 spin_lock_irqsave(&c->lock, flags);
 765                 if (c->status == DMA_IN_PROGRESS) {
 766                         c->status = DMA_PAUSED;
 767
 768                         p = c->phy;
 769                         if (p) {
 770                                 writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
 771                         } else {
 772                                 spin_lock(&d->lock);
 773                                 list_del_init(&c->node);
 774                                 spin_unlock(&d->lock);
 775                         }
 776                 }
 777                 spin_unlock_irqrestore(&c->lock, flags);
 778                 ret = 0;
 779                 break;
 780
 781         case DMA_RESUME:
 782                 dev_dbg(d->slave.dev, "vchan %p: resume\n", c);
 783                 spin_lock_irqsave(&c->lock, flags);
 784                 if (c->status == DMA_PAUSED) {
 785                         c->status = DMA_IN_PROGRESS;
 786
 787                         p = c->phy;
 788                         if (p) {
 789                                 writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_S);
 790                         } else if (!list_empty(&c->desc_issued)) {
 791                                 spin_lock(&d->lock);
 792                                 list_add_tail(&c->node, &d->chan_pending);
 793                                 spin_unlock(&d->lock);
 794                         }
 795                 }
 796                 spin_unlock_irqrestore(&c->lock, flags);
 797                 ret = 0;
 798                 break;
 799
 800         default:
 801                 ret = -ENXIO;
 802                 break;
 803         }
 804
 805         return ret;
 806 }
 807
 808 struct sa11x0_dma_channel_desc {
 809         u32 ddar;
 810         const char *name;
 811 };
 812
 813 #define CD(d1, d2) { .ddar = DDAR_##d1 | d2, .name = #d1 }
 814 static const struct sa11x0_dma_channel_desc chan_desc[] = {
 815         CD(Ser0UDCTr, 0),
 816         CD(Ser0UDCRc, DDAR_RW),
 817         CD(Ser1SDLCTr, 0),
 818         CD(Ser1SDLCRc, DDAR_RW),
 819         CD(Ser1UARTTr, 0),
 820         CD(Ser1UARTRc, DDAR_RW),
 821         CD(Ser2ICPTr, 0),
 822         CD(Ser2ICPRc, DDAR_RW),
 823         CD(Ser3UARTTr, 0),
 824         CD(Ser3UARTRc, DDAR_RW),
 825         CD(Ser4MCP0Tr, 0),
 826         CD(Ser4MCP0Rc, DDAR_RW),
 827         CD(Ser4MCP1Tr, 0),
 828         CD(Ser4MCP1Rc, DDAR_RW),
 829         CD(Ser4SSPTr, 0),
 830         CD(Ser4SSPRc, DDAR_RW),
 831 };
 832
 833 static int __devinit sa11x0_dma_init_dmadev(struct dma_device *dmadev,
 834         struct device *dev)
 835 {
 836         unsigned i;
 837
 838         dmadev->chancnt = ARRAY_SIZE(chan_desc);
 839         INIT_LIST_HEAD(&dmadev->channels);
 840         dmadev->dev = dev;
 841         dmadev->device_alloc_chan_resources = sa11x0_dma_alloc_chan_resources;
 842         dmadev->device_free_chan_resources = sa11x0_dma_free_chan_resources;
 843         dmadev->device_control = sa11x0_dma_control;
 844         dmadev->device_tx_status = sa11x0_dma_tx_status;
 845         dmadev->device_issue_pending = sa11x0_dma_issue_pending;
 846
 847         for (i = 0; i < dmadev->chancnt; i++) {
 848                 struct sa11x0_dma_chan *c;
 849
 850                 c = kzalloc(sizeof(*c), GFP_KERNEL);
 851                 if (!c) {
 852                         dev_err(dev, "no memory for channel %u\n", i);
 853                         return -ENOMEM;
 854                 }
 855
 856                 c->chan.device = dmadev;
 857                 c->status = DMA_IN_PROGRESS;
 858                 c->ddar = chan_desc[i].ddar;
 859                 c->name = chan_desc[i].name;
 860                 spin_lock_init(&c->lock);
 861                 INIT_LIST_HEAD(&c->desc_submitted);
 862                 INIT_LIST_HEAD(&c->desc_issued);
 863                 INIT_LIST_HEAD(&c->node);
 864                 list_add_tail(&c->chan.device_node, &dmadev->channels);
 865         }
 866
 867         return dma_async_device_register(dmadev);
 868 }
 869
 870 static int sa11x0_dma_request_irq(struct platform_device *pdev, int nr,
 871         void *data)
 872 {
 873         int irq = platform_get_irq(pdev, nr);
 874
 875         if (irq <= 0)
 876                 return -ENXIO;
 877
 878         return request_irq(irq, sa11x0_dma_irq, 0, dev_name(&pdev->dev), data);
 879 }
 880
 881 static void sa11x0_dma_free_irq(struct platform_device *pdev, int nr,
 882         void *data)
 883 {
 884         int irq = platform_get_irq(pdev, nr);
 885         if (irq > 0)
 886                 free_irq(irq, data);
 887 }
 888
 889 static void sa11x0_dma_free_channels(struct dma_device *dmadev)
 890 {
 891         struct sa11x0_dma_chan *c, *cn;
 892
 893         list_for_each_entry_safe(c, cn, &dmadev->channels, chan.device_node) {
 894                 list_del(&c->chan.device_node);
 895                 kfree(c);
 896         }
 897 }
 898
 899 static int __devinit sa11x0_dma_probe(struct platform_device *pdev)
 900 {
 901         struct sa11x0_dma_dev *d;
 902         struct resource *res;
 903         unsigned i;
 904         int ret;
 905
 906         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 907         if (!res)
 908                 return -ENXIO;
 909
 910         d = kzalloc(sizeof(*d), GFP_KERNEL);
 911         if (!d) {
 912                 ret = -ENOMEM;
 913                 goto err_alloc;
 914         }
 915
 916         spin_lock_init(&d->lock);
 917         INIT_LIST_HEAD(&d->chan_pending);
 918         INIT_LIST_HEAD(&d->desc_complete);
 919
 920         d->base = ioremap(res->start, resource_size(res));
 921         if (!d->base) {
 922                 ret = -ENOMEM;
 923                 goto err_ioremap;
 924         }
 925
 926         tasklet_init(&d->task, sa11x0_dma_tasklet, (unsigned long)d);
 927
 928         for (i = 0; i < NR_PHY_CHAN; i++) {
 929                 struct sa11x0_dma_phy *p = &d->phy[i];
 930
 931                 p->dev = d;
 932                 p->num = i;
 933                 p->base = d->base + i * DMA_SIZE;
 934                 writel_relaxed(DCSR_RUN | DCSR_IE | DCSR_ERROR |
 935                         DCSR_DONEA | DCSR_STRTA | DCSR_DONEB | DCSR_STRTB,
 936                         p->base + DMA_DCSR_C);
 937                 writel_relaxed(0, p->base + DMA_DDAR);
 938
 939                 ret = sa11x0_dma_request_irq(pdev, i, p);
 940                 if (ret) {
 941                         while (i) {
 942                                 i--;
 943                                 sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
 944                         }
 945                         goto err_irq;
 946                 }
 947         }
 948
 949         dma_cap_set(DMA_SLAVE, d->slave.cap_mask);
 950         d->slave.device_prep_slave_sg = sa11x0_dma_prep_slave_sg;
 951         ret = sa11x0_dma_init_dmadev(&d->slave, &pdev->dev);
 952         if (ret) {
 953                 dev_warn(d->slave.dev, "failed to register slave async device: %d\n",
 954                         ret);
 955                 goto err_slave_reg;
 956         }
 957
 958         platform_set_drvdata(pdev, d);
 959         return 0;
 960
 961  err_slave_reg:
 962         sa11x0_dma_free_channels(&d->slave);
 963         for (i = 0; i < NR_PHY_CHAN; i++)
 964                 sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
 965  err_irq:
 966         tasklet_kill(&d->task);
 967         iounmap(d->base);
 968  err_ioremap:
 969         kfree(d);
 970  err_alloc:
 971         return ret;
 972 }
 973
 974 static int __devexit sa11x0_dma_remove(struct platform_device *pdev)
 975 {
 976         struct sa11x0_dma_dev *d = platform_get_drvdata(pdev);
 977         unsigned pch;
 978
 979         dma_async_device_unregister(&d->slave);
 980
 981         sa11x0_dma_free_channels(&d->slave);
 982         for (pch = 0; pch < NR_PHY_CHAN; pch++)
 983                 sa11x0_dma_free_irq(pdev, pch, &d->phy[pch]);
 984         tasklet_kill(&d->task);
 985         iounmap(d->base);
 986         kfree(d);
 987
 988         return 0;
 989 }
 990
 991 #ifdef CONFIG_PM_SLEEP
 992 static int sa11x0_dma_suspend(struct device *dev)
 993 {
 994         struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
 995         unsigned pch;
 996
 997         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
 998                 struct sa11x0_dma_phy *p = &d->phy[pch];
 999                 u32 dcsr, saved_dcsr;
1000
1001                 dcsr = saved_dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1002                 if (dcsr & DCSR_RUN) {
1003                         writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
1004                         dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1005                 }
1006
1007                 saved_dcsr &= DCSR_RUN | DCSR_IE;
1008                 if (dcsr & DCSR_BIU) {
1009                         p->dbs[0] = readl_relaxed(p->base + DMA_DBSB);
1010                         p->dbt[0] = readl_relaxed(p->base + DMA_DBTB);
1011                         p->dbs[1] = readl_relaxed(p->base + DMA_DBSA);
1012                         p->dbt[1] = readl_relaxed(p->base + DMA_DBTA);
1013                         saved_dcsr |= (dcsr & DCSR_STRTA ? DCSR_STRTB : 0) |
1014                                       (dcsr & DCSR_STRTB ? DCSR_STRTA : 0);
1015                 } else {
1016                         p->dbs[0] = readl_relaxed(p->base + DMA_DBSA);
1017                         p->dbt[0] = readl_relaxed(p->base + DMA_DBTA);
1018                         p->dbs[1] = readl_relaxed(p->base + DMA_DBSB);
1019                         p->dbt[1] = readl_relaxed(p->base + DMA_DBTB);
1020                         saved_dcsr |= dcsr & (DCSR_STRTA | DCSR_STRTB);
1021                 }
1022                 p->dcsr = saved_dcsr;
1023
1024                 writel(DCSR_STRTA | DCSR_STRTB, p->base + DMA_DCSR_C);
1025         }
1026
1027         return 0;
1028 }
1029
1030 static int sa11x0_dma_resume(struct device *dev)
1031 {
1032         struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
1033         unsigned pch;
1034
1035         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
1036                 struct sa11x0_dma_phy *p = &d->phy[pch];
1037                 struct sa11x0_dma_desc *txd = NULL;
1038                 u32 dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1039
1040                 WARN_ON(dcsr & (DCSR_BIU | DCSR_STRTA | DCSR_STRTB | DCSR_RUN));
1041
1042                 if (p->txd_done)
1043                         txd = p->txd_done;
1044                 else if (p->txd_load)
1045                         txd = p->txd_load;
1046
1047                 if (!txd)
1048                         continue;
1049
1050                 writel_relaxed(txd->ddar, p->base + DMA_DDAR);
1051
1052                 writel_relaxed(p->dbs[0], p->base + DMA_DBSA);
1053                 writel_relaxed(p->dbt[0], p->base + DMA_DBTA);
1054                 writel_relaxed(p->dbs[1], p->base + DMA_DBSB);
1055                 writel_relaxed(p->dbt[1], p->base + DMA_DBTB);
1056                 writel_relaxed(p->dcsr, p->base + DMA_DCSR_S);
1057         }
1058
1059         return 0;
1060 }
1061 #endif
1062
1063 static const struct dev_pm_ops sa11x0_dma_pm_ops = {
1064         .suspend_noirq = sa11x0_dma_suspend,
1065         .resume_noirq = sa11x0_dma_resume,
1066         .freeze_noirq = sa11x0_dma_suspend,
1067         .thaw_noirq = sa11x0_dma_resume,
1068         .poweroff_noirq = sa11x0_dma_suspend,
1069         .restore_noirq = sa11x0_dma_resume,
1070 };
1071
1072 static struct platform_driver sa11x0_dma_driver = {
1073         .driver = {
1074                 .name   = "sa11x0-dma",
1075                 .owner  = THIS_MODULE,
1076                 .pm     = &sa11x0_dma_pm_ops,
1077         },
1078         .probe          = sa11x0_dma_probe,
1079         .remove         = __devexit_p(sa11x0_dma_remove),
1080 };
1081
1082 bool sa11x0_dma_filter_fn(struct dma_chan *chan, void *param)
1083 {
1084         if (chan->device->dev->driver == &sa11x0_dma_driver.driver) {
1085                 struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
1086                 const char *p = param;
1087
1088                 return !strcmp(c->name, p);
1089         }
1090         return false;
1091 }
1092 EXPORT_SYMBOL(sa11x0_dma_filter_fn);
1093
1094 static int __init sa11x0_dma_init(void)
1095 {
1096         return platform_driver_register(&sa11x0_dma_driver);
1097 }
1098 subsys_initcall(sa11x0_dma_init);
1099
1100 static void __exit sa11x0_dma_exit(void)
1101 {
1102         platform_driver_unregister(&sa11x0_dma_driver);
1103 }
1104 module_exit(sa11x0_dma_exit);
1105
1106 MODULE_AUTHOR("Russell King");
1107 MODULE_DESCRIPTION("SA-11x0 DMA driver");
1108 MODULE_LICENSE("GPL v2");
1109 MODULE_ALIAS("platform:sa11x0-dma");