drivers/dma/cppi41.c

   1 #include <linux/dmaengine.h>
   2 #include <linux/dma-mapping.h>
   3 #include <linux/platform_device.h>
   4 #include <linux/module.h>
   5 #include <linux/of.h>
   6 #include <linux/slab.h>
   7 #include <linux/of_dma.h>
   8 #include <linux/of_irq.h>
   9 #include <linux/dmapool.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/of_address.h>
  12 #include <linux/pm_runtime.h>
  13 #include "dmaengine.h"
  14
  15 #define DESC_TYPE       27
  16 #define DESC_TYPE_HOST  0x10
  17 #define DESC_TYPE_TEARD 0x13
  18
  19 #define TD_DESC_IS_RX   (1 << 16)
  20 #define TD_DESC_DMA_NUM 10
  21
  22 #define DESC_LENGTH_BITS_NUM    21
  23
  24 #define DESC_TYPE_USB   (5 << 26)
  25 #define DESC_PD_COMPLETE        (1 << 31)
  26
  27 /* DMA engine */
  28 #define DMA_TDFDQ       4
  29 #define DMA_TXGCR(x)    (0x800 + (x) * 0x20)
  30 #define DMA_RXGCR(x)    (0x808 + (x) * 0x20)
  31 #define RXHPCRA0                4
  32
  33 #define GCR_CHAN_ENABLE         (1 << 31)
  34 #define GCR_TEARDOWN            (1 << 30)
  35 #define GCR_STARV_RETRY         (1 << 24)
  36 #define GCR_DESC_TYPE_HOST      (1 << 14)
  37
  38 /* DMA scheduler */
  39 #define DMA_SCHED_CTRL          0
  40 #define DMA_SCHED_CTRL_EN       (1 << 31)
  41 #define DMA_SCHED_WORD(x)       ((x) * 4 + 0x800)
  42
  43 #define SCHED_ENTRY0_CHAN(x)    ((x) << 0)
  44 #define SCHED_ENTRY0_IS_RX      (1 << 7)
  45
  46 #define SCHED_ENTRY1_CHAN(x)    ((x) << 8)
  47 #define SCHED_ENTRY1_IS_RX      (1 << 15)
  48
  49 #define SCHED_ENTRY2_CHAN(x)    ((x) << 16)
  50 #define SCHED_ENTRY2_IS_RX      (1 << 23)
  51
  52 #define SCHED_ENTRY3_CHAN(x)    ((x) << 24)
  53 #define SCHED_ENTRY3_IS_RX      (1 << 31)
  54
  55 /* Queue manager */
  56 /* 4 KiB of memory for descriptors, 2 for each endpoint */
  57 #define ALLOC_DECS_NUM          128
  58 #define DESCS_AREAS             1
  59 #define TOTAL_DESCS_NUM         (ALLOC_DECS_NUM * DESCS_AREAS)
  60 #define QMGR_SCRATCH_SIZE       (TOTAL_DESCS_NUM * 4)
  61
  62 #define QMGR_LRAM0_BASE         0x80
  63 #define QMGR_LRAM_SIZE          0x84
  64 #define QMGR_LRAM1_BASE         0x88
  65 #define QMGR_MEMBASE(x)         (0x1000 + (x) * 0x10)
  66 #define QMGR_MEMCTRL(x)         (0x1004 + (x) * 0x10)
  67 #define QMGR_MEMCTRL_IDX_SH     16
  68 #define QMGR_MEMCTRL_DESC_SH    8
  69
  70 #define QMGR_NUM_PEND   5
  71 #define QMGR_PEND(x)    (0x90 + (x) * 4)
  72
  73 #define QMGR_PENDING_SLOT_Q(x)  (x / 32)
  74 #define QMGR_PENDING_BIT_Q(x)   (x % 32)
  75
  76 #define QMGR_QUEUE_A(n) (0x2000 + (n) * 0x10)
  77 #define QMGR_QUEUE_B(n) (0x2004 + (n) * 0x10)
  78 #define QMGR_QUEUE_C(n) (0x2008 + (n) * 0x10)
  79 #define QMGR_QUEUE_D(n) (0x200c + (n) * 0x10)
  80
  81 /* Glue layer specific */
  82 /* USBSS  / USB AM335x */
  83 #define USBSS_IRQ_STATUS        0x28
  84 #define USBSS_IRQ_ENABLER       0x2c
  85 #define USBSS_IRQ_CLEARR        0x30
  86
  87 #define USBSS_IRQ_PD_COMP       (1 <<  2)
  88
  89 /* Packet Descriptor */
  90 #define PD2_ZERO_LENGTH         (1 << 19)
  91
  92 struct cppi41_channel {
  93         struct dma_chan chan;
  94         struct dma_async_tx_descriptor txd;
  95         struct cppi41_dd *cdd;
  96         struct cppi41_desc *desc;
  97         dma_addr_t desc_phys;
  98         void __iomem *gcr_reg;
  99         int is_tx;
 100         u32 residue;
 101
 102         unsigned int q_num;
 103         unsigned int q_comp_num;
 104         unsigned int port_num;
 105
 106         unsigned td_retry;
 107         unsigned td_queued:1;
 108         unsigned td_seen:1;
 109         unsigned td_desc_seen:1;
 110 };
 111
 112 struct cppi41_desc {
 113         u32 pd0;
 114         u32 pd1;
 115         u32 pd2;
 116         u32 pd3;
 117         u32 pd4;
 118         u32 pd5;
 119         u32 pd6;
 120         u32 pd7;
 121 } __aligned(32);
 122
 123 struct chan_queues {
 124         u16 submit;
 125         u16 complete;
 126 };
 127
 128 struct cppi41_dd {
 129         struct dma_device ddev;
 130
 131         void *qmgr_scratch;
 132         dma_addr_t scratch_phys;
 133
 134         struct cppi41_desc *cd;
 135         dma_addr_t descs_phys;
 136         u32 first_td_desc;
 137         struct cppi41_channel *chan_busy[ALLOC_DECS_NUM];
 138
 139         void __iomem *usbss_mem;
 140         void __iomem *ctrl_mem;
 141         void __iomem *sched_mem;
 142         void __iomem *qmgr_mem;
 143         unsigned int irq;
 144         const struct chan_queues *queues_rx;
 145         const struct chan_queues *queues_tx;
 146         struct chan_queues td_queue;
 147
 148         /* context for suspend/resume */
 149         unsigned int dma_tdfdq;
 150 };
 151
 152 #define FIST_COMPLETION_QUEUE   93
 153 static struct chan_queues usb_queues_tx[] = {
 154         /* USB0 ENDP 1 */
 155         [ 0] = { .submit = 32, .complete =  93},
 156         [ 1] = { .submit = 34, .complete =  94},
 157         [ 2] = { .submit = 36, .complete =  95},
 158         [ 3] = { .submit = 38, .complete =  96},
 159         [ 4] = { .submit = 40, .complete =  97},
 160         [ 5] = { .submit = 42, .complete =  98},
 161         [ 6] = { .submit = 44, .complete =  99},
 162         [ 7] = { .submit = 46, .complete = 100},
 163         [ 8] = { .submit = 48, .complete = 101},
 164         [ 9] = { .submit = 50, .complete = 102},
 165         [10] = { .submit = 52, .complete = 103},
 166         [11] = { .submit = 54, .complete = 104},
 167         [12] = { .submit = 56, .complete = 105},
 168         [13] = { .submit = 58, .complete = 106},
 169         [14] = { .submit = 60, .complete = 107},
 170
 171         /* USB1 ENDP1 */
 172         [15] = { .submit = 62, .complete = 125},
 173         [16] = { .submit = 64, .complete = 126},
 174         [17] = { .submit = 66, .complete = 127},
 175         [18] = { .submit = 68, .complete = 128},
 176         [19] = { .submit = 70, .complete = 129},
 177         [20] = { .submit = 72, .complete = 130},
 178         [21] = { .submit = 74, .complete = 131},
 179         [22] = { .submit = 76, .complete = 132},
 180         [23] = { .submit = 78, .complete = 133},
 181         [24] = { .submit = 80, .complete = 134},
 182         [25] = { .submit = 82, .complete = 135},
 183         [26] = { .submit = 84, .complete = 136},
 184         [27] = { .submit = 86, .complete = 137},
 185         [28] = { .submit = 88, .complete = 138},
 186         [29] = { .submit = 90, .complete = 139},
 187 };
 188
 189 static const struct chan_queues usb_queues_rx[] = {
 190         /* USB0 ENDP 1 */
 191         [ 0] = { .submit =  1, .complete = 109},
 192         [ 1] = { .submit =  2, .complete = 110},
 193         [ 2] = { .submit =  3, .complete = 111},
 194         [ 3] = { .submit =  4, .complete = 112},
 195         [ 4] = { .submit =  5, .complete = 113},
 196         [ 5] = { .submit =  6, .complete = 114},
 197         [ 6] = { .submit =  7, .complete = 115},
 198         [ 7] = { .submit =  8, .complete = 116},
 199         [ 8] = { .submit =  9, .complete = 117},
 200         [ 9] = { .submit = 10, .complete = 118},
 201         [10] = { .submit = 11, .complete = 119},
 202         [11] = { .submit = 12, .complete = 120},
 203         [12] = { .submit = 13, .complete = 121},
 204         [13] = { .submit = 14, .complete = 122},
 205         [14] = { .submit = 15, .complete = 123},
 206
 207         /* USB1 ENDP 1 */
 208         [15] = { .submit = 16, .complete = 141},
 209         [16] = { .submit = 17, .complete = 142},
 210         [17] = { .submit = 18, .complete = 143},
 211         [18] = { .submit = 19, .complete = 144},
 212         [19] = { .submit = 20, .complete = 145},
 213         [20] = { .submit = 21, .complete = 146},
 214         [21] = { .submit = 22, .complete = 147},
 215         [22] = { .submit = 23, .complete = 148},
 216         [23] = { .submit = 24, .complete = 149},
 217         [24] = { .submit = 25, .complete = 150},
 218         [25] = { .submit = 26, .complete = 151},
 219         [26] = { .submit = 27, .complete = 152},
 220         [27] = { .submit = 28, .complete = 153},
 221         [28] = { .submit = 29, .complete = 154},
 222         [29] = { .submit = 30, .complete = 155},
 223 };
 224
 225 struct cppi_glue_infos {
 226         irqreturn_t (*isr)(int irq, void *data);
 227         const struct chan_queues *queues_rx;
 228         const struct chan_queues *queues_tx;
 229         struct chan_queues td_queue;
 230 };
 231
 232 static struct cppi41_channel *to_cpp41_chan(struct dma_chan *c)
 233 {
 234         return container_of(c, struct cppi41_channel, chan);
 235 }
 236
 237 static struct cppi41_channel *desc_to_chan(struct cppi41_dd *cdd, u32 desc)
 238 {
 239         struct cppi41_channel *c;
 240         u32 descs_size;
 241         u32 desc_num;
 242
 243         descs_size = sizeof(struct cppi41_desc) * ALLOC_DECS_NUM;
 244
 245         if (!((desc >= cdd->descs_phys) &&
 246                         (desc < (cdd->descs_phys + descs_size)))) {
 247                 return NULL;
 248         }
 249
 250         desc_num = (desc - cdd->descs_phys) / sizeof(struct cppi41_desc);
 251         BUG_ON(desc_num >= ALLOC_DECS_NUM);
 252         c = cdd->chan_busy[desc_num];
 253         cdd->chan_busy[desc_num] = NULL;
 254         return c;
 255 }
 256
 257 static void cppi_writel(u32 val, void *__iomem *mem)
 258 {
 259         __raw_writel(val, mem);
 260 }
 261
 262 static u32 cppi_readl(void *__iomem *mem)
 263 {
 264         return __raw_readl(mem);
 265 }
 266
 267 static u32 pd_trans_len(u32 val)
 268 {
 269         return val & ((1 << (DESC_LENGTH_BITS_NUM + 1)) - 1);
 270 }
 271
 272 static u32 cppi41_pop_desc(struct cppi41_dd *cdd, unsigned queue_num)
 273 {
 274         u32 desc;
 275
 276         desc = cppi_readl(cdd->qmgr_mem + QMGR_QUEUE_D(queue_num));
 277         desc &= ~0x1f;
 278         return desc;
 279 }
 280
 281 static irqreturn_t cppi41_irq(int irq, void *data)
 282 {
 283         struct cppi41_dd *cdd = data;
 284         struct cppi41_channel *c;
 285         u32 status;
 286         int i;
 287
 288         status = cppi_readl(cdd->usbss_mem + USBSS_IRQ_STATUS);
 289         if (!(status & USBSS_IRQ_PD_COMP))
 290                 return IRQ_NONE;
 291         cppi_writel(status, cdd->usbss_mem + USBSS_IRQ_STATUS);
 292
 293         for (i = QMGR_PENDING_SLOT_Q(FIST_COMPLETION_QUEUE); i < QMGR_NUM_PEND;
 294                         i++) {
 295                 u32 val;
 296                 u32 q_num;
 297
 298                 val = cppi_readl(cdd->qmgr_mem + QMGR_PEND(i));
 299                 if (i == QMGR_PENDING_SLOT_Q(FIST_COMPLETION_QUEUE) && val) {
 300                         u32 mask;
 301                         /* set corresponding bit for completetion Q 93 */
 302                         mask = 1 << QMGR_PENDING_BIT_Q(FIST_COMPLETION_QUEUE);
 303                         /* not set all bits for queues less than Q 93 */
 304                         mask--;
 305                         /* now invert and keep only Q 93+ set */
 306                         val &= ~mask;
 307                 }
 308
 309                 if (val)
 310                         __iormb();
 311
 312                 while (val) {
 313                         u32 desc, len;
 314
 315                         q_num = __fls(val);
 316                         val &= ~(1 << q_num);
 317                         q_num += 32 * i;
 318                         desc = cppi41_pop_desc(cdd, q_num);
 319                         c = desc_to_chan(cdd, desc);
 320                         if (WARN_ON(!c)) {
 321                                 pr_err("%s() q %d desc %08x\n", __func__,
 322                                                 q_num, desc);
 323                                 continue;
 324                         }
 325
 326                         if (c->desc->pd2 & PD2_ZERO_LENGTH)
 327                                 len = 0;
 328                         else
 329                                 len = pd_trans_len(c->desc->pd0);
 330
 331                         c->residue = pd_trans_len(c->desc->pd6) - len;
 332                         dma_cookie_complete(&c->txd);
 333                         c->txd.callback(c->txd.callback_param);
 334                 }
 335         }
 336         return IRQ_HANDLED;
 337 }
 338
 339 static dma_cookie_t cppi41_tx_submit(struct dma_async_tx_descriptor *tx)
 340 {
 341         dma_cookie_t cookie;
 342
 343         cookie = dma_cookie_assign(tx);
 344
 345         return cookie;
 346 }
 347
 348 static int cppi41_dma_alloc_chan_resources(struct dma_chan *chan)
 349 {
 350         struct cppi41_channel *c = to_cpp41_chan(chan);
 351
 352         dma_cookie_init(chan);
 353         dma_async_tx_descriptor_init(&c->txd, chan);
 354         c->txd.tx_submit = cppi41_tx_submit;
 355
 356         if (!c->is_tx)
 357                 cppi_writel(c->q_num, c->gcr_reg + RXHPCRA0);
 358
 359         return 0;
 360 }
 361
 362 static void cppi41_dma_free_chan_resources(struct dma_chan *chan)
 363 {
 364 }
 365
 366 static enum dma_status cppi41_dma_tx_status(struct dma_chan *chan,
 367         dma_cookie_t cookie, struct dma_tx_state *txstate)
 368 {
 369         struct cppi41_channel *c = to_cpp41_chan(chan);
 370         enum dma_status ret;
 371
 372         /* lock */
 373         ret = dma_cookie_status(chan, cookie, txstate);
 374         if (txstate && ret == DMA_COMPLETE)
 375                 txstate->residue = c->residue;
 376         /* unlock */
 377
 378         return ret;
 379 }
 380
 381 static void push_desc_queue(struct cppi41_channel *c)
 382 {
 383         struct cppi41_dd *cdd = c->cdd;
 384         u32 desc_num;
 385         u32 desc_phys;
 386         u32 reg;
 387
 388         desc_phys = lower_32_bits(c->desc_phys);
 389         desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc);
 390         WARN_ON(cdd->chan_busy[desc_num]);
 391         cdd->chan_busy[desc_num] = c;
 392
 393         reg = (sizeof(struct cppi41_desc) - 24) / 4;
 394         reg |= desc_phys;
 395         cppi_writel(reg, cdd->qmgr_mem + QMGR_QUEUE_D(c->q_num));
 396 }
 397
 398 static void cppi41_dma_issue_pending(struct dma_chan *chan)
 399 {
 400         struct cppi41_channel *c = to_cpp41_chan(chan);
 401         u32 reg;
 402
 403         c->residue = 0;
 404
 405         reg = GCR_CHAN_ENABLE;
 406         if (!c->is_tx) {
 407                 reg |= GCR_STARV_RETRY;
 408                 reg |= GCR_DESC_TYPE_HOST;
 409                 reg |= c->q_comp_num;
 410         }
 411
 412         cppi_writel(reg, c->gcr_reg);
 413
 414         /*
 415          * We don't use writel() but __raw_writel() so we have to make sure
 416          * that the DMA descriptor in coherent memory made to the main memory
 417          * before starting the dma engine.
 418          */
 419         __iowmb();
 420         push_desc_queue(c);
 421 }
 422
 423 static u32 get_host_pd0(u32 length)
 424 {
 425         u32 reg;
 426
 427         reg = DESC_TYPE_HOST << DESC_TYPE;
 428         reg |= length;
 429
 430         return reg;
 431 }
 432
 433 static u32 get_host_pd1(struct cppi41_channel *c)
 434 {
 435         u32 reg;
 436
 437         reg = 0;
 438
 439         return reg;
 440 }
 441
 442 static u32 get_host_pd2(struct cppi41_channel *c)
 443 {
 444         u32 reg;
 445
 446         reg = DESC_TYPE_USB;
 447         reg |= c->q_comp_num;
 448
 449         return reg;
 450 }
 451
 452 static u32 get_host_pd3(u32 length)
 453 {
 454         u32 reg;
 455
 456         /* PD3 = packet size */
 457         reg = length;
 458
 459         return reg;
 460 }
 461
 462 static u32 get_host_pd6(u32 length)
 463 {
 464         u32 reg;
 465
 466         /* PD6 buffer size */
 467         reg = DESC_PD_COMPLETE;
 468         reg |= length;
 469
 470         return reg;
 471 }
 472
 473 static u32 get_host_pd4_or_7(u32 addr)
 474 {
 475         u32 reg;
 476
 477         reg = addr;
 478
 479         return reg;
 480 }
 481
 482 static u32 get_host_pd5(void)
 483 {
 484         u32 reg;
 485
 486         reg = 0;
 487
 488         return reg;
 489 }
 490
 491 static struct dma_async_tx_descriptor *cppi41_dma_prep_slave_sg(
 492         struct dma_chan *chan, struct scatterlist *sgl, unsigned sg_len,
 493         enum dma_transfer_direction dir, unsigned long tx_flags, void *context)
 494 {
 495         struct cppi41_channel *c = to_cpp41_chan(chan);
 496         struct cppi41_desc *d;
 497         struct scatterlist *sg;
 498         unsigned int i;
 499         unsigned int num;
 500
 501         num = 0;
 502         d = c->desc;
 503         for_each_sg(sgl, sg, sg_len, i) {
 504                 u32 addr;
 505                 u32 len;
 506
 507                 /* We need to use more than one desc once musb supports sg */
 508                 BUG_ON(num > 0);
 509                 addr = lower_32_bits(sg_dma_address(sg));
 510                 len = sg_dma_len(sg);
 511
 512                 d->pd0 = get_host_pd0(len);
 513                 d->pd1 = get_host_pd1(c);
 514                 d->pd2 = get_host_pd2(c);
 515                 d->pd3 = get_host_pd3(len);
 516                 d->pd4 = get_host_pd4_or_7(addr);
 517                 d->pd5 = get_host_pd5();
 518                 d->pd6 = get_host_pd6(len);
 519                 d->pd7 = get_host_pd4_or_7(addr);
 520
 521                 d++;
 522         }
 523
 524         return &c->txd;
 525 }
 526
 527 static int cpp41_cfg_chan(struct cppi41_channel *c,
 528                 struct dma_slave_config *cfg)
 529 {
 530         return 0;
 531 }
 532
 533 static void cppi41_compute_td_desc(struct cppi41_desc *d)
 534 {
 535         d->pd0 = DESC_TYPE_TEARD << DESC_TYPE;
 536 }
 537
 538 static int cppi41_tear_down_chan(struct cppi41_channel *c)
 539 {
 540         struct cppi41_dd *cdd = c->cdd;
 541         struct cppi41_desc *td;
 542         u32 reg;
 543         u32 desc_phys;
 544         u32 td_desc_phys;
 545
 546         td = cdd->cd;
 547         td += cdd->first_td_desc;
 548
 549         td_desc_phys = cdd->descs_phys;
 550         td_desc_phys += cdd->first_td_desc * sizeof(struct cppi41_desc);
 551
 552         if (!c->td_queued) {
 553                 cppi41_compute_td_desc(td);
 554                 __iowmb();
 555
 556                 reg = (sizeof(struct cppi41_desc) - 24) / 4;
 557                 reg |= td_desc_phys;
 558                 cppi_writel(reg, cdd->qmgr_mem +
 559                                 QMGR_QUEUE_D(cdd->td_queue.submit));
 560
 561                 reg = GCR_CHAN_ENABLE;
 562                 if (!c->is_tx) {
 563                         reg |= GCR_STARV_RETRY;
 564                         reg |= GCR_DESC_TYPE_HOST;
 565                         reg |= c->q_comp_num;
 566                 }
 567                 reg |= GCR_TEARDOWN;
 568                 cppi_writel(reg, c->gcr_reg);
 569                 c->td_queued = 1;
 570                 c->td_retry = 100;
 571         }
 572
 573         if (!c->td_seen || !c->td_desc_seen) {
 574
 575                 desc_phys = cppi41_pop_desc(cdd, cdd->td_queue.complete);
 576                 if (!desc_phys)
 577                         desc_phys = cppi41_pop_desc(cdd, c->q_comp_num);
 578
 579                 if (desc_phys == c->desc_phys) {
 580                         c->td_desc_seen = 1;
 581
 582                 } else if (desc_phys == td_desc_phys) {
 583                         u32 pd0;
 584
 585                         __iormb();
 586                         pd0 = td->pd0;
 587                         WARN_ON((pd0 >> DESC_TYPE) != DESC_TYPE_TEARD);
 588                         WARN_ON(!c->is_tx && !(pd0 & TD_DESC_IS_RX));
 589                         WARN_ON((pd0 & 0x1f) != c->port_num);
 590                         c->td_seen = 1;
 591                 } else if (desc_phys) {
 592                         WARN_ON_ONCE(1);
 593                 }
 594         }
 595         c->td_retry--;
 596         /*
 597          * If the TX descriptor / channel is in use, the caller needs to poke
 598          * his TD bit multiple times. After that he hardware releases the
 599          * transfer descriptor followed by TD descriptor. Waiting seems not to
 600          * cause any difference.
 601          * RX seems to be thrown out right away. However once the TearDown
 602          * descriptor gets through we are done. If we have seens the transfer
 603          * descriptor before the TD we fetch it from enqueue, it has to be
 604          * there waiting for us.
 605          */
 606         if (!c->td_seen && c->td_retry)
 607                 return -EAGAIN;
 608
 609         WARN_ON(!c->td_retry);
 610         if (!c->td_desc_seen) {
 611                 desc_phys = cppi41_pop_desc(cdd, c->q_num);
 612                 WARN_ON(!desc_phys);
 613         }
 614
 615         c->td_queued = 0;
 616         c->td_seen = 0;
 617         c->td_desc_seen = 0;
 618         cppi_writel(0, c->gcr_reg);
 619         return 0;
 620 }
 621
 622 static int cppi41_stop_chan(struct dma_chan *chan)
 623 {
 624         struct cppi41_channel *c = to_cpp41_chan(chan);
 625         struct cppi41_dd *cdd = c->cdd;
 626         u32 desc_num;
 627         u32 desc_phys;
 628         int ret;
 629
 630         desc_phys = lower_32_bits(c->desc_phys);
 631         desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc);
 632         if (!cdd->chan_busy[desc_num])
 633                 return 0;
 634
 635         ret = cppi41_tear_down_chan(c);
 636         if (ret)
 637                 return ret;
 638
 639         WARN_ON(!cdd->chan_busy[desc_num]);
 640         cdd->chan_busy[desc_num] = NULL;
 641
 642         return 0;
 643 }
 644
 645 static int cppi41_dma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
 646         unsigned long arg)
 647 {
 648         struct cppi41_channel *c = to_cpp41_chan(chan);
 649         int ret;
 650
 651         switch (cmd) {
 652         case DMA_SLAVE_CONFIG:
 653                 ret = cpp41_cfg_chan(c, (struct dma_slave_config *) arg);
 654                 break;
 655
 656         case DMA_TERMINATE_ALL:
 657                 ret = cppi41_stop_chan(chan);
 658                 break;
 659
 660         default:
 661                 ret = -ENXIO;
 662                 break;
 663         }
 664         return ret;
 665 }
 666
 667 static void cleanup_chans(struct cppi41_dd *cdd)
 668 {
 669         while (!list_empty(&cdd->ddev.channels)) {
 670                 struct cppi41_channel *cchan;
 671
 672                 cchan = list_first_entry(&cdd->ddev.channels,
 673                                 struct cppi41_channel, chan.device_node);
 674                 list_del(&cchan->chan.device_node);
 675                 kfree(cchan);
 676         }
 677 }
 678
 679 static int cppi41_add_chans(struct device *dev, struct cppi41_dd *cdd)
 680 {
 681         struct cppi41_channel *cchan;
 682         int i;
 683         int ret;
 684         u32 n_chans;
 685
 686         ret = of_property_read_u32(dev->of_node, "#dma-channels",
 687                         &n_chans);
 688         if (ret)
 689                 return ret;
 690         /*
 691          * The channels can only be used as TX or as RX. So we add twice
 692          * that much dma channels because USB can only do RX or TX.
 693          */
 694         n_chans *= 2;
 695
 696         for (i = 0; i < n_chans; i++) {
 697                 cchan = kzalloc(sizeof(*cchan), GFP_KERNEL);
 698                 if (!cchan)
 699                         goto err;
 700
 701                 cchan->cdd = cdd;
 702                 if (i & 1) {
 703                         cchan->gcr_reg = cdd->ctrl_mem + DMA_TXGCR(i >> 1);
 704                         cchan->is_tx = 1;
 705                 } else {
 706                         cchan->gcr_reg = cdd->ctrl_mem + DMA_RXGCR(i >> 1);
 707                         cchan->is_tx = 0;
 708                 }
 709                 cchan->port_num = i >> 1;
 710                 cchan->desc = &cdd->cd[i];
 711                 cchan->desc_phys = cdd->descs_phys;
 712                 cchan->desc_phys += i * sizeof(struct cppi41_desc);
 713                 cchan->chan.device = &cdd->ddev;
 714                 list_add_tail(&cchan->chan.device_node, &cdd->ddev.channels);
 715         }
 716         cdd->first_td_desc = n_chans;
 717
 718         return 0;
 719 err:
 720         cleanup_chans(cdd);
 721         return -ENOMEM;
 722 }
 723
 724 static void purge_descs(struct device *dev, struct cppi41_dd *cdd)
 725 {
 726         unsigned int mem_decs;
 727         int i;
 728
 729         mem_decs = ALLOC_DECS_NUM * sizeof(struct cppi41_desc);
 730
 731         for (i = 0; i < DESCS_AREAS; i++) {
 732
 733                 cppi_writel(0, cdd->qmgr_mem + QMGR_MEMBASE(i));
 734                 cppi_writel(0, cdd->qmgr_mem + QMGR_MEMCTRL(i));
 735
 736                 dma_free_coherent(dev, mem_decs, cdd->cd,
 737                                 cdd->descs_phys);
 738         }
 739 }
 740
 741 static void disable_sched(struct cppi41_dd *cdd)
 742 {
 743         cppi_writel(0, cdd->sched_mem + DMA_SCHED_CTRL);
 744 }
 745
 746 static void deinit_cppi41(struct device *dev, struct cppi41_dd *cdd)
 747 {
 748         disable_sched(cdd);
 749
 750         purge_descs(dev, cdd);
 751
 752         cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM0_BASE);
 753         cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM0_BASE);
 754         dma_free_coherent(dev, QMGR_SCRATCH_SIZE, cdd->qmgr_scratch,
 755                         cdd->scratch_phys);
 756 }
 757
 758 static int init_descs(struct device *dev, struct cppi41_dd *cdd)
 759 {
 760         unsigned int desc_size;
 761         unsigned int mem_decs;
 762         int i;
 763         u32 reg;
 764         u32 idx;
 765
 766         BUILD_BUG_ON(sizeof(struct cppi41_desc) &
 767                         (sizeof(struct cppi41_desc) - 1));
 768         BUILD_BUG_ON(sizeof(struct cppi41_desc) < 32);
 769         BUILD_BUG_ON(ALLOC_DECS_NUM < 32);
 770
 771         desc_size = sizeof(struct cppi41_desc);
 772         mem_decs = ALLOC_DECS_NUM * desc_size;
 773
 774         idx = 0;
 775         for (i = 0; i < DESCS_AREAS; i++) {
 776
 777                 reg = idx << QMGR_MEMCTRL_IDX_SH;
 778                 reg |= (ilog2(desc_size) - 5) << QMGR_MEMCTRL_DESC_SH;
 779                 reg |= ilog2(ALLOC_DECS_NUM) - 5;
 780
 781                 BUILD_BUG_ON(DESCS_AREAS != 1);
 782                 cdd->cd = dma_alloc_coherent(dev, mem_decs,
 783                                 &cdd->descs_phys, GFP_KERNEL);
 784                 if (!cdd->cd)
 785                         return -ENOMEM;
 786
 787                 cppi_writel(cdd->descs_phys, cdd->qmgr_mem + QMGR_MEMBASE(i));
 788                 cppi_writel(reg, cdd->qmgr_mem + QMGR_MEMCTRL(i));
 789
 790                 idx += ALLOC_DECS_NUM;
 791         }
 792         return 0;
 793 }
 794
 795 static void init_sched(struct cppi41_dd *cdd)
 796 {
 797         unsigned ch;
 798         unsigned word;
 799         u32 reg;
 800
 801         word = 0;
 802         cppi_writel(0, cdd->sched_mem + DMA_SCHED_CTRL);
 803         for (ch = 0; ch < 15 * 2; ch += 2) {
 804
 805                 reg = SCHED_ENTRY0_CHAN(ch);
 806                 reg |= SCHED_ENTRY1_CHAN(ch) | SCHED_ENTRY1_IS_RX;
 807
 808                 reg |= SCHED_ENTRY2_CHAN(ch + 1);
 809                 reg |= SCHED_ENTRY3_CHAN(ch + 1) | SCHED_ENTRY3_IS_RX;
 810                 cppi_writel(reg, cdd->sched_mem + DMA_SCHED_WORD(word));
 811                 word++;
 812         }
 813         reg = 15 * 2 * 2 - 1;
 814         reg |= DMA_SCHED_CTRL_EN;
 815         cppi_writel(reg, cdd->sched_mem + DMA_SCHED_CTRL);
 816 }
 817
 818 static int init_cppi41(struct device *dev, struct cppi41_dd *cdd)
 819 {
 820         int ret;
 821
 822         BUILD_BUG_ON(QMGR_SCRATCH_SIZE > ((1 << 14) - 1));
 823         cdd->qmgr_scratch = dma_alloc_coherent(dev, QMGR_SCRATCH_SIZE,
 824                         &cdd->scratch_phys, GFP_KERNEL);
 825         if (!cdd->qmgr_scratch)
 826                 return -ENOMEM;
 827
 828         cppi_writel(cdd->scratch_phys, cdd->qmgr_mem + QMGR_LRAM0_BASE);
 829         cppi_writel(QMGR_SCRATCH_SIZE, cdd->qmgr_mem + QMGR_LRAM_SIZE);
 830         cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM1_BASE);
 831
 832         ret = init_descs(dev, cdd);
 833         if (ret)
 834                 goto err_td;
 835
 836         cppi_writel(cdd->td_queue.submit, cdd->ctrl_mem + DMA_TDFDQ);
 837         init_sched(cdd);
 838         return 0;
 839 err_td:
 840         deinit_cppi41(dev, cdd);
 841         return ret;
 842 }
 843
 844 static struct platform_driver cpp41_dma_driver;
 845 /*
 846  * The param format is:
 847  * X Y
 848  * X: Port
 849  * Y: 0 = RX else TX
 850  */
 851 #define INFO_PORT       0
 852 #define INFO_IS_TX      1
 853
 854 static bool cpp41_dma_filter_fn(struct dma_chan *chan, void *param)
 855 {
 856         struct cppi41_channel *cchan;
 857         struct cppi41_dd *cdd;
 858         const struct chan_queues *queues;
 859         u32 *num = param;
 860
 861         if (chan->device->dev->driver != &cpp41_dma_driver.driver)
 862                 return false;
 863
 864         cchan = to_cpp41_chan(chan);
 865
 866         if (cchan->port_num != num[INFO_PORT])
 867                 return false;
 868
 869         if (cchan->is_tx && !num[INFO_IS_TX])
 870                 return false;
 871         cdd = cchan->cdd;
 872         if (cchan->is_tx)
 873                 queues = cdd->queues_tx;
 874         else
 875                 queues = cdd->queues_rx;
 876
 877         BUILD_BUG_ON(ARRAY_SIZE(usb_queues_rx) != ARRAY_SIZE(usb_queues_tx));
 878         if (WARN_ON(cchan->port_num > ARRAY_SIZE(usb_queues_rx)))
 879                 return false;
 880
 881         cchan->q_num = queues[cchan->port_num].submit;
 882         cchan->q_comp_num = queues[cchan->port_num].complete;
 883         return true;
 884 }
 885
 886 static struct of_dma_filter_info cpp41_dma_info = {
 887         .filter_fn = cpp41_dma_filter_fn,
 888 };
 889
 890 static struct dma_chan *cppi41_dma_xlate(struct of_phandle_args *dma_spec,
 891                 struct of_dma *ofdma)
 892 {
 893         int count = dma_spec->args_count;
 894         struct of_dma_filter_info *info = ofdma->of_dma_data;
 895
 896         if (!info || !info->filter_fn)
 897                 return NULL;
 898
 899         if (count != 2)
 900                 return NULL;
 901
 902         return dma_request_channel(info->dma_cap, info->filter_fn,
 903                         &dma_spec->args[0]);
 904 }
 905
 906 static const struct cppi_glue_infos usb_infos = {
 907         .isr = cppi41_irq,
 908         .queues_rx = usb_queues_rx,
 909         .queues_tx = usb_queues_tx,
 910         .td_queue = { .submit = 31, .complete = 0 },
 911 };
 912
 913 static const struct of_device_id cppi41_dma_ids[] = {
 914         { .compatible = "ti,am3359-cppi41", .data = &usb_infos},
 915         {},
 916 };
 917 MODULE_DEVICE_TABLE(of, cppi41_dma_ids);
 918
 919 static const struct cppi_glue_infos *get_glue_info(struct device *dev)
 920 {
 921         const struct of_device_id *of_id;
 922
 923         of_id = of_match_node(cppi41_dma_ids, dev->of_node);
 924         if (!of_id)
 925                 return NULL;
 926         return of_id->data;
 927 }
 928
 929 static int cppi41_dma_probe(struct platform_device *pdev)
 930 {
 931         struct cppi41_dd *cdd;
 932         struct device *dev = &pdev->dev;
 933         const struct cppi_glue_infos *glue_info;
 934         int irq;
 935         int ret;
 936
 937         glue_info = get_glue_info(dev);
 938         if (!glue_info)
 939                 return -EINVAL;
 940
 941         cdd = devm_kzalloc(&pdev->dev, sizeof(*cdd), GFP_KERNEL);
 942         if (!cdd)
 943                 return -ENOMEM;
 944
 945         dma_cap_set(DMA_SLAVE, cdd->ddev.cap_mask);
 946         cdd->ddev.device_alloc_chan_resources = cppi41_dma_alloc_chan_resources;
 947         cdd->ddev.device_free_chan_resources = cppi41_dma_free_chan_resources;
 948         cdd->ddev.device_tx_status = cppi41_dma_tx_status;
 949         cdd->ddev.device_issue_pending = cppi41_dma_issue_pending;
 950         cdd->ddev.device_prep_slave_sg = cppi41_dma_prep_slave_sg;
 951         cdd->ddev.device_control = cppi41_dma_control;
 952         cdd->ddev.dev = dev;
 953         INIT_LIST_HEAD(&cdd->ddev.channels);
 954         cpp41_dma_info.dma_cap = cdd->ddev.cap_mask;
 955
 956         cdd->usbss_mem = of_iomap(dev->of_node, 0);
 957         cdd->ctrl_mem = of_iomap(dev->of_node, 1);
 958         cdd->sched_mem = of_iomap(dev->of_node, 2);
 959         cdd->qmgr_mem = of_iomap(dev->of_node, 3);
 960
 961         if (!cdd->usbss_mem || !cdd->ctrl_mem || !cdd->sched_mem ||
 962                         !cdd->qmgr_mem)
 963                 return -ENXIO;
 964
 965         pm_runtime_enable(dev);
 966         ret = pm_runtime_get_sync(dev);
 967         if (ret < 0)
 968                 goto err_get_sync;
 969
 970         cdd->queues_rx = glue_info->queues_rx;
 971         cdd->queues_tx = glue_info->queues_tx;
 972         cdd->td_queue = glue_info->td_queue;
 973
 974         ret = init_cppi41(dev, cdd);
 975         if (ret)
 976                 goto err_init_cppi;
 977
 978         ret = cppi41_add_chans(dev, cdd);
 979         if (ret)
 980                 goto err_chans;
 981
 982         irq = irq_of_parse_and_map(dev->of_node, 0);
 983         if (!irq) {
 984                 ret = -EINVAL;
 985                 goto err_irq;
 986         }
 987
 988         cppi_writel(USBSS_IRQ_PD_COMP, cdd->usbss_mem + USBSS_IRQ_ENABLER);
 989
 990         ret = devm_request_irq(&pdev->dev, irq, glue_info->isr, IRQF_SHARED,
 991                         dev_name(dev), cdd);
 992         if (ret)
 993                 goto err_irq;
 994         cdd->irq = irq;
 995
 996         ret = dma_async_device_register(&cdd->ddev);
 997         if (ret)
 998                 goto err_dma_reg;
 999
1000         ret = of_dma_controller_register(dev->of_node,
1001                         cppi41_dma_xlate, &cpp41_dma_info);
1002         if (ret)
1003                 goto err_of;
1004
1005         platform_set_drvdata(pdev, cdd);
1006         return 0;
1007 err_of:
1008         dma_async_device_unregister(&cdd->ddev);
1009 err_dma_reg:
1010 err_irq:
1011         cppi_writel(0, cdd->usbss_mem + USBSS_IRQ_CLEARR);
1012         cleanup_chans(cdd);
1013 err_chans:
1014         deinit_cppi41(dev, cdd);
1015 err_init_cppi:
1016         pm_runtime_put(dev);
1017 err_get_sync:
1018         pm_runtime_disable(dev);
1019         iounmap(cdd->usbss_mem);
1020         iounmap(cdd->ctrl_mem);
1021         iounmap(cdd->sched_mem);
1022         iounmap(cdd->qmgr_mem);
1023         return ret;
1024 }
1025
1026 static int cppi41_dma_remove(struct platform_device *pdev)
1027 {
1028         struct cppi41_dd *cdd = platform_get_drvdata(pdev);
1029
1030         of_dma_controller_free(pdev->dev.of_node);
1031         dma_async_device_unregister(&cdd->ddev);
1032
1033         cppi_writel(0, cdd->usbss_mem + USBSS_IRQ_CLEARR);
1034         devm_free_irq(&pdev->dev, cdd->irq, cdd);
1035         cleanup_chans(cdd);
1036         deinit_cppi41(&pdev->dev, cdd);
1037         iounmap(cdd->usbss_mem);
1038         iounmap(cdd->ctrl_mem);
1039         iounmap(cdd->sched_mem);
1040         iounmap(cdd->qmgr_mem);
1041         pm_runtime_put(&pdev->dev);
1042         pm_runtime_disable(&pdev->dev);
1043         return 0;
1044 }
1045
1046 #ifdef CONFIG_PM_SLEEP
1047 static int cppi41_suspend(struct device *dev)
1048 {
1049         struct cppi41_dd *cdd = dev_get_drvdata(dev);
1050
1051         cdd->dma_tdfdq = cppi_readl(cdd->ctrl_mem + DMA_TDFDQ);
1052         cppi_writel(0, cdd->usbss_mem + USBSS_IRQ_CLEARR);
1053         disable_sched(cdd);
1054
1055         return 0;
1056 }
1057
1058 static int cppi41_resume(struct device *dev)
1059 {
1060         struct cppi41_dd *cdd = dev_get_drvdata(dev);
1061         struct cppi41_channel *c;
1062         int i;
1063
1064         for (i = 0; i < DESCS_AREAS; i++)
1065                 cppi_writel(cdd->descs_phys, cdd->qmgr_mem + QMGR_MEMBASE(i));
1066
1067         list_for_each_entry(c, &cdd->ddev.channels, chan.device_node)
1068                 if (!c->is_tx)
1069                         cppi_writel(c->q_num, c->gcr_reg + RXHPCRA0);
1070
1071         init_sched(cdd);
1072
1073         cppi_writel(cdd->dma_tdfdq, cdd->ctrl_mem + DMA_TDFDQ);
1074         cppi_writel(cdd->scratch_phys, cdd->qmgr_mem + QMGR_LRAM0_BASE);
1075         cppi_writel(QMGR_SCRATCH_SIZE, cdd->qmgr_mem + QMGR_LRAM_SIZE);
1076         cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM1_BASE);
1077
1078         cppi_writel(USBSS_IRQ_PD_COMP, cdd->usbss_mem + USBSS_IRQ_ENABLER);
1079
1080         return 0;
1081 }
1082 #endif
1083
1084 static SIMPLE_DEV_PM_OPS(cppi41_pm_ops, cppi41_suspend, cppi41_resume);
1085
1086 static struct platform_driver cpp41_dma_driver = {
1087         .probe  = cppi41_dma_probe,
1088         .remove = cppi41_dma_remove,
1089         .driver = {
1090                 .name = "cppi41-dma-engine",
1091                 .owner = THIS_MODULE,
1092                 .pm = &cppi41_pm_ops,
1093                 .of_match_table = of_match_ptr(cppi41_dma_ids),
1094         },
1095 };
1096
1097 module_platform_driver(cpp41_dma_driver);
1098 MODULE_LICENSE("GPL");
1099 MODULE_AUTHOR("Sebastian Andrzej Siewior <bigeasy@linutronix.de>");