arch/arm/plat-omap/dma.c

   1 /*
   2  * linux/arch/arm/plat-omap/dma.c
   3  *
   4  * Copyright (C) 2003 - 2008 Nokia Corporation
   5  * Author: Juha Yrjölä <juha.yrjola@nokia.com>
   6  * DMA channel linking for 1610 by Samuel Ortiz <samuel.ortiz@nokia.com>
   7  * Graphics DMA and LCD DMA graphics tranformations
   8  * by Imre Deak <imre.deak@nokia.com>
   9  * OMAP2/3 support Copyright (C) 2004-2007 Texas Instruments, Inc.
  10  * Merged to support both OMAP1 and OMAP2 by Tony Lindgren <tony@atomide.com>
  11  * Some functions based on earlier dma-omap.c Copyright (C) 2001 RidgeRun, Inc.
  12  *
  13  * Copyright (C) 2009 Texas Instruments
  14  * Added OMAP4 support - Santosh Shilimkar <santosh.shilimkar@ti.com>
  15  *
  16  * Support functions for the OMAP internal DMA channels.
  17  *
  18  * Copyright (C) 2010 Texas Instruments Incorporated - http://www.ti.com/
  19  * Converted DMA library into DMA platform driver.
  20  *      - G, Manjunath Kondaiah <manjugk@ti.com>
  21  *
  22  * This program is free software; you can redistribute it and/or modify
  23  * it under the terms of the GNU General Public License version 2 as
  24  * published by the Free Software Foundation.
  25  *
  26  */
  27
  28 #include <linux/module.h>
  29 #include <linux/init.h>
  30 #include <linux/sched.h>
  31 #include <linux/spinlock.h>
  32 #include <linux/errno.h>
  33 #include <linux/interrupt.h>
  34 #include <linux/irq.h>
  35 #include <linux/io.h>
  36 #include <linux/slab.h>
  37 #include <linux/delay.h>
  38
  39 #include <asm/system.h>
  40 #include <mach/hardware.h>
  41 #include <plat/dma.h>
  42
  43 #include <plat/tc.h>
  44
  45 #undef DEBUG
  46
  47 #ifndef CONFIG_ARCH_OMAP1
  48 enum { DMA_CH_ALLOC_DONE, DMA_CH_PARAMS_SET_DONE, DMA_CH_STARTED,
  49         DMA_CH_QUEUED, DMA_CH_NOTSTARTED, DMA_CH_PAUSED, DMA_CH_LINK_ENABLED
  50 };
  51
  52 enum { DMA_CHAIN_STARTED, DMA_CHAIN_NOTSTARTED };
  53 #endif
  54
  55 #define OMAP_DMA_ACTIVE                 0x01
  56 #define OMAP2_DMA_CSR_CLEAR_MASK        0xffffffff
  57
  58 #define OMAP_FUNC_MUX_ARM_BASE          (0xfffe1000 + 0xec)
  59
  60 static struct omap_system_dma_plat_info *p;
  61 static struct omap_dma_dev_attr *d;
  62
  63 static int enable_1510_mode;
  64 static u32 errata;
  65
  66 static struct omap_dma_global_context_registers {
  67         u32 dma_irqenable_l0;
  68         u32 dma_ocp_sysconfig;
  69         u32 dma_gcr;
  70 } omap_dma_global_context;
  71
  72 struct dma_link_info {
  73         int *linked_dmach_q;
  74         int no_of_lchs_linked;
  75
  76         int q_count;
  77         int q_tail;
  78         int q_head;
  79
  80         int chain_state;
  81         int chain_mode;
  82
  83 };
  84
  85 static struct dma_link_info *dma_linked_lch;
  86
  87 #ifndef CONFIG_ARCH_OMAP1
  88
  89 /* Chain handling macros */
  90 #define OMAP_DMA_CHAIN_QINIT(chain_id)                                  \
  91         do {                                                            \
  92                 dma_linked_lch[chain_id].q_head =                       \
  93                 dma_linked_lch[chain_id].q_tail =                       \
  94                 dma_linked_lch[chain_id].q_count = 0;                   \
  95         } while (0)
  96 #define OMAP_DMA_CHAIN_QFULL(chain_id)                                  \
  97                 (dma_linked_lch[chain_id].no_of_lchs_linked ==          \
  98                 dma_linked_lch[chain_id].q_count)
  99 #define OMAP_DMA_CHAIN_QLAST(chain_id)                                  \
 100         do {                                                            \
 101                 ((dma_linked_lch[chain_id].no_of_lchs_linked-1) ==      \
 102                 dma_linked_lch[chain_id].q_count)                       \
 103         } while (0)
 104 #define OMAP_DMA_CHAIN_QEMPTY(chain_id)                                 \
 105                 (0 == dma_linked_lch[chain_id].q_count)
 106 #define __OMAP_DMA_CHAIN_INCQ(end)                                      \
 107         ((end) = ((end)+1) % dma_linked_lch[chain_id].no_of_lchs_linked)
 108 #define OMAP_DMA_CHAIN_INCQHEAD(chain_id)                               \
 109         do {                                                            \
 110                 __OMAP_DMA_CHAIN_INCQ(dma_linked_lch[chain_id].q_head); \
 111                 dma_linked_lch[chain_id].q_count--;                     \
 112         } while (0)
 113
 114 #define OMAP_DMA_CHAIN_INCQTAIL(chain_id)                               \
 115         do {                                                            \
 116                 __OMAP_DMA_CHAIN_INCQ(dma_linked_lch[chain_id].q_tail); \
 117                 dma_linked_lch[chain_id].q_count++; \
 118         } while (0)
 119 #endif
 120
 121 static int dma_lch_count;
 122 static int dma_chan_count;
 123 static int omap_dma_reserve_channels;
 124
 125 static spinlock_t dma_chan_lock;
 126 static struct omap_dma_lch *dma_chan;
 127
 128 static inline void disable_lnk(int lch);
 129 static void omap_disable_channel_irq(int lch);
 130 static inline void omap_enable_channel_irq(int lch);
 131
 132 #define REVISIT_24XX()          printk(KERN_ERR "FIXME: no %s on 24xx\n", \
 133                                                 __func__);
 134
 135 #ifdef CONFIG_ARCH_OMAP15XX
 136 /* Returns 1 if the DMA module is in OMAP1510-compatible mode, 0 otherwise */
 137 static int omap_dma_in_1510_mode(void)
 138 {
 139         return enable_1510_mode;
 140 }
 141 #else
 142 #define omap_dma_in_1510_mode()         0
 143 #endif
 144
 145 #ifdef CONFIG_ARCH_OMAP1
 146 static inline int get_gdma_dev(int req)
 147 {
 148         u32 reg = OMAP_FUNC_MUX_ARM_BASE + ((req - 1) / 5) * 4;
 149         int shift = ((req - 1) % 5) * 6;
 150
 151         return ((omap_readl(reg) >> shift) & 0x3f) + 1;
 152 }
 153
 154 static inline void set_gdma_dev(int req, int dev)
 155 {
 156         u32 reg = OMAP_FUNC_MUX_ARM_BASE + ((req - 1) / 5) * 4;
 157         int shift = ((req - 1) % 5) * 6;
 158         u32 l;
 159
 160         l = omap_readl(reg);
 161         l &= ~(0x3f << shift);
 162         l |= (dev - 1) << shift;
 163         omap_writel(l, reg);
 164 }
 165 #else
 166 #define set_gdma_dev(req, dev)  do {} while (0)
 167 #endif
 168
 169 void omap_set_dma_priority(int lch, int dst_port, int priority)
 170 {
 171         unsigned long reg;
 172         u32 l;
 173
 174         if (cpu_class_is_omap1()) {
 175                 switch (dst_port) {
 176                 case OMAP_DMA_PORT_OCP_T1:      /* FFFECC00 */
 177                         reg = OMAP_TC_OCPT1_PRIOR;
 178                         break;
 179                 case OMAP_DMA_PORT_OCP_T2:      /* FFFECCD0 */
 180                         reg = OMAP_TC_OCPT2_PRIOR;
 181                         break;
 182                 case OMAP_DMA_PORT_EMIFF:       /* FFFECC08 */
 183                         reg = OMAP_TC_EMIFF_PRIOR;
 184                         break;
 185                 case OMAP_DMA_PORT_EMIFS:       /* FFFECC04 */
 186                         reg = OMAP_TC_EMIFS_PRIOR;
 187                         break;
 188                 default:
 189                         BUG();
 190                         return;
 191                 }
 192                 l = omap_readl(reg);
 193                 l &= ~(0xf << 8);
 194                 l |= (priority & 0xf) << 8;
 195                 omap_writel(l, reg);
 196         }
 197
 198         if (cpu_class_is_omap2()) {
 199                 u32 ccr;
 200
 201                 ccr = p->dma_read(CCR, lch);
 202                 if (priority)
 203                         ccr |= (1 << 6);
 204                 else
 205                         ccr &= ~(1 << 6);
 206                 p->dma_write(ccr, CCR, lch);
 207         }
 208 }
 209 EXPORT_SYMBOL(omap_set_dma_priority);
 210
 211 void omap_set_dma_transfer_params(int lch, int data_type, int elem_count,
 212                                   int frame_count, int sync_mode,
 213                                   int dma_trigger, int src_or_dst_synch)
 214 {
 215         u32 l;
 216
 217         l = p->dma_read(CSDP, lch);
 218         l &= ~0x03;
 219         l |= data_type;
 220         p->dma_write(l, CSDP, lch);
 221
 222         if (cpu_class_is_omap1()) {
 223                 u16 ccr;
 224
 225                 ccr = p->dma_read(CCR, lch);
 226                 ccr &= ~(1 << 5);
 227                 if (sync_mode == OMAP_DMA_SYNC_FRAME)
 228                         ccr |= 1 << 5;
 229                 p->dma_write(ccr, CCR, lch);
 230
 231                 ccr = p->dma_read(CCR2, lch);
 232                 ccr &= ~(1 << 2);
 233                 if (sync_mode == OMAP_DMA_SYNC_BLOCK)
 234                         ccr |= 1 << 2;
 235                 p->dma_write(ccr, CCR2, lch);
 236         }
 237
 238         if (cpu_class_is_omap2() && dma_trigger) {
 239                 u32 val;
 240
 241                 val = p->dma_read(CCR, lch);
 242
 243                 /* DMA_SYNCHRO_CONTROL_UPPER depends on the channel number */
 244                 val &= ~((1 << 23) | (3 << 19) | 0x1f);
 245                 val |= (dma_trigger & ~0x1f) << 14;
 246                 val |= dma_trigger & 0x1f;
 247
 248                 if (sync_mode & OMAP_DMA_SYNC_FRAME)
 249                         val |= 1 << 5;
 250                 else
 251                         val &= ~(1 << 5);
 252
 253                 if (sync_mode & OMAP_DMA_SYNC_BLOCK)
 254                         val |= 1 << 18;
 255                 else
 256                         val &= ~(1 << 18);
 257
 258                 if (src_or_dst_synch == OMAP_DMA_DST_SYNC_PREFETCH) {
 259                         val &= ~(1 << 24);      /* dest synch */
 260                         val |= (1 << 23);       /* Prefetch */
 261                 } else if (src_or_dst_synch) {
 262                         val |= 1 << 24;         /* source synch */
 263                 } else {
 264                         val &= ~(1 << 24);      /* dest synch */
 265                 }
 266                 p->dma_write(val, CCR, lch);
 267         }
 268
 269         p->dma_write(elem_count, CEN, lch);
 270         p->dma_write(frame_count, CFN, lch);
 271 }
 272 EXPORT_SYMBOL(omap_set_dma_transfer_params);
 273
 274 void omap_set_dma_color_mode(int lch, enum omap_dma_color_mode mode, u32 color)
 275 {
 276         BUG_ON(omap_dma_in_1510_mode());
 277
 278         if (cpu_class_is_omap1()) {
 279                 u16 w;
 280
 281                 w = p->dma_read(CCR2, lch);
 282                 w &= ~0x03;
 283
 284                 switch (mode) {
 285                 case OMAP_DMA_CONSTANT_FILL:
 286                         w |= 0x01;
 287                         break;
 288                 case OMAP_DMA_TRANSPARENT_COPY:
 289                         w |= 0x02;
 290                         break;
 291                 case OMAP_DMA_COLOR_DIS:
 292                         break;
 293                 default:
 294                         BUG();
 295                 }
 296                 p->dma_write(w, CCR2, lch);
 297
 298                 w = p->dma_read(LCH_CTRL, lch);
 299                 w &= ~0x0f;
 300                 /* Default is channel type 2D */
 301                 if (mode) {
 302                         p->dma_write(color, COLOR, lch);
 303                         w |= 1;         /* Channel type G */
 304                 }
 305                 p->dma_write(w, LCH_CTRL, lch);
 306         }
 307
 308         if (cpu_class_is_omap2()) {
 309                 u32 val;
 310
 311                 val = p->dma_read(CCR, lch);
 312                 val &= ~((1 << 17) | (1 << 16));
 313
 314                 switch (mode) {
 315                 case OMAP_DMA_CONSTANT_FILL:
 316                         val |= 1 << 16;
 317                         break;
 318                 case OMAP_DMA_TRANSPARENT_COPY:
 319                         val |= 1 << 17;
 320                         break;
 321                 case OMAP_DMA_COLOR_DIS:
 322                         break;
 323                 default:
 324                         BUG();
 325                 }
 326                 p->dma_write(val, CCR, lch);
 327
 328                 color &= 0xffffff;
 329                 p->dma_write(color, COLOR, lch);
 330         }
 331 }
 332 EXPORT_SYMBOL(omap_set_dma_color_mode);
 333
 334 void omap_set_dma_write_mode(int lch, enum omap_dma_write_mode mode)
 335 {
 336         if (cpu_class_is_omap2()) {
 337                 u32 csdp;
 338
 339                 csdp = p->dma_read(CSDP, lch);
 340                 csdp &= ~(0x3 << 16);
 341                 csdp |= (mode << 16);
 342                 p->dma_write(csdp, CSDP, lch);
 343         }
 344 }
 345 EXPORT_SYMBOL(omap_set_dma_write_mode);
 346
 347 void omap_set_dma_channel_mode(int lch, enum omap_dma_channel_mode mode)
 348 {
 349         if (cpu_class_is_omap1() && !cpu_is_omap15xx()) {
 350                 u32 l;
 351
 352                 l = p->dma_read(LCH_CTRL, lch);
 353                 l &= ~0x7;
 354                 l |= mode;
 355                 p->dma_write(l, LCH_CTRL, lch);
 356         }
 357 }
 358 EXPORT_SYMBOL(omap_set_dma_channel_mode);
 359
 360 /* Note that src_port is only for omap1 */
 361 void omap_set_dma_src_params(int lch, int src_port, int src_amode,
 362                              unsigned long src_start,
 363                              int src_ei, int src_fi)
 364 {
 365         u32 l;
 366
 367         if (cpu_class_is_omap1()) {
 368                 u16 w;
 369
 370                 w = p->dma_read(CSDP, lch);
 371                 w &= ~(0x1f << 2);
 372                 w |= src_port << 2;
 373                 p->dma_write(w, CSDP, lch);
 374         }
 375
 376         l = p->dma_read(CCR, lch);
 377         l &= ~(0x03 << 12);
 378         l |= src_amode << 12;
 379         p->dma_write(l, CCR, lch);
 380
 381         p->dma_write(src_start, CSSA, lch);
 382
 383         p->dma_write(src_ei, CSEI, lch);
 384         p->dma_write(src_fi, CSFI, lch);
 385 }
 386 EXPORT_SYMBOL(omap_set_dma_src_params);
 387
 388 void omap_set_dma_params(int lch, struct omap_dma_channel_params *params)
 389 {
 390         omap_set_dma_transfer_params(lch, params->data_type,
 391                                      params->elem_count, params->frame_count,
 392                                      params->sync_mode, params->trigger,
 393                                      params->src_or_dst_synch);
 394         omap_set_dma_src_params(lch, params->src_port,
 395                                 params->src_amode, params->src_start,
 396                                 params->src_ei, params->src_fi);
 397
 398         omap_set_dma_dest_params(lch, params->dst_port,
 399                                  params->dst_amode, params->dst_start,
 400                                  params->dst_ei, params->dst_fi);
 401         if (params->read_prio || params->write_prio)
 402                 omap_dma_set_prio_lch(lch, params->read_prio,
 403                                       params->write_prio);
 404 }
 405 EXPORT_SYMBOL(omap_set_dma_params);
 406
 407 void omap_set_dma_src_index(int lch, int eidx, int fidx)
 408 {
 409         if (cpu_class_is_omap2())
 410                 return;
 411
 412         p->dma_write(eidx, CSEI, lch);
 413         p->dma_write(fidx, CSFI, lch);
 414 }
 415 EXPORT_SYMBOL(omap_set_dma_src_index);
 416
 417 void omap_set_dma_src_data_pack(int lch, int enable)
 418 {
 419         u32 l;
 420
 421         l = p->dma_read(CSDP, lch);
 422         l &= ~(1 << 6);
 423         if (enable)
 424                 l |= (1 << 6);
 425         p->dma_write(l, CSDP, lch);
 426 }
 427 EXPORT_SYMBOL(omap_set_dma_src_data_pack);
 428
 429 void omap_set_dma_src_burst_mode(int lch, enum omap_dma_burst_mode burst_mode)
 430 {
 431         unsigned int burst = 0;
 432         u32 l;
 433
 434         l = p->dma_read(CSDP, lch);
 435         l &= ~(0x03 << 7);
 436
 437         switch (burst_mode) {
 438         case OMAP_DMA_DATA_BURST_DIS:
 439                 break;
 440         case OMAP_DMA_DATA_BURST_4:
 441                 if (cpu_class_is_omap2())
 442                         burst = 0x1;
 443                 else
 444                         burst = 0x2;
 445                 break;
 446         case OMAP_DMA_DATA_BURST_8:
 447                 if (cpu_class_is_omap2()) {
 448                         burst = 0x2;
 449                         break;
 450                 }
 451                 /*
 452                  * not supported by current hardware on OMAP1
 453                  * w |= (0x03 << 7);
 454                  * fall through
 455                  */
 456         case OMAP_DMA_DATA_BURST_16:
 457                 if (cpu_class_is_omap2()) {
 458                         burst = 0x3;
 459                         break;
 460                 }
 461                 /*
 462                  * OMAP1 don't support burst 16
 463                  * fall through
 464                  */
 465         default:
 466                 BUG();
 467         }
 468
 469         l |= (burst << 7);
 470         p->dma_write(l, CSDP, lch);
 471 }
 472 EXPORT_SYMBOL(omap_set_dma_src_burst_mode);
 473
 474 /* Note that dest_port is only for OMAP1 */
 475 void omap_set_dma_dest_params(int lch, int dest_port, int dest_amode,
 476                               unsigned long dest_start,
 477                               int dst_ei, int dst_fi)
 478 {
 479         u32 l;
 480
 481         if (cpu_class_is_omap1()) {
 482                 l = p->dma_read(CSDP, lch);
 483                 l &= ~(0x1f << 9);
 484                 l |= dest_port << 9;
 485                 p->dma_write(l, CSDP, lch);
 486         }
 487
 488         l = p->dma_read(CCR, lch);
 489         l &= ~(0x03 << 14);
 490         l |= dest_amode << 14;
 491         p->dma_write(l, CCR, lch);
 492
 493         p->dma_write(dest_start, CDSA, lch);
 494
 495         p->dma_write(dst_ei, CDEI, lch);
 496         p->dma_write(dst_fi, CDFI, lch);
 497 }
 498 EXPORT_SYMBOL(omap_set_dma_dest_params);
 499
 500 void omap_set_dma_dest_index(int lch, int eidx, int fidx)
 501 {
 502         if (cpu_class_is_omap2())
 503                 return;
 504
 505         p->dma_write(eidx, CDEI, lch);
 506         p->dma_write(fidx, CDFI, lch);
 507 }
 508 EXPORT_SYMBOL(omap_set_dma_dest_index);
 509
 510 void omap_set_dma_dest_data_pack(int lch, int enable)
 511 {
 512         u32 l;
 513
 514         l = p->dma_read(CSDP, lch);
 515         l &= ~(1 << 13);
 516         if (enable)
 517                 l |= 1 << 13;
 518         p->dma_write(l, CSDP, lch);
 519 }
 520 EXPORT_SYMBOL(omap_set_dma_dest_data_pack);
 521
 522 void omap_set_dma_dest_burst_mode(int lch, enum omap_dma_burst_mode burst_mode)
 523 {
 524         unsigned int burst = 0;
 525         u32 l;
 526
 527         l = p->dma_read(CSDP, lch);
 528         l &= ~(0x03 << 14);
 529
 530         switch (burst_mode) {
 531         case OMAP_DMA_DATA_BURST_DIS:
 532                 break;
 533         case OMAP_DMA_DATA_BURST_4:
 534                 if (cpu_class_is_omap2())
 535                         burst = 0x1;
 536                 else
 537                         burst = 0x2;
 538                 break;
 539         case OMAP_DMA_DATA_BURST_8:
 540                 if (cpu_class_is_omap2())
 541                         burst = 0x2;
 542                 else
 543                         burst = 0x3;
 544                 break;
 545         case OMAP_DMA_DATA_BURST_16:
 546                 if (cpu_class_is_omap2()) {
 547                         burst = 0x3;
 548                         break;
 549                 }
 550                 /*
 551                  * OMAP1 don't support burst 16
 552                  * fall through
 553                  */
 554         default:
 555                 printk(KERN_ERR "Invalid DMA burst mode\n");
 556                 BUG();
 557                 return;
 558         }
 559         l |= (burst << 14);
 560         p->dma_write(l, CSDP, lch);
 561 }
 562 EXPORT_SYMBOL(omap_set_dma_dest_burst_mode);
 563
 564 static inline void omap_enable_channel_irq(int lch)
 565 {
 566         u32 status;
 567
 568         /* Clear CSR */
 569         if (cpu_class_is_omap1())
 570                 status = p->dma_read(CSR, lch);
 571         else if (cpu_class_is_omap2())
 572                 p->dma_write(OMAP2_DMA_CSR_CLEAR_MASK, CSR, lch);
 573
 574         /* Enable some nice interrupts. */
 575         p->dma_write(dma_chan[lch].enabled_irqs, CICR, lch);
 576 }
 577
 578 static void omap_disable_channel_irq(int lch)
 579 {
 580         if (cpu_class_is_omap2())
 581                 p->dma_write(0, CICR, lch);
 582 }
 583
 584 void omap_enable_dma_irq(int lch, u16 bits)
 585 {
 586         dma_chan[lch].enabled_irqs |= bits;
 587 }
 588 EXPORT_SYMBOL(omap_enable_dma_irq);
 589
 590 void omap_disable_dma_irq(int lch, u16 bits)
 591 {
 592         dma_chan[lch].enabled_irqs &= ~bits;
 593 }
 594 EXPORT_SYMBOL(omap_disable_dma_irq);
 595
 596 static inline void enable_lnk(int lch)
 597 {
 598         u32 l;
 599
 600         l = p->dma_read(CLNK_CTRL, lch);
 601
 602         if (cpu_class_is_omap1())
 603                 l &= ~(1 << 14);
 604
 605         /* Set the ENABLE_LNK bits */
 606         if (dma_chan[lch].next_lch != -1)
 607                 l = dma_chan[lch].next_lch | (1 << 15);
 608
 609 #ifndef CONFIG_ARCH_OMAP1
 610         if (cpu_class_is_omap2())
 611                 if (dma_chan[lch].next_linked_ch != -1)
 612                         l = dma_chan[lch].next_linked_ch | (1 << 15);
 613 #endif
 614
 615         p->dma_write(l, CLNK_CTRL, lch);
 616 }
 617
 618 static inline void disable_lnk(int lch)
 619 {
 620         u32 l;
 621
 622         l = p->dma_read(CLNK_CTRL, lch);
 623
 624         /* Disable interrupts */
 625         if (cpu_class_is_omap1()) {
 626                 p->dma_write(0, CICR, lch);
 627                 /* Set the STOP_LNK bit */
 628                 l |= 1 << 14;
 629         }
 630
 631         if (cpu_class_is_omap2()) {
 632                 omap_disable_channel_irq(lch);
 633                 /* Clear the ENABLE_LNK bit */
 634                 l &= ~(1 << 15);
 635         }
 636
 637         p->dma_write(l, CLNK_CTRL, lch);
 638         dma_chan[lch].flags &= ~OMAP_DMA_ACTIVE;
 639 }
 640
 641 static inline void omap2_enable_irq_lch(int lch)
 642 {
 643         u32 val;
 644         unsigned long flags;
 645
 646         if (!cpu_class_is_omap2())
 647                 return;
 648
 649         spin_lock_irqsave(&dma_chan_lock, flags);
 650         val = p->dma_read(IRQENABLE_L0, lch);
 651         val |= 1 << lch;
 652         p->dma_write(val, IRQENABLE_L0, lch);
 653         spin_unlock_irqrestore(&dma_chan_lock, flags);
 654 }
 655
 656 static inline void omap2_disable_irq_lch(int lch)
 657 {
 658         u32 val;
 659         unsigned long flags;
 660
 661         if (!cpu_class_is_omap2())
 662                 return;
 663
 664         spin_lock_irqsave(&dma_chan_lock, flags);
 665         val = p->dma_read(IRQENABLE_L0, lch);
 666         val &= ~(1 << lch);
 667         p->dma_write(val, IRQENABLE_L0, lch);
 668         spin_unlock_irqrestore(&dma_chan_lock, flags);
 669 }
 670
 671 int omap_request_dma(int dev_id, const char *dev_name,
 672                      void (*callback)(int lch, u16 ch_status, void *data),
 673                      void *data, int *dma_ch_out)
 674 {
 675         int ch, free_ch = -1;
 676         unsigned long flags;
 677         struct omap_dma_lch *chan;
 678
 679         spin_lock_irqsave(&dma_chan_lock, flags);
 680         for (ch = 0; ch < dma_chan_count; ch++) {
 681                 if (free_ch == -1 && dma_chan[ch].dev_id == -1) {
 682                         free_ch = ch;
 683                         if (dev_id == 0)
 684                                 break;
 685                 }
 686         }
 687         if (free_ch == -1) {
 688                 spin_unlock_irqrestore(&dma_chan_lock, flags);
 689                 return -EBUSY;
 690         }
 691         chan = dma_chan + free_ch;
 692         chan->dev_id = dev_id;
 693
 694         if (p->clear_lch_regs)
 695                 p->clear_lch_regs(free_ch);
 696
 697         if (cpu_class_is_omap2())
 698                 omap_clear_dma(free_ch);
 699
 700         spin_unlock_irqrestore(&dma_chan_lock, flags);
 701
 702         chan->dev_name = dev_name;
 703         chan->callback = callback;
 704         chan->data = data;
 705         chan->flags = 0;
 706
 707 #ifndef CONFIG_ARCH_OMAP1
 708         if (cpu_class_is_omap2()) {
 709                 chan->chain_id = -1;
 710                 chan->next_linked_ch = -1;
 711         }
 712 #endif
 713
 714         chan->enabled_irqs = OMAP_DMA_DROP_IRQ | OMAP_DMA_BLOCK_IRQ;
 715
 716         if (cpu_class_is_omap1())
 717                 chan->enabled_irqs |= OMAP1_DMA_TOUT_IRQ;
 718         else if (cpu_class_is_omap2())
 719                 chan->enabled_irqs |= OMAP2_DMA_MISALIGNED_ERR_IRQ |
 720                         OMAP2_DMA_TRANS_ERR_IRQ;
 721
 722         if (cpu_is_omap16xx()) {
 723                 /* If the sync device is set, configure it dynamically. */
 724                 if (dev_id != 0) {
 725                         set_gdma_dev(free_ch + 1, dev_id);
 726                         dev_id = free_ch + 1;
 727                 }
 728                 /*
 729                  * Disable the 1510 compatibility mode and set the sync device
 730                  * id.
 731                  */
 732                 p->dma_write(dev_id | (1 << 10), CCR, free_ch);
 733         } else if (cpu_is_omap7xx() || cpu_is_omap15xx()) {
 734                 p->dma_write(dev_id, CCR, free_ch);
 735         }
 736
 737         if (cpu_class_is_omap2()) {
 738                 omap2_enable_irq_lch(free_ch);
 739                 omap_enable_channel_irq(free_ch);
 740                 /* Clear the CSR register and IRQ status register */
 741                 p->dma_write(OMAP2_DMA_CSR_CLEAR_MASK, CSR, free_ch);
 742                 p->dma_write(1 << free_ch, IRQSTATUS_L0, 0);
 743         }
 744
 745         *dma_ch_out = free_ch;
 746
 747         return 0;
 748 }
 749 EXPORT_SYMBOL(omap_request_dma);
 750
 751 void omap_free_dma(int lch)
 752 {
 753         unsigned long flags;
 754
 755         if (dma_chan[lch].dev_id == -1) {
 756                 pr_err("omap_dma: trying to free unallocated DMA channel %d\n",
 757                        lch);
 758                 return;
 759         }
 760
 761         if (cpu_class_is_omap1()) {
 762                 /* Disable all DMA interrupts for the channel. */
 763                 p->dma_write(0, CICR, lch);
 764                 /* Make sure the DMA transfer is stopped. */
 765                 p->dma_write(0, CCR, lch);
 766         }
 767
 768         if (cpu_class_is_omap2()) {
 769                 omap2_disable_irq_lch(lch);
 770
 771                 /* Clear the CSR register and IRQ status register */
 772                 p->dma_write(OMAP2_DMA_CSR_CLEAR_MASK, CSR, lch);
 773                 p->dma_write(1 << lch, IRQSTATUS_L0, lch);
 774
 775                 /* Disable all DMA interrupts for the channel. */
 776                 p->dma_write(0, CICR, lch);
 777
 778                 /* Make sure the DMA transfer is stopped. */
 779                 p->dma_write(0, CCR, lch);
 780                 omap_clear_dma(lch);
 781         }
 782
 783         spin_lock_irqsave(&dma_chan_lock, flags);
 784         dma_chan[lch].dev_id = -1;
 785         dma_chan[lch].next_lch = -1;
 786         dma_chan[lch].callback = NULL;
 787         spin_unlock_irqrestore(&dma_chan_lock, flags);
 788 }
 789 EXPORT_SYMBOL(omap_free_dma);
 790
 791 /**
 792  * @brief omap_dma_set_global_params : Set global priority settings for dma
 793  *
 794  * @param arb_rate
 795  * @param max_fifo_depth
 796  * @param tparams - Number of threads to reserve : DMA_THREAD_RESERVE_NORM
 797  *                                                 DMA_THREAD_RESERVE_ONET
 798  *                                                 DMA_THREAD_RESERVE_TWOT
 799  *                                                 DMA_THREAD_RESERVE_THREET
 800  */
 801 void
 802 omap_dma_set_global_params(int arb_rate, int max_fifo_depth, int tparams)
 803 {
 804         u32 reg;
 805
 806         if (!cpu_class_is_omap2()) {
 807                 printk(KERN_ERR "FIXME: no %s on 15xx/16xx\n", __func__);
 808                 return;
 809         }
 810
 811         if (max_fifo_depth == 0)
 812                 max_fifo_depth = 1;
 813         if (arb_rate == 0)
 814                 arb_rate = 1;
 815
 816         reg = 0xff & max_fifo_depth;
 817         reg |= (0x3 & tparams) << 12;
 818         reg |= (arb_rate & 0xff) << 16;
 819
 820         p->dma_write(reg, GCR, 0);
 821 }
 822 EXPORT_SYMBOL(omap_dma_set_global_params);
 823
 824 /**
 825  * @brief omap_dma_set_prio_lch : Set channel wise priority settings
 826  *
 827  * @param lch
 828  * @param read_prio - Read priority
 829  * @param write_prio - Write priority
 830  * Both of the above can be set with one of the following values :
 831  *      DMA_CH_PRIO_HIGH/DMA_CH_PRIO_LOW
 832  */
 833 int
 834 omap_dma_set_prio_lch(int lch, unsigned char read_prio,
 835                       unsigned char write_prio)
 836 {
 837         u32 l;
 838
 839         if (unlikely((lch < 0 || lch >= dma_lch_count))) {
 840                 printk(KERN_ERR "Invalid channel id\n");
 841                 return -EINVAL;
 842         }
 843         l = p->dma_read(CCR, lch);
 844         l &= ~((1 << 6) | (1 << 26));
 845         if (cpu_is_omap2430() || cpu_is_omap34xx() ||  cpu_is_omap44xx())
 846                 l |= ((read_prio & 0x1) << 6) | ((write_prio & 0x1) << 26);
 847         else
 848                 l |= ((read_prio & 0x1) << 6);
 849
 850         p->dma_write(l, CCR, lch);
 851
 852         return 0;
 853 }
 854 EXPORT_SYMBOL(omap_dma_set_prio_lch);
 855
 856 /*
 857  * Clears any DMA state so the DMA engine is ready to restart with new buffers
 858  * through omap_start_dma(). Any buffers in flight are discarded.
 859  */
 860 void omap_clear_dma(int lch)
 861 {
 862         unsigned long flags;
 863
 864         local_irq_save(flags);
 865         p->clear_dma(lch);
 866         local_irq_restore(flags);
 867 }
 868 EXPORT_SYMBOL(omap_clear_dma);
 869
 870 void omap_start_dma(int lch)
 871 {
 872         u32 l;
 873
 874         /*
 875          * The CPC/CDAC register needs to be initialized to zero
 876          * before starting dma transfer.
 877          */
 878         if (cpu_is_omap15xx())
 879                 p->dma_write(0, CPC, lch);
 880         else
 881                 p->dma_write(0, CDAC, lch);
 882
 883         if (!omap_dma_in_1510_mode() && dma_chan[lch].next_lch != -1) {
 884                 int next_lch, cur_lch;
 885                 char dma_chan_link_map[dma_lch_count];
 886
 887                 dma_chan_link_map[lch] = 1;
 888                 /* Set the link register of the first channel */
 889                 enable_lnk(lch);
 890
 891                 memset(dma_chan_link_map, 0, sizeof(dma_chan_link_map));
 892                 cur_lch = dma_chan[lch].next_lch;
 893                 do {
 894                         next_lch = dma_chan[cur_lch].next_lch;
 895
 896                         /* The loop case: we've been here already */
 897                         if (dma_chan_link_map[cur_lch])
 898                                 break;
 899                         /* Mark the current channel */
 900                         dma_chan_link_map[cur_lch] = 1;
 901
 902                         enable_lnk(cur_lch);
 903                         omap_enable_channel_irq(cur_lch);
 904
 905                         cur_lch = next_lch;
 906                 } while (next_lch != -1);
 907         } else if (IS_DMA_ERRATA(DMA_ERRATA_PARALLEL_CHANNELS))
 908                 p->dma_write(lch, CLNK_CTRL, lch);
 909
 910         omap_enable_channel_irq(lch);
 911
 912         l = p->dma_read(CCR, lch);
 913
 914         if (IS_DMA_ERRATA(DMA_ERRATA_IFRAME_BUFFERING))
 915                         l |= OMAP_DMA_CCR_BUFFERING_DISABLE;
 916         l |= OMAP_DMA_CCR_EN;
 917
 918         /*
 919          * As dma_write() uses IO accessors which are weakly ordered, there
 920          * is no guarantee that data in coherent DMA memory will be visible
 921          * to the DMA device.  Add a memory barrier here to ensure that any
 922          * such data is visible prior to enabling DMA.
 923          */
 924         mb();
 925         p->dma_write(l, CCR, lch);
 926
 927         dma_chan[lch].flags |= OMAP_DMA_ACTIVE;
 928 }
 929 EXPORT_SYMBOL(omap_start_dma);
 930
 931 void omap_stop_dma(int lch)
 932 {
 933         u32 l;
 934
 935         /* Disable all interrupts on the channel */
 936         if (cpu_class_is_omap1())
 937                 p->dma_write(0, CICR, lch);
 938
 939         l = p->dma_read(CCR, lch);
 940         if (IS_DMA_ERRATA(DMA_ERRATA_i541) &&
 941                         (l & OMAP_DMA_CCR_SEL_SRC_DST_SYNC)) {
 942                 int i = 0;
 943                 u32 sys_cf;
 944
 945                 /* Configure No-Standby */
 946                 l = p->dma_read(OCP_SYSCONFIG, lch);
 947                 sys_cf = l;
 948                 l &= ~DMA_SYSCONFIG_MIDLEMODE_MASK;
 949                 l |= DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_NO_IDLE);
 950                 p->dma_write(l , OCP_SYSCONFIG, 0);
 951
 952                 l = p->dma_read(CCR, lch);
 953                 l &= ~OMAP_DMA_CCR_EN;
 954                 p->dma_write(l, CCR, lch);
 955
 956                 /* Wait for sDMA FIFO drain */
 957                 l = p->dma_read(CCR, lch);
 958                 while (i < 100 && (l & (OMAP_DMA_CCR_RD_ACTIVE |
 959                                         OMAP_DMA_CCR_WR_ACTIVE))) {
 960                         udelay(5);
 961                         i++;
 962                         l = p->dma_read(CCR, lch);
 963                 }
 964                 if (i >= 100)
 965                         printk(KERN_ERR "DMA drain did not complete on "
 966                                         "lch %d\n", lch);
 967                 /* Restore OCP_SYSCONFIG */
 968                 p->dma_write(sys_cf, OCP_SYSCONFIG, lch);
 969         } else {
 970                 l &= ~OMAP_DMA_CCR_EN;
 971                 p->dma_write(l, CCR, lch);
 972         }
 973
 974         /*
 975          * Ensure that data transferred by DMA is visible to any access
 976          * after DMA has been disabled.  This is important for coherent
 977          * DMA regions.
 978          */
 979         mb();
 980
 981         if (!omap_dma_in_1510_mode() && dma_chan[lch].next_lch != -1) {
 982                 int next_lch, cur_lch = lch;
 983                 char dma_chan_link_map[dma_lch_count];
 984
 985                 memset(dma_chan_link_map, 0, sizeof(dma_chan_link_map));
 986                 do {
 987                         /* The loop case: we've been here already */
 988                         if (dma_chan_link_map[cur_lch])
 989                                 break;
 990                         /* Mark the current channel */
 991                         dma_chan_link_map[cur_lch] = 1;
 992
 993                         disable_lnk(cur_lch);
 994
 995                         next_lch = dma_chan[cur_lch].next_lch;
 996                         cur_lch = next_lch;
 997                 } while (next_lch != -1);
 998         }
 999
1000         dma_chan[lch].flags &= ~OMAP_DMA_ACTIVE;
1001 }
1002 EXPORT_SYMBOL(omap_stop_dma);
1003
1004 /*
1005  * Allows changing the DMA callback function or data. This may be needed if
1006  * the driver shares a single DMA channel for multiple dma triggers.
1007  */
1008 int omap_set_dma_callback(int lch,
1009                           void (*callback)(int lch, u16 ch_status, void *data),
1010                           void *data)
1011 {
1012         unsigned long flags;
1013
1014         if (lch < 0)
1015                 return -ENODEV;
1016
1017         spin_lock_irqsave(&dma_chan_lock, flags);
1018         if (dma_chan[lch].dev_id == -1) {
1019                 printk(KERN_ERR "DMA callback for not set for free channel\n");
1020                 spin_unlock_irqrestore(&dma_chan_lock, flags);
1021                 return -EINVAL;
1022         }
1023         dma_chan[lch].callback = callback;
1024         dma_chan[lch].data = data;
1025         spin_unlock_irqrestore(&dma_chan_lock, flags);
1026
1027         return 0;
1028 }
1029 EXPORT_SYMBOL(omap_set_dma_callback);
1030
1031 /*
1032  * Returns current physical source address for the given DMA channel.
1033  * If the channel is running the caller must disable interrupts prior calling
1034  * this function and process the returned value before re-enabling interrupt to
1035  * prevent races with the interrupt handler. Note that in continuous mode there
1036  * is a chance for CSSA_L register overflow between the two reads resulting
1037  * in incorrect return value.
1038  */
1039 dma_addr_t omap_get_dma_src_pos(int lch)
1040 {
1041         dma_addr_t offset = 0;
1042
1043         if (cpu_is_omap15xx())
1044                 offset = p->dma_read(CPC, lch);
1045         else
1046                 offset = p->dma_read(CSAC, lch);
1047
1048         if (IS_DMA_ERRATA(DMA_ERRATA_3_3) && offset == 0)
1049                 offset = p->dma_read(CSAC, lch);
1050
1051         if (!cpu_is_omap15xx()) {
1052                 /*
1053                  * CDAC == 0 indicates that the DMA transfer on the channel has
1054                  * not been started (no data has been transferred so far).
1055                  * Return the programmed source start address in this case.
1056                  */
1057                 if (likely(p->dma_read(CDAC, lch)))
1058                         offset = p->dma_read(CSAC, lch);
1059                 else
1060                         offset = p->dma_read(CSSA, lch);
1061         }
1062
1063         if (cpu_class_is_omap1())
1064                 offset |= (p->dma_read(CSSA, lch) & 0xFFFF0000);
1065
1066         return offset;
1067 }
1068 EXPORT_SYMBOL(omap_get_dma_src_pos);
1069
1070 /*
1071  * Returns current physical destination address for the given DMA channel.
1072  * If the channel is running the caller must disable interrupts prior calling
1073  * this function and process the returned value before re-enabling interrupt to
1074  * prevent races with the interrupt handler. Note that in continuous mode there
1075  * is a chance for CDSA_L register overflow between the two reads resulting
1076  * in incorrect return value.
1077  */
1078 dma_addr_t omap_get_dma_dst_pos(int lch)
1079 {
1080         dma_addr_t offset = 0;
1081
1082         if (cpu_is_omap15xx())
1083                 offset = p->dma_read(CPC, lch);
1084         else
1085                 offset = p->dma_read(CDAC, lch);
1086
1087         /*
1088          * omap 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is
1089          * read before the DMA controller finished disabling the channel.
1090          */
1091         if (!cpu_is_omap15xx() && offset == 0) {
1092                 offset = p->dma_read(CDAC, lch);
1093                 /*
1094                  * CDAC == 0 indicates that the DMA transfer on the channel has
1095                  * not been started (no data has been transferred so far).
1096                  * Return the programmed destination start address in this case.
1097                  */
1098                 if (unlikely(!offset))
1099                         offset = p->dma_read(CDSA, lch);
1100         }
1101
1102         if (cpu_class_is_omap1())
1103                 offset |= (p->dma_read(CDSA, lch) & 0xFFFF0000);
1104
1105         return offset;
1106 }
1107 EXPORT_SYMBOL(omap_get_dma_dst_pos);
1108
1109 int omap_get_dma_active_status(int lch)
1110 {
1111         return (p->dma_read(CCR, lch) & OMAP_DMA_CCR_EN) != 0;
1112 }
1113 EXPORT_SYMBOL(omap_get_dma_active_status);
1114
1115 int omap_dma_running(void)
1116 {
1117         int lch;
1118
1119         if (cpu_class_is_omap1())
1120                 if (omap_lcd_dma_running())
1121                         return 1;
1122
1123         for (lch = 0; lch < dma_chan_count; lch++)
1124                 if (p->dma_read(CCR, lch) & OMAP_DMA_CCR_EN)
1125                         return 1;
1126
1127         return 0;
1128 }
1129
1130 /*
1131  * lch_queue DMA will start right after lch_head one is finished.
1132  * For this DMA link to start, you still need to start (see omap_start_dma)
1133  * the first one. That will fire up the entire queue.
1134  */
1135 void omap_dma_link_lch(int lch_head, int lch_queue)
1136 {
1137         if (omap_dma_in_1510_mode()) {
1138                 if (lch_head == lch_queue) {
1139                         p->dma_write(p->dma_read(CCR, lch_head) | (3 << 8),
1140                                                                 CCR, lch_head);
1141                         return;
1142                 }
1143                 printk(KERN_ERR "DMA linking is not supported in 1510 mode\n");
1144                 BUG();
1145                 return;
1146         }
1147
1148         if ((dma_chan[lch_head].dev_id == -1) ||
1149             (dma_chan[lch_queue].dev_id == -1)) {
1150                 printk(KERN_ERR "omap_dma: trying to link "
1151                        "non requested channels\n");
1152                 dump_stack();
1153         }
1154
1155         dma_chan[lch_head].next_lch = lch_queue;
1156 }
1157 EXPORT_SYMBOL(omap_dma_link_lch);
1158
1159 /*
1160  * Once the DMA queue is stopped, we can destroy it.
1161  */
1162 void omap_dma_unlink_lch(int lch_head, int lch_queue)
1163 {
1164         if (omap_dma_in_1510_mode()) {
1165                 if (lch_head == lch_queue) {
1166                         p->dma_write(p->dma_read(CCR, lch_head) & ~(3 << 8),
1167                                                                 CCR, lch_head);
1168                         return;
1169                 }
1170                 printk(KERN_ERR "DMA linking is not supported in 1510 mode\n");
1171                 BUG();
1172                 return;
1173         }
1174
1175         if (dma_chan[lch_head].next_lch != lch_queue ||
1176             dma_chan[lch_head].next_lch == -1) {
1177                 printk(KERN_ERR "omap_dma: trying to unlink "
1178                        "non linked channels\n");
1179                 dump_stack();
1180         }
1181
1182         if ((dma_chan[lch_head].flags & OMAP_DMA_ACTIVE) ||
1183             (dma_chan[lch_queue].flags & OMAP_DMA_ACTIVE)) {
1184                 printk(KERN_ERR "omap_dma: You need to stop the DMA channels "
1185                        "before unlinking\n");
1186                 dump_stack();
1187         }
1188
1189         dma_chan[lch_head].next_lch = -1;
1190 }
1191 EXPORT_SYMBOL(omap_dma_unlink_lch);
1192
1193 #ifndef CONFIG_ARCH_OMAP1
1194 /* Create chain of DMA channesls */
1195 static void create_dma_lch_chain(int lch_head, int lch_queue)
1196 {
1197         u32 l;
1198
1199         /* Check if this is the first link in chain */
1200         if (dma_chan[lch_head].next_linked_ch == -1) {
1201                 dma_chan[lch_head].next_linked_ch = lch_queue;
1202                 dma_chan[lch_head].prev_linked_ch = lch_queue;
1203                 dma_chan[lch_queue].next_linked_ch = lch_head;
1204                 dma_chan[lch_queue].prev_linked_ch = lch_head;
1205         }
1206
1207         /* a link exists, link the new channel in circular chain */
1208         else {
1209                 dma_chan[lch_queue].next_linked_ch =
1210                                         dma_chan[lch_head].next_linked_ch;
1211                 dma_chan[lch_queue].prev_linked_ch = lch_head;
1212                 dma_chan[lch_head].next_linked_ch = lch_queue;
1213                 dma_chan[dma_chan[lch_queue].next_linked_ch].prev_linked_ch =
1214                                         lch_queue;
1215         }
1216
1217         l = p->dma_read(CLNK_CTRL, lch_head);
1218         l &= ~(0x1f);
1219         l |= lch_queue;
1220         p->dma_write(l, CLNK_CTRL, lch_head);
1221
1222         l = p->dma_read(CLNK_CTRL, lch_queue);
1223         l &= ~(0x1f);
1224         l |= (dma_chan[lch_queue].next_linked_ch);
1225         p->dma_write(l, CLNK_CTRL, lch_queue);
1226 }
1227
1228 /**
1229  * @brief omap_request_dma_chain : Request a chain of DMA channels
1230  *
1231  * @param dev_id - Device id using the dma channel
1232  * @param dev_name - Device name
1233  * @param callback - Call back function
1234  * @chain_id -
1235  * @no_of_chans - Number of channels requested
1236  * @chain_mode - Dynamic or static chaining : OMAP_DMA_STATIC_CHAIN
1237  *                                            OMAP_DMA_DYNAMIC_CHAIN
1238  * @params - Channel parameters
1239  *
1240  * @return - Success : 0
1241  *           Failure: -EINVAL/-ENOMEM
1242  */
1243 int omap_request_dma_chain(int dev_id, const char *dev_name,
1244                            void (*callback) (int lch, u16 ch_status,
1245                                              void *data),
1246                            int *chain_id, int no_of_chans, int chain_mode,
1247                            struct omap_dma_channel_params params)
1248 {
1249         int *channels;
1250         int i, err;
1251
1252         /* Is the chain mode valid ? */
1253         if (chain_mode != OMAP_DMA_STATIC_CHAIN
1254                         && chain_mode != OMAP_DMA_DYNAMIC_CHAIN) {
1255                 printk(KERN_ERR "Invalid chain mode requested\n");
1256                 return -EINVAL;
1257         }
1258
1259         if (unlikely((no_of_chans < 1
1260                         || no_of_chans > dma_lch_count))) {
1261                 printk(KERN_ERR "Invalid Number of channels requested\n");
1262                 return -EINVAL;
1263         }
1264
1265         /*
1266          * Allocate a queue to maintain the status of the channels
1267          * in the chain
1268          */
1269         channels = kmalloc(sizeof(*channels) * no_of_chans, GFP_KERNEL);
1270         if (channels == NULL) {
1271                 printk(KERN_ERR "omap_dma: No memory for channel queue\n");
1272                 return -ENOMEM;
1273         }
1274
1275         /* request and reserve DMA channels for the chain */
1276         for (i = 0; i < no_of_chans; i++) {
1277                 err = omap_request_dma(dev_id, dev_name,
1278                                         callback, NULL, &channels[i]);
1279                 if (err < 0) {
1280                         int j;
1281                         for (j = 0; j < i; j++)
1282                                 omap_free_dma(channels[j]);
1283                         kfree(channels);
1284                         printk(KERN_ERR "omap_dma: Request failed %d\n", err);
1285                         return err;
1286                 }
1287                 dma_chan[channels[i]].prev_linked_ch = -1;
1288                 dma_chan[channels[i]].state = DMA_CH_NOTSTARTED;
1289
1290                 /*
1291                  * Allowing client drivers to set common parameters now,
1292                  * so that later only relevant (src_start, dest_start
1293                  * and element count) can be set
1294                  */
1295                 omap_set_dma_params(channels[i], &params);
1296         }
1297
1298         *chain_id = channels[0];
1299         dma_linked_lch[*chain_id].linked_dmach_q = channels;
1300         dma_linked_lch[*chain_id].chain_mode = chain_mode;
1301         dma_linked_lch[*chain_id].chain_state = DMA_CHAIN_NOTSTARTED;
1302         dma_linked_lch[*chain_id].no_of_lchs_linked = no_of_chans;
1303
1304         for (i = 0; i < no_of_chans; i++)
1305                 dma_chan[channels[i]].chain_id = *chain_id;
1306
1307         /* Reset the Queue pointers */
1308         OMAP_DMA_CHAIN_QINIT(*chain_id);
1309
1310         /* Set up the chain */
1311         if (no_of_chans == 1)
1312                 create_dma_lch_chain(channels[0], channels[0]);
1313         else {
1314                 for (i = 0; i < (no_of_chans - 1); i++)
1315                         create_dma_lch_chain(channels[i], channels[i + 1]);
1316         }
1317
1318         return 0;
1319 }
1320 EXPORT_SYMBOL(omap_request_dma_chain);
1321
1322 /**
1323  * @brief omap_modify_dma_chain_param : Modify the chain's params - Modify the
1324  * params after setting it. Dont do this while dma is running!!
1325  *
1326  * @param chain_id - Chained logical channel id.
1327  * @param params
1328  *
1329  * @return - Success : 0
1330  *           Failure : -EINVAL
1331  */
1332 int omap_modify_dma_chain_params(int chain_id,
1333                                 struct omap_dma_channel_params params)
1334 {
1335         int *channels;
1336         u32 i;
1337
1338         /* Check for input params */
1339         if (unlikely((chain_id < 0
1340                         || chain_id >= dma_lch_count))) {
1341                 printk(KERN_ERR "Invalid chain id\n");
1342                 return -EINVAL;
1343         }
1344
1345         /* Check if the chain exists */
1346         if (dma_linked_lch[chain_id].linked_dmach_q == NULL) {
1347                 printk(KERN_ERR "Chain doesn't exists\n");
1348                 return -EINVAL;
1349         }
1350         channels = dma_linked_lch[chain_id].linked_dmach_q;
1351
1352         for (i = 0; i < dma_linked_lch[chain_id].no_of_lchs_linked; i++) {
1353                 /*
1354                  * Allowing client drivers to set common parameters now,
1355                  * so that later only relevant (src_start, dest_start
1356                  * and element count) can be set
1357                  */
1358                 omap_set_dma_params(channels[i], &params);
1359         }
1360
1361         return 0;
1362 }
1363 EXPORT_SYMBOL(omap_modify_dma_chain_params);
1364
1365 /**
1366  * @brief omap_free_dma_chain - Free all the logical channels in a chain.
1367  *
1368  * @param chain_id
1369  *
1370  * @return - Success : 0
1371  *           Failure : -EINVAL
1372  */
1373 int omap_free_dma_chain(int chain_id)
1374 {
1375         int *channels;
1376         u32 i;
1377
1378         /* Check for input params */
1379         if (unlikely((chain_id < 0 || chain_id >= dma_lch_count))) {
1380                 printk(KERN_ERR "Invalid chain id\n");
1381                 return -EINVAL;
1382         }
1383
1384         /* Check if the chain exists */
1385         if (dma_linked_lch[chain_id].linked_dmach_q == NULL) {
1386                 printk(KERN_ERR "Chain doesn't exists\n");
1387                 return -EINVAL;
1388         }
1389
1390         channels = dma_linked_lch[chain_id].linked_dmach_q;
1391         for (i = 0; i < dma_linked_lch[chain_id].no_of_lchs_linked; i++) {
1392                 dma_chan[channels[i]].next_linked_ch = -1;
1393                 dma_chan[channels[i]].prev_linked_ch = -1;
1394                 dma_chan[channels[i]].chain_id = -1;
1395                 dma_chan[channels[i]].state = DMA_CH_NOTSTARTED;
1396                 omap_free_dma(channels[i]);
1397         }
1398
1399         kfree(channels);
1400
1401         dma_linked_lch[chain_id].linked_dmach_q = NULL;
1402         dma_linked_lch[chain_id].chain_mode = -1;
1403         dma_linked_lch[chain_id].chain_state = -1;
1404
1405         return (0);
1406 }
1407 EXPORT_SYMBOL(omap_free_dma_chain);
1408
1409 /**
1410  * @brief omap_dma_chain_status - Check if the chain is in
1411  * active / inactive state.
1412  * @param chain_id
1413  *
1414  * @return - Success : OMAP_DMA_CHAIN_ACTIVE/OMAP_DMA_CHAIN_INACTIVE
1415  *           Failure : -EINVAL
1416  */
1417 int omap_dma_chain_status(int chain_id)
1418 {
1419         /* Check for input params */
1420         if (unlikely((chain_id < 0 || chain_id >= dma_lch_count))) {
1421                 printk(KERN_ERR "Invalid chain id\n");
1422                 return -EINVAL;
1423         }
1424
1425         /* Check if the chain exists */
1426         if (dma_linked_lch[chain_id].linked_dmach_q == NULL) {
1427                 printk(KERN_ERR "Chain doesn't exists\n");
1428                 return -EINVAL;
1429         }
1430         pr_debug("CHAINID=%d, qcnt=%d\n", chain_id,
1431                         dma_linked_lch[chain_id].q_count);
1432
1433         if (OMAP_DMA_CHAIN_QEMPTY(chain_id))
1434                 return OMAP_DMA_CHAIN_INACTIVE;
1435
1436         return OMAP_DMA_CHAIN_ACTIVE;
1437 }
1438 EXPORT_SYMBOL(omap_dma_chain_status);
1439
1440 /**
1441  * @brief omap_dma_chain_a_transfer - Get a free channel from a chain,
1442  * set the params and start the transfer.
1443  *
1444  * @param chain_id
1445  * @param src_start - buffer start address
1446  * @param dest_start - Dest address
1447  * @param elem_count
1448  * @param frame_count
1449  * @param callbk_data - channel callback parameter data.
1450  *
1451  * @return  - Success : 0
1452  *            Failure: -EINVAL/-EBUSY
1453  */
1454 int omap_dma_chain_a_transfer(int chain_id, int src_start, int dest_start,
1455                         int elem_count, int frame_count, void *callbk_data)
1456 {
1457         int *channels;
1458         u32 l, lch;
1459         int start_dma = 0;
1460
1461         /*
1462          * if buffer size is less than 1 then there is
1463          * no use of starting the chain
1464          */
1465         if (elem_count < 1) {
1466                 printk(KERN_ERR "Invalid buffer size\n");
1467                 return -EINVAL;
1468         }
1469
1470         /* Check for input params */
1471         if (unlikely((chain_id < 0
1472                         || chain_id >= dma_lch_count))) {
1473                 printk(KERN_ERR "Invalid chain id\n");
1474                 return -EINVAL;
1475         }
1476
1477         /* Check if the chain exists */
1478         if (dma_linked_lch[chain_id].linked_dmach_q == NULL) {
1479                 printk(KERN_ERR "Chain doesn't exist\n");
1480                 return -EINVAL;
1481         }
1482
1483         /* Check if all the channels in chain are in use */
1484         if (OMAP_DMA_CHAIN_QFULL(chain_id))
1485                 return -EBUSY;
1486
1487         /* Frame count may be negative in case of indexed transfers */
1488         channels = dma_linked_lch[chain_id].linked_dmach_q;
1489
1490         /* Get a free channel */
1491         lch = channels[dma_linked_lch[chain_id].q_tail];
1492
1493         /* Store the callback data */
1494         dma_chan[lch].data = callbk_data;
1495
1496         /* Increment the q_tail */
1497         OMAP_DMA_CHAIN_INCQTAIL(chain_id);
1498
1499         /* Set the params to the free channel */
1500         if (src_start != 0)
1501                 p->dma_write(src_start, CSSA, lch);
1502         if (dest_start != 0)
1503                 p->dma_write(dest_start, CDSA, lch);
1504
1505         /* Write the buffer size */
1506         p->dma_write(elem_count, CEN, lch);
1507         p->dma_write(frame_count, CFN, lch);
1508
1509         /*
1510          * If the chain is dynamically linked,
1511          * then we may have to start the chain if its not active
1512          */
1513         if (dma_linked_lch[chain_id].chain_mode == OMAP_DMA_DYNAMIC_CHAIN) {
1514
1515                 /*
1516                  * In Dynamic chain, if the chain is not started,
1517                  * queue the channel
1518                  */
1519                 if (dma_linked_lch[chain_id].chain_state ==
1520                                                 DMA_CHAIN_NOTSTARTED) {
1521                         /* Enable the link in previous channel */
1522                         if (dma_chan[dma_chan[lch].prev_linked_ch].state ==
1523                                                                 DMA_CH_QUEUED)
1524                                 enable_lnk(dma_chan[lch].prev_linked_ch);
1525                         dma_chan[lch].state = DMA_CH_QUEUED;
1526                 }
1527
1528                 /*
1529                  * Chain is already started, make sure its active,
1530                  * if not then start the chain
1531                  */
1532                 else {
1533                         start_dma = 1;
1534
1535                         if (dma_chan[dma_chan[lch].prev_linked_ch].state ==
1536                                                         DMA_CH_STARTED) {
1537                                 enable_lnk(dma_chan[lch].prev_linked_ch);
1538                                 dma_chan[lch].state = DMA_CH_QUEUED;
1539                                 start_dma = 0;
1540                                 if (0 == ((1 << 7) & p->dma_read(
1541                                         CCR, dma_chan[lch].prev_linked_ch))) {
1542                                         disable_lnk(dma_chan[lch].
1543                                                     prev_linked_ch);
1544                                         pr_debug("\n prev ch is stopped\n");
1545                                         start_dma = 1;
1546                                 }
1547                         }
1548
1549                         else if (dma_chan[dma_chan[lch].prev_linked_ch].state
1550                                                         == DMA_CH_QUEUED) {
1551                                 enable_lnk(dma_chan[lch].prev_linked_ch);
1552                                 dma_chan[lch].state = DMA_CH_QUEUED;
1553                                 start_dma = 0;
1554                         }
1555                         omap_enable_channel_irq(lch);
1556
1557                         l = p->dma_read(CCR, lch);
1558
1559                         if ((0 == (l & (1 << 24))))
1560                                 l &= ~(1 << 25);
1561                         else
1562                                 l |= (1 << 25);
1563                         if (start_dma == 1) {
1564                                 if (0 == (l & (1 << 7))) {
1565                                         l |= (1 << 7);
1566                                         dma_chan[lch].state = DMA_CH_STARTED;
1567                                         pr_debug("starting %d\n", lch);
1568                                         p->dma_write(l, CCR, lch);
1569                                 } else
1570                                         start_dma = 0;
1571                         } else {
1572                                 if (0 == (l & (1 << 7)))
1573                                         p->dma_write(l, CCR, lch);
1574                         }
1575                         dma_chan[lch].flags |= OMAP_DMA_ACTIVE;
1576                 }
1577         }
1578
1579         return 0;
1580 }
1581 EXPORT_SYMBOL(omap_dma_chain_a_transfer);
1582
1583 /**
1584  * @brief omap_start_dma_chain_transfers - Start the chain
1585  *
1586  * @param chain_id
1587  *
1588  * @return - Success : 0
1589  *           Failure : -EINVAL/-EBUSY
1590  */
1591 int omap_start_dma_chain_transfers(int chain_id)
1592 {
1593         int *channels;
1594         u32 l, i;
1595
1596         if (unlikely((chain_id < 0 || chain_id >= dma_lch_count))) {
1597                 printk(KERN_ERR "Invalid chain id\n");
1598                 return -EINVAL;
1599         }
1600
1601         channels = dma_linked_lch[chain_id].linked_dmach_q;
1602
1603         if (dma_linked_lch[channels[0]].chain_state == DMA_CHAIN_STARTED) {
1604                 printk(KERN_ERR "Chain is already started\n");
1605                 return -EBUSY;
1606         }
1607
1608         if (dma_linked_lch[chain_id].chain_mode == OMAP_DMA_STATIC_CHAIN) {
1609                 for (i = 0; i < dma_linked_lch[chain_id].no_of_lchs_linked;
1610                                                                         i++) {
1611                         enable_lnk(channels[i]);
1612                         omap_enable_channel_irq(channels[i]);
1613                 }
1614         } else {
1615                 omap_enable_channel_irq(channels[0]);
1616         }
1617
1618         l = p->dma_read(CCR, channels[0]);
1619         l |= (1 << 7);
1620         dma_linked_lch[chain_id].chain_state = DMA_CHAIN_STARTED;
1621         dma_chan[channels[0]].state = DMA_CH_STARTED;
1622
1623         if ((0 == (l & (1 << 24))))
1624                 l &= ~(1 << 25);
1625         else
1626                 l |= (1 << 25);
1627         p->dma_write(l, CCR, channels[0]);
1628
1629         dma_chan[channels[0]].flags |= OMAP_DMA_ACTIVE;
1630
1631         return 0;
1632 }
1633 EXPORT_SYMBOL(omap_start_dma_chain_transfers);
1634
1635 /**
1636  * @brief omap_stop_dma_chain_transfers - Stop the dma transfer of a chain.
1637  *
1638  * @param chain_id
1639  *
1640  * @return - Success : 0
1641  *           Failure : EINVAL
1642  */
1643 int omap_stop_dma_chain_transfers(int chain_id)
1644 {
1645         int *channels;
1646         u32 l, i;
1647         u32 sys_cf = 0;
1648
1649         /* Check for input params */
1650         if (unlikely((chain_id < 0 || chain_id >= dma_lch_count))) {
1651                 printk(KERN_ERR "Invalid chain id\n");
1652                 return -EINVAL;
1653         }
1654
1655         /* Check if the chain exists */
1656         if (dma_linked_lch[chain_id].linked_dmach_q == NULL) {
1657                 printk(KERN_ERR "Chain doesn't exists\n");
1658                 return -EINVAL;
1659         }
1660         channels = dma_linked_lch[chain_id].linked_dmach_q;
1661
1662         if (IS_DMA_ERRATA(DMA_ERRATA_i88)) {
1663                 sys_cf = p->dma_read(OCP_SYSCONFIG, 0);
1664                 l = sys_cf;
1665                 /* Middle mode reg set no Standby */
1666                 l &= ~((1 << 12)|(1 << 13));
1667                 p->dma_write(l, OCP_SYSCONFIG, 0);
1668         }
1669
1670         for (i = 0; i < dma_linked_lch[chain_id].no_of_lchs_linked; i++) {
1671
1672                 /* Stop the Channel transmission */
1673                 l = p->dma_read(CCR, channels[i]);
1674                 l &= ~(1 << 7);
1675                 p->dma_write(l, CCR, channels[i]);
1676
1677                 /* Disable the link in all the channels */
1678                 disable_lnk(channels[i]);
1679                 dma_chan[channels[i]].state = DMA_CH_NOTSTARTED;
1680
1681         }
1682         dma_linked_lch[chain_id].chain_state = DMA_CHAIN_NOTSTARTED;
1683
1684         /* Reset the Queue pointers */
1685         OMAP_DMA_CHAIN_QINIT(chain_id);
1686
1687         if (IS_DMA_ERRATA(DMA_ERRATA_i88))
1688                 p->dma_write(sys_cf, OCP_SYSCONFIG, 0);
1689
1690         return 0;
1691 }
1692 EXPORT_SYMBOL(omap_stop_dma_chain_transfers);
1693
1694 /* Get the index of the ongoing DMA in chain */
1695 /**
1696  * @brief omap_get_dma_chain_index - Get the element and frame index
1697  * of the ongoing DMA in chain
1698  *
1699  * @param chain_id
1700  * @param ei - Element index
1701  * @param fi - Frame index
1702  *
1703  * @return - Success : 0
1704  *           Failure : -EINVAL
1705  */
1706 int omap_get_dma_chain_index(int chain_id, int *ei, int *fi)
1707 {
1708         int lch;
1709         int *channels;
1710
1711         /* Check for input params */
1712         if (unlikely((chain_id < 0 || chain_id >= dma_lch_count))) {
1713                 printk(KERN_ERR "Invalid chain id\n");
1714                 return -EINVAL;
1715         }
1716
1717         /* Check if the chain exists */
1718         if (dma_linked_lch[chain_id].linked_dmach_q == NULL) {
1719                 printk(KERN_ERR "Chain doesn't exists\n");
1720                 return -EINVAL;
1721         }
1722         if ((!ei) || (!fi))
1723                 return -EINVAL;
1724
1725         channels = dma_linked_lch[chain_id].linked_dmach_q;
1726
1727         /* Get the current channel */
1728         lch = channels[dma_linked_lch[chain_id].q_head];
1729
1730         *ei = p->dma_read(CCEN, lch);
1731         *fi = p->dma_read(CCFN, lch);
1732
1733         return 0;
1734 }
1735 EXPORT_SYMBOL(omap_get_dma_chain_index);
1736
1737 /**
1738  * @brief omap_get_dma_chain_dst_pos - Get the destination position of the
1739  * ongoing DMA in chain
1740  *
1741  * @param chain_id
1742  *
1743  * @return - Success : Destination position
1744  *           Failure : -EINVAL
1745  */
1746 int omap_get_dma_chain_dst_pos(int chain_id)
1747 {
1748         int lch;
1749         int *channels;
1750
1751         /* Check for input params */
1752         if (unlikely((chain_id < 0 || chain_id >= dma_lch_count))) {
1753                 printk(KERN_ERR "Invalid chain id\n");
1754                 return -EINVAL;
1755         }
1756
1757         /* Check if the chain exists */
1758         if (dma_linked_lch[chain_id].linked_dmach_q == NULL) {
1759                 printk(KERN_ERR "Chain doesn't exists\n");
1760                 return -EINVAL;
1761         }
1762
1763         channels = dma_linked_lch[chain_id].linked_dmach_q;
1764
1765         /* Get the current channel */
1766         lch = channels[dma_linked_lch[chain_id].q_head];
1767
1768         return p->dma_read(CDAC, lch);
1769 }
1770 EXPORT_SYMBOL(omap_get_dma_chain_dst_pos);
1771
1772 /**
1773  * @brief omap_get_dma_chain_src_pos - Get the source position
1774  * of the ongoing DMA in chain
1775  * @param chain_id
1776  *
1777  * @return - Success : Destination position
1778  *           Failure : -EINVAL
1779  */
1780 int omap_get_dma_chain_src_pos(int chain_id)
1781 {
1782         int lch;
1783         int *channels;
1784
1785         /* Check for input params */
1786         if (unlikely((chain_id < 0 || chain_id >= dma_lch_count))) {
1787                 printk(KERN_ERR "Invalid chain id\n");
1788                 return -EINVAL;
1789         }
1790
1791         /* Check if the chain exists */
1792         if (dma_linked_lch[chain_id].linked_dmach_q == NULL) {
1793                 printk(KERN_ERR "Chain doesn't exists\n");
1794                 return -EINVAL;
1795         }
1796
1797         channels = dma_linked_lch[chain_id].linked_dmach_q;
1798
1799         /* Get the current channel */
1800         lch = channels[dma_linked_lch[chain_id].q_head];
1801
1802         return p->dma_read(CSAC, lch);
1803 }
1804 EXPORT_SYMBOL(omap_get_dma_chain_src_pos);
1805 #endif  /* ifndef CONFIG_ARCH_OMAP1 */
1806
1807 /*----------------------------------------------------------------------------*/
1808
1809 #ifdef CONFIG_ARCH_OMAP1
1810
1811 static int omap1_dma_handle_ch(int ch)
1812 {
1813         u32 csr;
1814
1815         if (enable_1510_mode && ch >= 6) {
1816                 csr = dma_chan[ch].saved_csr;
1817                 dma_chan[ch].saved_csr = 0;
1818         } else
1819                 csr = p->dma_read(CSR, ch);
1820         if (enable_1510_mode && ch <= 2 && (csr >> 7) != 0) {
1821                 dma_chan[ch + 6].saved_csr = csr >> 7;
1822                 csr &= 0x7f;
1823         }
1824         if ((csr & 0x3f) == 0)
1825                 return 0;
1826         if (unlikely(dma_chan[ch].dev_id == -1)) {
1827                 printk(KERN_WARNING "Spurious interrupt from DMA channel "
1828                        "%d (CSR %04x)\n", ch, csr);
1829                 return 0;
1830         }
1831         if (unlikely(csr & OMAP1_DMA_TOUT_IRQ))
1832                 printk(KERN_WARNING "DMA timeout with device %d\n",
1833                        dma_chan[ch].dev_id);
1834         if (unlikely(csr & OMAP_DMA_DROP_IRQ))
1835                 printk(KERN_WARNING "DMA synchronization event drop occurred "
1836                        "with device %d\n", dma_chan[ch].dev_id);
1837         if (likely(csr & OMAP_DMA_BLOCK_IRQ))
1838                 dma_chan[ch].flags &= ~OMAP_DMA_ACTIVE;
1839         if (likely(dma_chan[ch].callback != NULL))
1840                 dma_chan[ch].callback(ch, csr, dma_chan[ch].data);
1841
1842         return 1;
1843 }
1844
1845 static irqreturn_t omap1_dma_irq_handler(int irq, void *dev_id)
1846 {
1847         int ch = ((int) dev_id) - 1;
1848         int handled = 0;
1849
1850         for (;;) {
1851                 int handled_now = 0;
1852
1853                 handled_now += omap1_dma_handle_ch(ch);
1854                 if (enable_1510_mode && dma_chan[ch + 6].saved_csr)
1855                         handled_now += omap1_dma_handle_ch(ch + 6);
1856                 if (!handled_now)
1857                         break;
1858                 handled += handled_now;
1859         }
1860
1861         return handled ? IRQ_HANDLED : IRQ_NONE;
1862 }
1863
1864 #else
1865 #define omap1_dma_irq_handler   NULL
1866 #endif
1867
1868 #ifdef CONFIG_ARCH_OMAP2PLUS
1869
1870 static int omap2_dma_handle_ch(int ch)
1871 {
1872         u32 status = p->dma_read(CSR, ch);
1873
1874         if (!status) {
1875                 if (printk_ratelimit())
1876                         printk(KERN_WARNING "Spurious DMA IRQ for lch %d\n",
1877                                 ch);
1878                 p->dma_write(1 << ch, IRQSTATUS_L0, ch);
1879                 return 0;
1880         }
1881         if (unlikely(dma_chan[ch].dev_id == -1)) {
1882                 if (printk_ratelimit())
1883                         printk(KERN_WARNING "IRQ %04x for non-allocated DMA"
1884                                         "channel %d\n", status, ch);
1885                 return 0;
1886         }
1887         if (unlikely(status & OMAP_DMA_DROP_IRQ))
1888                 printk(KERN_INFO
1889                        "DMA synchronization event drop occurred with device "
1890                        "%d\n", dma_chan[ch].dev_id);
1891         if (unlikely(status & OMAP2_DMA_TRANS_ERR_IRQ)) {
1892                 printk(KERN_INFO "DMA transaction error with device %d\n",
1893                        dma_chan[ch].dev_id);
1894                 if (IS_DMA_ERRATA(DMA_ERRATA_i378)) {
1895                         u32 ccr;
1896
1897                         ccr = p->dma_read(CCR, ch);
1898                         ccr &= ~OMAP_DMA_CCR_EN;
1899                         p->dma_write(ccr, CCR, ch);
1900                         dma_chan[ch].flags &= ~OMAP_DMA_ACTIVE;
1901                 }
1902         }
1903         if (unlikely(status & OMAP2_DMA_SECURE_ERR_IRQ))
1904                 printk(KERN_INFO "DMA secure error with device %d\n",
1905                        dma_chan[ch].dev_id);
1906         if (unlikely(status & OMAP2_DMA_MISALIGNED_ERR_IRQ))
1907                 printk(KERN_INFO "DMA misaligned error with device %d\n",
1908                        dma_chan[ch].dev_id);
1909
1910         p->dma_write(status, CSR, ch);
1911         p->dma_write(1 << ch, IRQSTATUS_L0, ch);
1912         /* read back the register to flush the write */
1913         p->dma_read(IRQSTATUS_L0, ch);
1914
1915         /* If the ch is not chained then chain_id will be -1 */
1916         if (dma_chan[ch].chain_id != -1) {
1917                 int chain_id = dma_chan[ch].chain_id;
1918                 dma_chan[ch].state = DMA_CH_NOTSTARTED;
1919                 if (p->dma_read(CLNK_CTRL, ch) & (1 << 15))
1920                         dma_chan[dma_chan[ch].next_linked_ch].state =
1921                                                         DMA_CH_STARTED;
1922                 if (dma_linked_lch[chain_id].chain_mode ==
1923                                                 OMAP_DMA_DYNAMIC_CHAIN)
1924                         disable_lnk(ch);
1925
1926                 if (!OMAP_DMA_CHAIN_QEMPTY(chain_id))
1927                         OMAP_DMA_CHAIN_INCQHEAD(chain_id);
1928
1929                 status = p->dma_read(CSR, ch);
1930                 p->dma_write(status, CSR, ch);
1931         }
1932
1933         if (likely(dma_chan[ch].callback != NULL))
1934                 dma_chan[ch].callback(ch, status, dma_chan[ch].data);
1935
1936         return 0;
1937 }
1938
1939 /* STATUS register count is from 1-32 while our is 0-31 */
1940 static irqreturn_t omap2_dma_irq_handler(int irq, void *dev_id)
1941 {
1942         u32 val, enable_reg;
1943         int i;
1944
1945         val = p->dma_read(IRQSTATUS_L0, 0);
1946         if (val == 0) {
1947                 if (printk_ratelimit())
1948                         printk(KERN_WARNING "Spurious DMA IRQ\n");
1949                 return IRQ_HANDLED;
1950         }
1951         enable_reg = p->dma_read(IRQENABLE_L0, 0);
1952         val &= enable_reg; /* Dispatch only relevant interrupts */
1953         for (i = 0; i < dma_lch_count && val != 0; i++) {
1954                 if (val & 1)
1955                         omap2_dma_handle_ch(i);
1956                 val >>= 1;
1957         }
1958
1959         return IRQ_HANDLED;
1960 }
1961
1962 static struct irqaction omap24xx_dma_irq = {
1963         .name = "DMA",
1964         .handler = omap2_dma_irq_handler,
1965         .flags = IRQF_DISABLED
1966 };
1967
1968 #else
1969 static struct irqaction omap24xx_dma_irq;
1970 #endif
1971
1972 /*----------------------------------------------------------------------------*/
1973
1974 void omap_dma_global_context_save(void)
1975 {
1976         omap_dma_global_context.dma_irqenable_l0 =
1977                 p->dma_read(IRQENABLE_L0, 0);
1978         omap_dma_global_context.dma_ocp_sysconfig =
1979                 p->dma_read(OCP_SYSCONFIG, 0);
1980         omap_dma_global_context.dma_gcr = p->dma_read(GCR, 0);
1981 }
1982
1983 void omap_dma_global_context_restore(void)
1984 {
1985         int ch;
1986
1987         p->dma_write(omap_dma_global_context.dma_gcr, GCR, 0);
1988         p->dma_write(omap_dma_global_context.dma_ocp_sysconfig,
1989                 OCP_SYSCONFIG, 0);
1990         p->dma_write(omap_dma_global_context.dma_irqenable_l0,
1991                 IRQENABLE_L0, 0);
1992
1993         if (IS_DMA_ERRATA(DMA_ROMCODE_BUG))
1994                 p->dma_write(0x3 , IRQSTATUS_L0, 0);
1995
1996         for (ch = 0; ch < dma_chan_count; ch++)
1997                 if (dma_chan[ch].dev_id != -1)
1998                         omap_clear_dma(ch);
1999 }
2000
2001 static int __devinit omap_system_dma_probe(struct platform_device *pdev)
2002 {
2003         int ch, ret = 0;
2004         int dma_irq;
2005         char irq_name[4];
2006         int irq_rel;
2007
2008         p = pdev->dev.platform_data;
2009         if (!p) {
2010                 dev_err(&pdev->dev, "%s: System DMA initialized without"
2011                         "platform data\n", __func__);
2012                 return -EINVAL;
2013         }
2014
2015         d                       = p->dma_attr;
2016         errata                  = p->errata;
2017
2018         if ((d->dev_caps & RESERVE_CHANNEL) && omap_dma_reserve_channels
2019                         && (omap_dma_reserve_channels <= dma_lch_count))
2020                 d->lch_count    = omap_dma_reserve_channels;
2021
2022         dma_lch_count           = d->lch_count;
2023         dma_chan_count          = dma_lch_count;
2024         dma_chan                = d->chan;
2025         enable_1510_mode        = d->dev_caps & ENABLE_1510_MODE;
2026
2027         if (cpu_class_is_omap2()) {
2028                 dma_linked_lch = kzalloc(sizeof(struct dma_link_info) *
2029                                                 dma_lch_count, GFP_KERNEL);
2030                 if (!dma_linked_lch) {
2031                         ret = -ENOMEM;
2032                         goto exit_dma_lch_fail;
2033                 }
2034         }
2035
2036         spin_lock_init(&dma_chan_lock);
2037         for (ch = 0; ch < dma_chan_count; ch++) {
2038                 omap_clear_dma(ch);
2039                 if (cpu_class_is_omap2())
2040                         omap2_disable_irq_lch(ch);
2041
2042                 dma_chan[ch].dev_id = -1;
2043                 dma_chan[ch].next_lch = -1;
2044
2045                 if (ch >= 6 && enable_1510_mode)
2046                         continue;
2047
2048                 if (cpu_class_is_omap1()) {
2049                         /*
2050                          * request_irq() doesn't like dev_id (ie. ch) being
2051                          * zero, so we have to kludge around this.
2052                          */
2053                         sprintf(&irq_name[0], "%d", ch);
2054                         dma_irq = platform_get_irq_byname(pdev, irq_name);
2055
2056                         if (dma_irq < 0) {
2057                                 ret = dma_irq;
2058                                 goto exit_dma_irq_fail;
2059                         }
2060
2061                         /* INT_DMA_LCD is handled in lcd_dma.c */
2062                         if (dma_irq == INT_DMA_LCD)
2063                                 continue;
2064
2065                         ret = request_irq(dma_irq,
2066                                         omap1_dma_irq_handler, 0, "DMA",
2067                                         (void *) (ch + 1));
2068                         if (ret != 0)
2069                                 goto exit_dma_irq_fail;
2070                 }
2071         }
2072
2073         if (cpu_is_omap2430() || cpu_is_omap34xx() || cpu_is_omap44xx())
2074                 omap_dma_set_global_params(DMA_DEFAULT_ARB_RATE,
2075                                 DMA_DEFAULT_FIFO_DEPTH, 0);
2076
2077         if (cpu_class_is_omap2()) {
2078                 strcpy(irq_name, "0");
2079                 dma_irq = platform_get_irq_byname(pdev, irq_name);
2080                 if (dma_irq < 0) {
2081                         dev_err(&pdev->dev, "failed: request IRQ %d", dma_irq);
2082                         goto exit_dma_lch_fail;
2083                 }
2084                 ret = setup_irq(dma_irq, &omap24xx_dma_irq);
2085                 if (ret) {
2086                         dev_err(&pdev->dev, "set_up failed for IRQ %d"
2087                                 "for DMA (error %d)\n", dma_irq, ret);
2088                         goto exit_dma_lch_fail;
2089                 }
2090         }
2091
2092         /* reserve dma channels 0 and 1 in high security devices */
2093         if (cpu_is_omap34xx() &&
2094                 (omap_type() != OMAP2_DEVICE_TYPE_GP)) {
2095                 printk(KERN_INFO "Reserving DMA channels 0 and 1 for "
2096                                 "HS ROM code\n");
2097                 dma_chan[0].dev_id = 0;
2098                 dma_chan[1].dev_id = 1;
2099         }
2100         p->show_dma_caps();
2101         return 0;
2102
2103 exit_dma_irq_fail:
2104         dev_err(&pdev->dev, "unable to request IRQ %d"
2105                         "for DMA (error %d)\n", dma_irq, ret);
2106         for (irq_rel = 0; irq_rel < ch; irq_rel++) {
2107                 dma_irq = platform_get_irq(pdev, irq_rel);
2108                 free_irq(dma_irq, (void *)(irq_rel + 1));
2109         }
2110
2111 exit_dma_lch_fail:
2112         kfree(p);
2113         kfree(d);
2114         kfree(dma_chan);
2115         return ret;
2116 }
2117
2118 static int __devexit omap_system_dma_remove(struct platform_device *pdev)
2119 {
2120         int dma_irq;
2121
2122         if (cpu_class_is_omap2()) {
2123                 char irq_name[4];
2124                 strcpy(irq_name, "0");
2125                 dma_irq = platform_get_irq_byname(pdev, irq_name);
2126                 remove_irq(dma_irq, &omap24xx_dma_irq);
2127         } else {
2128                 int irq_rel = 0;
2129                 for ( ; irq_rel < dma_chan_count; irq_rel++) {
2130                         dma_irq = platform_get_irq(pdev, irq_rel);
2131                         free_irq(dma_irq, (void *)(irq_rel + 1));
2132                 }
2133         }
2134         kfree(p);
2135         kfree(d);
2136         kfree(dma_chan);
2137         return 0;
2138 }
2139
2140 static struct platform_driver omap_system_dma_driver = {
2141         .probe          = omap_system_dma_probe,
2142         .remove         = omap_system_dma_remove,
2143         .driver         = {
2144                 .name   = "omap_dma_system"
2145         },
2146 };
2147
2148 static int __init omap_system_dma_init(void)
2149 {
2150         return platform_driver_register(&omap_system_dma_driver);
2151 }
2152 arch_initcall(omap_system_dma_init);
2153
2154 static void __exit omap_system_dma_exit(void)
2155 {
2156         platform_driver_unregister(&omap_system_dma_driver);
2157 }
2158
2159 MODULE_DESCRIPTION("OMAP SYSTEM DMA DRIVER");
2160 MODULE_LICENSE("GPL");
2161 MODULE_ALIAS("platform:" DRIVER_NAME);
2162 MODULE_AUTHOR("Texas Instruments Inc");
2163
2164 /*
2165  * Reserve the omap SDMA channels using cmdline bootarg
2166  * "omap_dma_reserve_ch=". The valid range is 1 to 32
2167  */
2168 static int __init omap_dma_cmdline_reserve_ch(char *str)
2169 {
2170         if (get_option(&str, &omap_dma_reserve_channels) != 1)
2171                 omap_dma_reserve_channels = 0;
2172         return 1;
2173 }
2174
2175 __setup("omap_dma_reserve_ch=", omap_dma_cmdline_reserve_ch);
2176
2177