arch/arm/plat-omap/dma.c

   1 /*
   2  * linux/arch/arm/plat-omap/dma.c
   3  *
   4  * Copyright (C) 2003 - 2008 Nokia Corporation
   5  * Author: Juha Yrjölä <juha.yrjola@nokia.com>
   6  * DMA channel linking for 1610 by Samuel Ortiz <samuel.ortiz@nokia.com>
   7  * Graphics DMA and LCD DMA graphics tranformations
   8  * by Imre Deak <imre.deak@nokia.com>
   9  * OMAP2/3 support Copyright (C) 2004-2007 Texas Instruments, Inc.
  10  * Merged to support both OMAP1 and OMAP2 by Tony Lindgren <tony@atomide.com>
  11  * Some functions based on earlier dma-omap.c Copyright (C) 2001 RidgeRun, Inc.
  12  *
  13  * Copyright (C) 2009 Texas Instruments
  14  * Added OMAP4 support - Santosh Shilimkar <santosh.shilimkar@ti.com>
  15  *
  16  * Support functions for the OMAP internal DMA channels.
  17  *
  18  * This program is free software; you can redistribute it and/or modify
  19  * it under the terms of the GNU General Public License version 2 as
  20  * published by the Free Software Foundation.
  21  *
  22  */
  23
  24 #include <linux/module.h>
  25 #include <linux/init.h>
  26 #include <linux/sched.h>
  27 #include <linux/spinlock.h>
  28 #include <linux/errno.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/irq.h>
  31 #include <linux/io.h>
  32 #include <linux/slab.h>
  33
  34 #include <asm/system.h>
  35 #include <mach/hardware.h>
  36 #include <plat/dma.h>
  37
  38 #include <plat/tc.h>
  39
  40 #undef DEBUG
  41
  42 #ifndef CONFIG_ARCH_OMAP1
  43 enum { DMA_CH_ALLOC_DONE, DMA_CH_PARAMS_SET_DONE, DMA_CH_STARTED,
  44         DMA_CH_QUEUED, DMA_CH_NOTSTARTED, DMA_CH_PAUSED, DMA_CH_LINK_ENABLED
  45 };
  46
  47 enum { DMA_CHAIN_STARTED, DMA_CHAIN_NOTSTARTED };
  48 #endif
  49
  50 #define OMAP_DMA_ACTIVE                 0x01
  51 #define OMAP2_DMA_CSR_CLEAR_MASK        0xffe
  52
  53 #define OMAP_FUNC_MUX_ARM_BASE          (0xfffe1000 + 0xec)
  54
  55 static int enable_1510_mode;
  56
  57 static struct omap_dma_global_context_registers {
  58         u32 dma_irqenable_l0;
  59         u32 dma_ocp_sysconfig;
  60         u32 dma_gcr;
  61 } omap_dma_global_context;
  62
  63 struct omap_dma_lch {
  64         int next_lch;
  65         int dev_id;
  66         u16 saved_csr;
  67         u16 enabled_irqs;
  68         const char *dev_name;
  69         void (*callback)(int lch, u16 ch_status, void *data);
  70         void *data;
  71
  72 #ifndef CONFIG_ARCH_OMAP1
  73         /* required for Dynamic chaining */
  74         int prev_linked_ch;
  75         int next_linked_ch;
  76         int state;
  77         int chain_id;
  78
  79         int status;
  80 #endif
  81         long flags;
  82 };
  83
  84 struct dma_link_info {
  85         int *linked_dmach_q;
  86         int no_of_lchs_linked;
  87
  88         int q_count;
  89         int q_tail;
  90         int q_head;
  91
  92         int chain_state;
  93         int chain_mode;
  94
  95 };
  96
  97 static struct dma_link_info *dma_linked_lch;
  98
  99 #ifndef CONFIG_ARCH_OMAP1
 100
 101 /* Chain handling macros */
 102 #define OMAP_DMA_CHAIN_QINIT(chain_id)                                  \
 103         do {                                                            \
 104                 dma_linked_lch[chain_id].q_head =                       \
 105                 dma_linked_lch[chain_id].q_tail =                       \
 106                 dma_linked_lch[chain_id].q_count = 0;                   \
 107         } while (0)
 108 #define OMAP_DMA_CHAIN_QFULL(chain_id)                                  \
 109                 (dma_linked_lch[chain_id].no_of_lchs_linked ==          \
 110                 dma_linked_lch[chain_id].q_count)
 111 #define OMAP_DMA_CHAIN_QLAST(chain_id)                                  \
 112         do {                                                            \
 113                 ((dma_linked_lch[chain_id].no_of_lchs_linked-1) ==      \
 114                 dma_linked_lch[chain_id].q_count)                       \
 115         } while (0)
 116 #define OMAP_DMA_CHAIN_QEMPTY(chain_id)                                 \
 117                 (0 == dma_linked_lch[chain_id].q_count)
 118 #define __OMAP_DMA_CHAIN_INCQ(end)                                      \
 119         ((end) = ((end)+1) % dma_linked_lch[chain_id].no_of_lchs_linked)
 120 #define OMAP_DMA_CHAIN_INCQHEAD(chain_id)                               \
 121         do {                                                            \
 122                 __OMAP_DMA_CHAIN_INCQ(dma_linked_lch[chain_id].q_head); \
 123                 dma_linked_lch[chain_id].q_count--;                     \
 124         } while (0)
 125
 126 #define OMAP_DMA_CHAIN_INCQTAIL(chain_id)                               \
 127         do {                                                            \
 128                 __OMAP_DMA_CHAIN_INCQ(dma_linked_lch[chain_id].q_tail); \
 129                 dma_linked_lch[chain_id].q_count++; \
 130         } while (0)
 131 #endif
 132
 133 static int dma_lch_count;
 134 static int dma_chan_count;
 135 static int omap_dma_reserve_channels;
 136
 137 static spinlock_t dma_chan_lock;
 138 static struct omap_dma_lch *dma_chan;
 139 static void __iomem *omap_dma_base;
 140
 141 static const u8 omap1_dma_irq[OMAP1_LOGICAL_DMA_CH_COUNT] = {
 142         INT_DMA_CH0_6, INT_DMA_CH1_7, INT_DMA_CH2_8, INT_DMA_CH3,
 143         INT_DMA_CH4, INT_DMA_CH5, INT_1610_DMA_CH6, INT_1610_DMA_CH7,
 144         INT_1610_DMA_CH8, INT_1610_DMA_CH9, INT_1610_DMA_CH10,
 145         INT_1610_DMA_CH11, INT_1610_DMA_CH12, INT_1610_DMA_CH13,
 146         INT_1610_DMA_CH14, INT_1610_DMA_CH15, INT_DMA_LCD
 147 };
 148
 149 static inline void disable_lnk(int lch);
 150 static void omap_disable_channel_irq(int lch);
 151 static inline void omap_enable_channel_irq(int lch);
 152
 153 #define REVISIT_24XX()          printk(KERN_ERR "FIXME: no %s on 24xx\n", \
 154                                                 __func__);
 155
 156 #define dma_read(reg)                                                   \
 157 ({                                                                      \
 158         u32 __val;                                                      \
 159         if (cpu_class_is_omap1())                                       \
 160                 __val = __raw_readw(omap_dma_base + OMAP1_DMA_##reg);   \
 161         else                                                            \
 162                 __val = __raw_readl(omap_dma_base + OMAP_DMA4_##reg);   \
 163         __val;                                                          \
 164 })
 165
 166 #define dma_write(val, reg)                                             \
 167 ({                                                                      \
 168         if (cpu_class_is_omap1())                                       \
 169                 __raw_writew((u16)(val), omap_dma_base + OMAP1_DMA_##reg); \
 170         else                                                            \
 171                 __raw_writel((val), omap_dma_base + OMAP_DMA4_##reg);   \
 172 })
 173
 174 #ifdef CONFIG_ARCH_OMAP15XX
 175 /* Returns 1 if the DMA module is in OMAP1510-compatible mode, 0 otherwise */
 176 int omap_dma_in_1510_mode(void)
 177 {
 178         return enable_1510_mode;
 179 }
 180 #else
 181 #define omap_dma_in_1510_mode()         0
 182 #endif
 183
 184 #ifdef CONFIG_ARCH_OMAP1
 185 static inline int get_gdma_dev(int req)
 186 {
 187         u32 reg = OMAP_FUNC_MUX_ARM_BASE + ((req - 1) / 5) * 4;
 188         int shift = ((req - 1) % 5) * 6;
 189
 190         return ((omap_readl(reg) >> shift) & 0x3f) + 1;
 191 }
 192
 193 static inline void set_gdma_dev(int req, int dev)
 194 {
 195         u32 reg = OMAP_FUNC_MUX_ARM_BASE + ((req - 1) / 5) * 4;
 196         int shift = ((req - 1) % 5) * 6;
 197         u32 l;
 198
 199         l = omap_readl(reg);
 200         l &= ~(0x3f << shift);
 201         l |= (dev - 1) << shift;
 202         omap_writel(l, reg);
 203 }
 204 #else
 205 #define set_gdma_dev(req, dev)  do {} while (0)
 206 #endif
 207
 208 /* Omap1 only */
 209 static void clear_lch_regs(int lch)
 210 {
 211         int i;
 212         void __iomem *lch_base = omap_dma_base + OMAP1_DMA_CH_BASE(lch);
 213
 214         for (i = 0; i < 0x2c; i += 2)
 215                 __raw_writew(0, lch_base + i);
 216 }
 217
 218 void omap_set_dma_priority(int lch, int dst_port, int priority)
 219 {
 220         unsigned long reg;
 221         u32 l;
 222
 223         if (cpu_class_is_omap1()) {
 224                 switch (dst_port) {
 225                 case OMAP_DMA_PORT_OCP_T1:      /* FFFECC00 */
 226                         reg = OMAP_TC_OCPT1_PRIOR;
 227                         break;
 228                 case OMAP_DMA_PORT_OCP_T2:      /* FFFECCD0 */
 229                         reg = OMAP_TC_OCPT2_PRIOR;
 230                         break;
 231                 case OMAP_DMA_PORT_EMIFF:       /* FFFECC08 */
 232                         reg = OMAP_TC_EMIFF_PRIOR;
 233                         break;
 234                 case OMAP_DMA_PORT_EMIFS:       /* FFFECC04 */
 235                         reg = OMAP_TC_EMIFS_PRIOR;
 236                         break;
 237                 default:
 238                         BUG();
 239                         return;
 240                 }
 241                 l = omap_readl(reg);
 242                 l &= ~(0xf << 8);
 243                 l |= (priority & 0xf) << 8;
 244                 omap_writel(l, reg);
 245         }
 246
 247         if (cpu_class_is_omap2()) {
 248                 u32 ccr;
 249
 250                 ccr = dma_read(CCR(lch));
 251                 if (priority)
 252                         ccr |= (1 << 6);
 253                 else
 254                         ccr &= ~(1 << 6);
 255                 dma_write(ccr, CCR(lch));
 256         }
 257 }
 258 EXPORT_SYMBOL(omap_set_dma_priority);
 259
 260 void omap_set_dma_transfer_params(int lch, int data_type, int elem_count,
 261                                   int frame_count, int sync_mode,
 262                                   int dma_trigger, int src_or_dst_synch)
 263 {
 264         u32 l;
 265
 266         l = dma_read(CSDP(lch));
 267         l &= ~0x03;
 268         l |= data_type;
 269         dma_write(l, CSDP(lch));
 270
 271         if (cpu_class_is_omap1()) {
 272                 u16 ccr;
 273
 274                 ccr = dma_read(CCR(lch));
 275                 ccr &= ~(1 << 5);
 276                 if (sync_mode == OMAP_DMA_SYNC_FRAME)
 277                         ccr |= 1 << 5;
 278                 dma_write(ccr, CCR(lch));
 279
 280                 ccr = dma_read(CCR2(lch));
 281                 ccr &= ~(1 << 2);
 282                 if (sync_mode == OMAP_DMA_SYNC_BLOCK)
 283                         ccr |= 1 << 2;
 284                 dma_write(ccr, CCR2(lch));
 285         }
 286
 287         if (cpu_class_is_omap2() && dma_trigger) {
 288                 u32 val;
 289
 290                 val = dma_read(CCR(lch));
 291
 292                 /* DMA_SYNCHRO_CONTROL_UPPER depends on the channel number */
 293                 val &= ~((3 << 19) | 0x1f);
 294                 val |= (dma_trigger & ~0x1f) << 14;
 295                 val |= dma_trigger & 0x1f;
 296
 297                 if (sync_mode & OMAP_DMA_SYNC_FRAME)
 298                         val |= 1 << 5;
 299                 else
 300                         val &= ~(1 << 5);
 301
 302                 if (sync_mode & OMAP_DMA_SYNC_BLOCK)
 303                         val |= 1 << 18;
 304                 else
 305                         val &= ~(1 << 18);
 306
 307                 if (src_or_dst_synch)
 308                         val |= 1 << 24;         /* source synch */
 309                 else
 310                         val &= ~(1 << 24);      /* dest synch */
 311
 312                 dma_write(val, CCR(lch));
 313         }
 314
 315         dma_write(elem_count, CEN(lch));
 316         dma_write(frame_count, CFN(lch));
 317 }
 318 EXPORT_SYMBOL(omap_set_dma_transfer_params);
 319
 320 void omap_set_dma_color_mode(int lch, enum omap_dma_color_mode mode, u32 color)
 321 {
 322         BUG_ON(omap_dma_in_1510_mode());
 323
 324         if (cpu_class_is_omap1()) {
 325                 u16 w;
 326
 327                 w = dma_read(CCR2(lch));
 328                 w &= ~0x03;
 329
 330                 switch (mode) {
 331                 case OMAP_DMA_CONSTANT_FILL:
 332                         w |= 0x01;
 333                         break;
 334                 case OMAP_DMA_TRANSPARENT_COPY:
 335                         w |= 0x02;
 336                         break;
 337                 case OMAP_DMA_COLOR_DIS:
 338                         break;
 339                 default:
 340                         BUG();
 341                 }
 342                 dma_write(w, CCR2(lch));
 343
 344                 w = dma_read(LCH_CTRL(lch));
 345                 w &= ~0x0f;
 346                 /* Default is channel type 2D */
 347                 if (mode) {
 348                         dma_write((u16)color, COLOR_L(lch));
 349                         dma_write((u16)(color >> 16), COLOR_U(lch));
 350                         w |= 1;         /* Channel type G */
 351                 }
 352                 dma_write(w, LCH_CTRL(lch));
 353         }
 354
 355         if (cpu_class_is_omap2()) {
 356                 u32 val;
 357
 358                 val = dma_read(CCR(lch));
 359                 val &= ~((1 << 17) | (1 << 16));
 360
 361                 switch (mode) {
 362                 case OMAP_DMA_CONSTANT_FILL:
 363                         val |= 1 << 16;
 364                         break;
 365                 case OMAP_DMA_TRANSPARENT_COPY:
 366                         val |= 1 << 17;
 367                         break;
 368                 case OMAP_DMA_COLOR_DIS:
 369                         break;
 370                 default:
 371                         BUG();
 372                 }
 373                 dma_write(val, CCR(lch));
 374
 375                 color &= 0xffffff;
 376                 dma_write(color, COLOR(lch));
 377         }
 378 }
 379 EXPORT_SYMBOL(omap_set_dma_color_mode);
 380
 381 void omap_set_dma_write_mode(int lch, enum omap_dma_write_mode mode)
 382 {
 383         if (cpu_class_is_omap2()) {
 384                 u32 csdp;
 385
 386                 csdp = dma_read(CSDP(lch));
 387                 csdp &= ~(0x3 << 16);
 388                 csdp |= (mode << 16);
 389                 dma_write(csdp, CSDP(lch));
 390         }
 391 }
 392 EXPORT_SYMBOL(omap_set_dma_write_mode);
 393
 394 void omap_set_dma_channel_mode(int lch, enum omap_dma_channel_mode mode)
 395 {
 396         if (cpu_class_is_omap1() && !cpu_is_omap15xx()) {
 397                 u32 l;
 398
 399                 l = dma_read(LCH_CTRL(lch));
 400                 l &= ~0x7;
 401                 l |= mode;
 402                 dma_write(l, LCH_CTRL(lch));
 403         }
 404 }
 405 EXPORT_SYMBOL(omap_set_dma_channel_mode);
 406
 407 /* Note that src_port is only for omap1 */
 408 void omap_set_dma_src_params(int lch, int src_port, int src_amode,
 409                              unsigned long src_start,
 410                              int src_ei, int src_fi)
 411 {
 412         u32 l;
 413
 414         if (cpu_class_is_omap1()) {
 415                 u16 w;
 416
 417                 w = dma_read(CSDP(lch));
 418                 w &= ~(0x1f << 2);
 419                 w |= src_port << 2;
 420                 dma_write(w, CSDP(lch));
 421         }
 422
 423         l = dma_read(CCR(lch));
 424         l &= ~(0x03 << 12);
 425         l |= src_amode << 12;
 426         dma_write(l, CCR(lch));
 427
 428         if (cpu_class_is_omap1()) {
 429                 dma_write(src_start >> 16, CSSA_U(lch));
 430                 dma_write((u16)src_start, CSSA_L(lch));
 431         }
 432
 433         if (cpu_class_is_omap2())
 434                 dma_write(src_start, CSSA(lch));
 435
 436         dma_write(src_ei, CSEI(lch));
 437         dma_write(src_fi, CSFI(lch));
 438 }
 439 EXPORT_SYMBOL(omap_set_dma_src_params);
 440
 441 void omap_set_dma_params(int lch, struct omap_dma_channel_params *params)
 442 {
 443         omap_set_dma_transfer_params(lch, params->data_type,
 444                                      params->elem_count, params->frame_count,
 445                                      params->sync_mode, params->trigger,
 446                                      params->src_or_dst_synch);
 447         omap_set_dma_src_params(lch, params->src_port,
 448                                 params->src_amode, params->src_start,
 449                                 params->src_ei, params->src_fi);
 450
 451         omap_set_dma_dest_params(lch, params->dst_port,
 452                                  params->dst_amode, params->dst_start,
 453                                  params->dst_ei, params->dst_fi);
 454         if (params->read_prio || params->write_prio)
 455                 omap_dma_set_prio_lch(lch, params->read_prio,
 456                                       params->write_prio);
 457 }
 458 EXPORT_SYMBOL(omap_set_dma_params);
 459
 460 void omap_set_dma_src_index(int lch, int eidx, int fidx)
 461 {
 462         if (cpu_class_is_omap2())
 463                 return;
 464
 465         dma_write(eidx, CSEI(lch));
 466         dma_write(fidx, CSFI(lch));
 467 }
 468 EXPORT_SYMBOL(omap_set_dma_src_index);
 469
 470 void omap_set_dma_src_data_pack(int lch, int enable)
 471 {
 472         u32 l;
 473
 474         l = dma_read(CSDP(lch));
 475         l &= ~(1 << 6);
 476         if (enable)
 477                 l |= (1 << 6);
 478         dma_write(l, CSDP(lch));
 479 }
 480 EXPORT_SYMBOL(omap_set_dma_src_data_pack);
 481
 482 void omap_set_dma_src_burst_mode(int lch, enum omap_dma_burst_mode burst_mode)
 483 {
 484         unsigned int burst = 0;
 485         u32 l;
 486
 487         l = dma_read(CSDP(lch));
 488         l &= ~(0x03 << 7);
 489
 490         switch (burst_mode) {
 491         case OMAP_DMA_DATA_BURST_DIS:
 492                 break;
 493         case OMAP_DMA_DATA_BURST_4:
 494                 if (cpu_class_is_omap2())
 495                         burst = 0x1;
 496                 else
 497                         burst = 0x2;
 498                 break;
 499         case OMAP_DMA_DATA_BURST_8:
 500                 if (cpu_class_is_omap2()) {
 501                         burst = 0x2;
 502                         break;
 503                 }
 504                 /*
 505                  * not supported by current hardware on OMAP1
 506                  * w |= (0x03 << 7);
 507                  * fall through
 508                  */
 509         case OMAP_DMA_DATA_BURST_16:
 510                 if (cpu_class_is_omap2()) {
 511                         burst = 0x3;
 512                         break;
 513                 }
 514                 /*
 515                  * OMAP1 don't support burst 16
 516                  * fall through
 517                  */
 518         default:
 519                 BUG();
 520         }
 521
 522         l |= (burst << 7);
 523         dma_write(l, CSDP(lch));
 524 }
 525 EXPORT_SYMBOL(omap_set_dma_src_burst_mode);
 526
 527 /* Note that dest_port is only for OMAP1 */
 528 void omap_set_dma_dest_params(int lch, int dest_port, int dest_amode,
 529                               unsigned long dest_start,
 530                               int dst_ei, int dst_fi)
 531 {
 532         u32 l;
 533
 534         if (cpu_class_is_omap1()) {
 535                 l = dma_read(CSDP(lch));
 536                 l &= ~(0x1f << 9);
 537                 l |= dest_port << 9;
 538                 dma_write(l, CSDP(lch));
 539         }
 540
 541         l = dma_read(CCR(lch));
 542         l &= ~(0x03 << 14);
 543         l |= dest_amode << 14;
 544         dma_write(l, CCR(lch));
 545
 546         if (cpu_class_is_omap1()) {
 547                 dma_write(dest_start >> 16, CDSA_U(lch));
 548                 dma_write(dest_start, CDSA_L(lch));
 549         }
 550
 551         if (cpu_class_is_omap2())
 552                 dma_write(dest_start, CDSA(lch));
 553
 554         dma_write(dst_ei, CDEI(lch));
 555         dma_write(dst_fi, CDFI(lch));
 556 }
 557 EXPORT_SYMBOL(omap_set_dma_dest_params);
 558
 559 void omap_set_dma_dest_index(int lch, int eidx, int fidx)
 560 {
 561         if (cpu_class_is_omap2())
 562                 return;
 563
 564         dma_write(eidx, CDEI(lch));
 565         dma_write(fidx, CDFI(lch));
 566 }
 567 EXPORT_SYMBOL(omap_set_dma_dest_index);
 568
 569 void omap_set_dma_dest_data_pack(int lch, int enable)
 570 {
 571         u32 l;
 572
 573         l = dma_read(CSDP(lch));
 574         l &= ~(1 << 13);
 575         if (enable)
 576                 l |= 1 << 13;
 577         dma_write(l, CSDP(lch));
 578 }
 579 EXPORT_SYMBOL(omap_set_dma_dest_data_pack);
 580
 581 void omap_set_dma_dest_burst_mode(int lch, enum omap_dma_burst_mode burst_mode)
 582 {
 583         unsigned int burst = 0;
 584         u32 l;
 585
 586         l = dma_read(CSDP(lch));
 587         l &= ~(0x03 << 14);
 588
 589         switch (burst_mode) {
 590         case OMAP_DMA_DATA_BURST_DIS:
 591                 break;
 592         case OMAP_DMA_DATA_BURST_4:
 593                 if (cpu_class_is_omap2())
 594                         burst = 0x1;
 595                 else
 596                         burst = 0x2;
 597                 break;
 598         case OMAP_DMA_DATA_BURST_8:
 599                 if (cpu_class_is_omap2())
 600                         burst = 0x2;
 601                 else
 602                         burst = 0x3;
 603                 break;
 604         case OMAP_DMA_DATA_BURST_16:
 605                 if (cpu_class_is_omap2()) {
 606                         burst = 0x3;
 607                         break;
 608                 }
 609                 /*
 610                  * OMAP1 don't support burst 16
 611                  * fall through
 612                  */
 613         default:
 614                 printk(KERN_ERR "Invalid DMA burst mode\n");
 615                 BUG();
 616                 return;
 617         }
 618         l |= (burst << 14);
 619         dma_write(l, CSDP(lch));
 620 }
 621 EXPORT_SYMBOL(omap_set_dma_dest_burst_mode);
 622
 623 static inline void omap_enable_channel_irq(int lch)
 624 {
 625         u32 status;
 626
 627         /* Clear CSR */
 628         if (cpu_class_is_omap1())
 629                 status = dma_read(CSR(lch));
 630         else if (cpu_class_is_omap2())
 631                 dma_write(OMAP2_DMA_CSR_CLEAR_MASK, CSR(lch));
 632
 633         /* Enable some nice interrupts. */
 634         dma_write(dma_chan[lch].enabled_irqs, CICR(lch));
 635 }
 636
 637 static void omap_disable_channel_irq(int lch)
 638 {
 639         if (cpu_class_is_omap2())
 640                 dma_write(0, CICR(lch));
 641 }
 642
 643 void omap_enable_dma_irq(int lch, u16 bits)
 644 {
 645         dma_chan[lch].enabled_irqs |= bits;
 646 }
 647 EXPORT_SYMBOL(omap_enable_dma_irq);
 648
 649 void omap_disable_dma_irq(int lch, u16 bits)
 650 {
 651         dma_chan[lch].enabled_irqs &= ~bits;
 652 }
 653 EXPORT_SYMBOL(omap_disable_dma_irq);
 654
 655 static inline void enable_lnk(int lch)
 656 {
 657         u32 l;
 658
 659         l = dma_read(CLNK_CTRL(lch));
 660
 661         if (cpu_class_is_omap1())
 662                 l &= ~(1 << 14);
 663
 664         /* Set the ENABLE_LNK bits */
 665         if (dma_chan[lch].next_lch != -1)
 666                 l = dma_chan[lch].next_lch | (1 << 15);
 667
 668 #ifndef CONFIG_ARCH_OMAP1
 669         if (cpu_class_is_omap2())
 670                 if (dma_chan[lch].next_linked_ch != -1)
 671                         l = dma_chan[lch].next_linked_ch | (1 << 15);
 672 #endif
 673
 674         dma_write(l, CLNK_CTRL(lch));
 675 }
 676
 677 static inline void disable_lnk(int lch)
 678 {
 679         u32 l;
 680
 681         l = dma_read(CLNK_CTRL(lch));
 682
 683         /* Disable interrupts */
 684         if (cpu_class_is_omap1()) {
 685                 dma_write(0, CICR(lch));
 686                 /* Set the STOP_LNK bit */
 687                 l |= 1 << 14;
 688         }
 689
 690         if (cpu_class_is_omap2()) {
 691                 omap_disable_channel_irq(lch);
 692                 /* Clear the ENABLE_LNK bit */
 693                 l &= ~(1 << 15);
 694         }
 695
 696         dma_write(l, CLNK_CTRL(lch));
 697         dma_chan[lch].flags &= ~OMAP_DMA_ACTIVE;
 698 }
 699
 700 static inline void omap2_enable_irq_lch(int lch)
 701 {
 702         u32 val;
 703         unsigned long flags;
 704
 705         if (!cpu_class_is_omap2())
 706                 return;
 707
 708         spin_lock_irqsave(&dma_chan_lock, flags);
 709         val = dma_read(IRQENABLE_L0);
 710         val |= 1 << lch;
 711         dma_write(val, IRQENABLE_L0);
 712         spin_unlock_irqrestore(&dma_chan_lock, flags);
 713 }
 714
 715 static inline void omap2_disable_irq_lch(int lch)
 716 {
 717         u32 val;
 718         unsigned long flags;
 719
 720         if (!cpu_class_is_omap2())
 721                 return;
 722
 723         spin_lock_irqsave(&dma_chan_lock, flags);
 724         val = dma_read(IRQENABLE_L0);
 725         val &= ~(1 << lch);
 726         dma_write(val, IRQENABLE_L0);
 727         spin_unlock_irqrestore(&dma_chan_lock, flags);
 728 }
 729
 730 int omap_request_dma(int dev_id, const char *dev_name,
 731                      void (*callback)(int lch, u16 ch_status, void *data),
 732                      void *data, int *dma_ch_out)
 733 {
 734         int ch, free_ch = -1;
 735         unsigned long flags;
 736         struct omap_dma_lch *chan;
 737
 738         spin_lock_irqsave(&dma_chan_lock, flags);
 739         for (ch = 0; ch < dma_chan_count; ch++) {
 740                 if (free_ch == -1 && dma_chan[ch].dev_id == -1) {
 741                         free_ch = ch;
 742                         if (dev_id == 0)
 743                                 break;
 744                 }
 745         }
 746         if (free_ch == -1) {
 747                 spin_unlock_irqrestore(&dma_chan_lock, flags);
 748                 return -EBUSY;
 749         }
 750         chan = dma_chan + free_ch;
 751         chan->dev_id = dev_id;
 752
 753         if (cpu_class_is_omap1())
 754                 clear_lch_regs(free_ch);
 755
 756         if (cpu_class_is_omap2())
 757                 omap_clear_dma(free_ch);
 758
 759         spin_unlock_irqrestore(&dma_chan_lock, flags);
 760
 761         chan->dev_name = dev_name;
 762         chan->callback = callback;
 763         chan->data = data;
 764         chan->flags = 0;
 765
 766 #ifndef CONFIG_ARCH_OMAP1
 767         if (cpu_class_is_omap2()) {
 768                 chan->chain_id = -1;
 769                 chan->next_linked_ch = -1;
 770         }
 771 #endif
 772
 773         chan->enabled_irqs = OMAP_DMA_DROP_IRQ | OMAP_DMA_BLOCK_IRQ;
 774
 775         if (cpu_class_is_omap1())
 776                 chan->enabled_irqs |= OMAP1_DMA_TOUT_IRQ;
 777         else if (cpu_class_is_omap2())
 778                 chan->enabled_irqs |= OMAP2_DMA_MISALIGNED_ERR_IRQ |
 779                         OMAP2_DMA_TRANS_ERR_IRQ;
 780
 781         if (cpu_is_omap16xx()) {
 782                 /* If the sync device is set, configure it dynamically. */
 783                 if (dev_id != 0) {
 784                         set_gdma_dev(free_ch + 1, dev_id);
 785                         dev_id = free_ch + 1;
 786                 }
 787                 /*
 788                  * Disable the 1510 compatibility mode and set the sync device
 789                  * id.
 790                  */
 791                 dma_write(dev_id | (1 << 10), CCR(free_ch));
 792         } else if (cpu_is_omap7xx() || cpu_is_omap15xx()) {
 793                 dma_write(dev_id, CCR(free_ch));
 794         }
 795
 796         if (cpu_class_is_omap2()) {
 797                 omap2_enable_irq_lch(free_ch);
 798                 omap_enable_channel_irq(free_ch);
 799                 /* Clear the CSR register and IRQ status register */
 800                 dma_write(OMAP2_DMA_CSR_CLEAR_MASK, CSR(free_ch));
 801                 dma_write(1 << free_ch, IRQSTATUS_L0);
 802         }
 803
 804         *dma_ch_out = free_ch;
 805
 806         return 0;
 807 }
 808 EXPORT_SYMBOL(omap_request_dma);
 809
 810 void omap_free_dma(int lch)
 811 {
 812         unsigned long flags;
 813
 814         if (dma_chan[lch].dev_id == -1) {
 815                 pr_err("omap_dma: trying to free unallocated DMA channel %d\n",
 816                        lch);
 817                 return;
 818         }
 819
 820         if (cpu_class_is_omap1()) {
 821                 /* Disable all DMA interrupts for the channel. */
 822                 dma_write(0, CICR(lch));
 823                 /* Make sure the DMA transfer is stopped. */
 824                 dma_write(0, CCR(lch));
 825         }
 826
 827         if (cpu_class_is_omap2()) {
 828                 omap2_disable_irq_lch(lch);
 829
 830                 /* Clear the CSR register and IRQ status register */
 831                 dma_write(OMAP2_DMA_CSR_CLEAR_MASK, CSR(lch));
 832                 dma_write(1 << lch, IRQSTATUS_L0);
 833
 834                 /* Disable all DMA interrupts for the channel. */
 835                 dma_write(0, CICR(lch));
 836
 837                 /* Make sure the DMA transfer is stopped. */
 838                 dma_write(0, CCR(lch));
 839                 omap_clear_dma(lch);
 840         }
 841
 842         spin_lock_irqsave(&dma_chan_lock, flags);
 843         dma_chan[lch].dev_id = -1;
 844         dma_chan[lch].next_lch = -1;
 845         dma_chan[lch].callback = NULL;
 846         spin_unlock_irqrestore(&dma_chan_lock, flags);
 847 }
 848 EXPORT_SYMBOL(omap_free_dma);
 849
 850 /**
 851  * @brief omap_dma_set_global_params : Set global priority settings for dma
 852  *
 853  * @param arb_rate
 854  * @param max_fifo_depth
 855  * @param tparams - Number of threads to reserve : DMA_THREAD_RESERVE_NORM
 856  *                                                 DMA_THREAD_RESERVE_ONET
 857  *                                                 DMA_THREAD_RESERVE_TWOT
 858  *                                                 DMA_THREAD_RESERVE_THREET
 859  */
 860 void
 861 omap_dma_set_global_params(int arb_rate, int max_fifo_depth, int tparams)
 862 {
 863         u32 reg;
 864
 865         if (!cpu_class_is_omap2()) {
 866                 printk(KERN_ERR "FIXME: no %s on 15xx/16xx\n", __func__);
 867                 return;
 868         }
 869
 870         if (max_fifo_depth == 0)
 871                 max_fifo_depth = 1;
 872         if (arb_rate == 0)
 873                 arb_rate = 1;
 874
 875         reg = 0xff & max_fifo_depth;
 876         reg |= (0x3 & tparams) << 12;
 877         reg |= (arb_rate & 0xff) << 16;
 878
 879         dma_write(reg, GCR);
 880 }
 881 EXPORT_SYMBOL(omap_dma_set_global_params);
 882
 883 /**
 884  * @brief omap_dma_set_prio_lch : Set channel wise priority settings
 885  *
 886  * @param lch
 887  * @param read_prio - Read priority
 888  * @param write_prio - Write priority
 889  * Both of the above can be set with one of the following values :
 890  *      DMA_CH_PRIO_HIGH/DMA_CH_PRIO_LOW
 891  */
 892 int
 893 omap_dma_set_prio_lch(int lch, unsigned char read_prio,
 894                       unsigned char write_prio)
 895 {
 896         u32 l;
 897
 898         if (unlikely((lch < 0 || lch >= dma_lch_count))) {
 899                 printk(KERN_ERR "Invalid channel id\n");
 900                 return -EINVAL;
 901         }
 902         l = dma_read(CCR(lch));
 903         l &= ~((1 << 6) | (1 << 26));
 904         if (cpu_is_omap2430() || cpu_is_omap34xx() ||  cpu_is_omap44xx())
 905                 l |= ((read_prio & 0x1) << 6) | ((write_prio & 0x1) << 26);
 906         else
 907                 l |= ((read_prio & 0x1) << 6);
 908
 909         dma_write(l, CCR(lch));
 910
 911         return 0;
 912 }
 913 EXPORT_SYMBOL(omap_dma_set_prio_lch);
 914
 915 /*
 916  * Clears any DMA state so the DMA engine is ready to restart with new buffers
 917  * through omap_start_dma(). Any buffers in flight are discarded.
 918  */
 919 void omap_clear_dma(int lch)
 920 {
 921         unsigned long flags;
 922
 923         local_irq_save(flags);
 924
 925         if (cpu_class_is_omap1()) {
 926                 u32 l;
 927
 928                 l = dma_read(CCR(lch));
 929                 l &= ~OMAP_DMA_CCR_EN;
 930                 dma_write(l, CCR(lch));
 931
 932                 /* Clear pending interrupts */
 933                 l = dma_read(CSR(lch));
 934         }
 935
 936         if (cpu_class_is_omap2()) {
 937                 int i;
 938                 void __iomem *lch_base = omap_dma_base + OMAP_DMA4_CH_BASE(lch);
 939                 for (i = 0; i < 0x44; i += 4)
 940                         __raw_writel(0, lch_base + i);
 941         }
 942
 943         local_irq_restore(flags);
 944 }
 945 EXPORT_SYMBOL(omap_clear_dma);
 946
 947 void omap_start_dma(int lch)
 948 {
 949         u32 l;
 950
 951         /*
 952          * The CPC/CDAC register needs to be initialized to zero
 953          * before starting dma transfer.
 954          */
 955         if (cpu_is_omap15xx())
 956                 dma_write(0, CPC(lch));
 957         else
 958                 dma_write(0, CDAC(lch));
 959
 960         if (!omap_dma_in_1510_mode() && dma_chan[lch].next_lch != -1) {
 961                 int next_lch, cur_lch;
 962                 char dma_chan_link_map[OMAP_DMA4_LOGICAL_DMA_CH_COUNT];
 963
 964                 dma_chan_link_map[lch] = 1;
 965                 /* Set the link register of the first channel */
 966                 enable_lnk(lch);
 967
 968                 memset(dma_chan_link_map, 0, sizeof(dma_chan_link_map));
 969                 cur_lch = dma_chan[lch].next_lch;
 970                 do {
 971                         next_lch = dma_chan[cur_lch].next_lch;
 972
 973                         /* The loop case: we've been here already */
 974                         if (dma_chan_link_map[cur_lch])
 975                                 break;
 976                         /* Mark the current channel */
 977                         dma_chan_link_map[cur_lch] = 1;
 978
 979                         enable_lnk(cur_lch);
 980                         omap_enable_channel_irq(cur_lch);
 981
 982                         cur_lch = next_lch;
 983                 } while (next_lch != -1);
 984         } else if (cpu_is_omap242x() ||
 985                 (cpu_is_omap243x() &&  omap_type() <= OMAP2430_REV_ES1_0)) {
 986
 987                 /* Errata: Need to write lch even if not using chaining */
 988                 dma_write(lch, CLNK_CTRL(lch));
 989         }
 990
 991         omap_enable_channel_irq(lch);
 992
 993         l = dma_read(CCR(lch));
 994
 995         /*
 996          * Errata: On ES2.0 BUFFERING disable must be set.
 997          * This will always fail on ES1.0
 998          */
 999         if (cpu_is_omap24xx())
1000                 l |= OMAP_DMA_CCR_EN;
1001
1002         l |= OMAP_DMA_CCR_EN;
1003         dma_write(l, CCR(lch));
1004
1005         dma_chan[lch].flags |= OMAP_DMA_ACTIVE;
1006 }
1007 EXPORT_SYMBOL(omap_start_dma);
1008
1009 void omap_stop_dma(int lch)
1010 {
1011         u32 l;
1012
1013         /* Disable all interrupts on the channel */
1014         if (cpu_class_is_omap1())
1015                 dma_write(0, CICR(lch));
1016
1017         l = dma_read(CCR(lch));
1018         l &= ~OMAP_DMA_CCR_EN;
1019         dma_write(l, CCR(lch));
1020
1021         if (!omap_dma_in_1510_mode() && dma_chan[lch].next_lch != -1) {
1022                 int next_lch, cur_lch = lch;
1023                 char dma_chan_link_map[OMAP_DMA4_LOGICAL_DMA_CH_COUNT];
1024
1025                 memset(dma_chan_link_map, 0, sizeof(dma_chan_link_map));
1026                 do {
1027                         /* The loop case: we've been here already */
1028                         if (dma_chan_link_map[cur_lch])
1029                                 break;
1030                         /* Mark the current channel */
1031                         dma_chan_link_map[cur_lch] = 1;
1032
1033                         disable_lnk(cur_lch);
1034
1035                         next_lch = dma_chan[cur_lch].next_lch;
1036                         cur_lch = next_lch;
1037                 } while (next_lch != -1);
1038         }
1039
1040         dma_chan[lch].flags &= ~OMAP_DMA_ACTIVE;
1041 }
1042 EXPORT_SYMBOL(omap_stop_dma);
1043
1044 /*
1045  * Allows changing the DMA callback function or data. This may be needed if
1046  * the driver shares a single DMA channel for multiple dma triggers.
1047  */
1048 int omap_set_dma_callback(int lch,
1049                           void (*callback)(int lch, u16 ch_status, void *data),
1050                           void *data)
1051 {
1052         unsigned long flags;
1053
1054         if (lch < 0)
1055                 return -ENODEV;
1056
1057         spin_lock_irqsave(&dma_chan_lock, flags);
1058         if (dma_chan[lch].dev_id == -1) {
1059                 printk(KERN_ERR "DMA callback for not set for free channel\n");
1060                 spin_unlock_irqrestore(&dma_chan_lock, flags);
1061                 return -EINVAL;
1062         }
1063         dma_chan[lch].callback = callback;
1064         dma_chan[lch].data = data;
1065         spin_unlock_irqrestore(&dma_chan_lock, flags);
1066
1067         return 0;
1068 }
1069 EXPORT_SYMBOL(omap_set_dma_callback);
1070
1071 /*
1072  * Returns current physical source address for the given DMA channel.
1073  * If the channel is running the caller must disable interrupts prior calling
1074  * this function and process the returned value before re-enabling interrupt to
1075  * prevent races with the interrupt handler. Note that in continuous mode there
1076  * is a chance for CSSA_L register overflow inbetween the two reads resulting
1077  * in incorrect return value.
1078  */
1079 dma_addr_t omap_get_dma_src_pos(int lch)
1080 {
1081         dma_addr_t offset = 0;
1082
1083         if (cpu_is_omap15xx())
1084                 offset = dma_read(CPC(lch));
1085         else
1086                 offset = dma_read(CSAC(lch));
1087
1088         /*
1089          * omap 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is
1090          * read before the DMA controller finished disabling the channel.
1091          */
1092         if (!cpu_is_omap15xx() && offset == 0)
1093                 offset = dma_read(CSAC(lch));
1094
1095         if (cpu_class_is_omap1())
1096                 offset |= (dma_read(CSSA_U(lch)) << 16);
1097
1098         return offset;
1099 }
1100 EXPORT_SYMBOL(omap_get_dma_src_pos);
1101
1102 /*
1103  * Returns current physical destination address for the given DMA channel.
1104  * If the channel is running the caller must disable interrupts prior calling
1105  * this function and process the returned value before re-enabling interrupt to
1106  * prevent races with the interrupt handler. Note that in continuous mode there
1107  * is a chance for CDSA_L register overflow inbetween the two reads resulting
1108  * in incorrect return value.
1109  */
1110 dma_addr_t omap_get_dma_dst_pos(int lch)
1111 {
1112         dma_addr_t offset = 0;
1113
1114         if (cpu_is_omap15xx())
1115                 offset = dma_read(CPC(lch));
1116         else
1117                 offset = dma_read(CDAC(lch));
1118
1119         /*
1120          * omap 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is
1121          * read before the DMA controller finished disabling the channel.
1122          */
1123         if (!cpu_is_omap15xx() && offset == 0)
1124                 offset = dma_read(CDAC(lch));
1125
1126         if (cpu_class_is_omap1())
1127                 offset |= (dma_read(CDSA_U(lch)) << 16);
1128
1129         return offset;
1130 }
1131 EXPORT_SYMBOL(omap_get_dma_dst_pos);
1132
1133 int omap_get_dma_active_status(int lch)
1134 {
1135         return (dma_read(CCR(lch)) & OMAP_DMA_CCR_EN) != 0;
1136 }
1137 EXPORT_SYMBOL(omap_get_dma_active_status);
1138
1139 int omap_dma_running(void)
1140 {
1141         int lch;
1142
1143         if (cpu_class_is_omap1())
1144                 if (omap_lcd_dma_running())
1145                         return 1;
1146
1147         for (lch = 0; lch < dma_chan_count; lch++)
1148                 if (dma_read(CCR(lch)) & OMAP_DMA_CCR_EN)
1149                         return 1;
1150
1151         return 0;
1152 }
1153
1154 /*
1155  * lch_queue DMA will start right after lch_head one is finished.
1156  * For this DMA link to start, you still need to start (see omap_start_dma)
1157  * the first one. That will fire up the entire queue.
1158  */
1159 void omap_dma_link_lch(int lch_head, int lch_queue)
1160 {
1161         if (omap_dma_in_1510_mode()) {
1162                 if (lch_head == lch_queue) {
1163                         dma_write(dma_read(CCR(lch_head)) | (3 << 8),
1164                                                                 CCR(lch_head));
1165                         return;
1166                 }
1167                 printk(KERN_ERR "DMA linking is not supported in 1510 mode\n");
1168                 BUG();
1169                 return;
1170         }
1171
1172         if ((dma_chan[lch_head].dev_id == -1) ||
1173             (dma_chan[lch_queue].dev_id == -1)) {
1174                 printk(KERN_ERR "omap_dma: trying to link "
1175                        "non requested channels\n");
1176                 dump_stack();
1177         }
1178
1179         dma_chan[lch_head].next_lch = lch_queue;
1180 }
1181 EXPORT_SYMBOL(omap_dma_link_lch);
1182
1183 /*
1184  * Once the DMA queue is stopped, we can destroy it.
1185  */
1186 void omap_dma_unlink_lch(int lch_head, int lch_queue)
1187 {
1188         if (omap_dma_in_1510_mode()) {
1189                 if (lch_head == lch_queue) {
1190                         dma_write(dma_read(CCR(lch_head)) & ~(3 << 8),
1191                                                                 CCR(lch_head));
1192                         return;
1193                 }
1194                 printk(KERN_ERR "DMA linking is not supported in 1510 mode\n");
1195                 BUG();
1196                 return;
1197         }
1198
1199         if (dma_chan[lch_head].next_lch != lch_queue ||
1200             dma_chan[lch_head].next_lch == -1) {
1201                 printk(KERN_ERR "omap_dma: trying to unlink "
1202                        "non linked channels\n");
1203                 dump_stack();
1204         }
1205
1206         if ((dma_chan[lch_head].flags & OMAP_DMA_ACTIVE) ||
1207             (dma_chan[lch_queue].flags & OMAP_DMA_ACTIVE)) {
1208                 printk(KERN_ERR "omap_dma: You need to stop the DMA channels "
1209                        "before unlinking\n");
1210                 dump_stack();
1211         }
1212
1213         dma_chan[lch_head].next_lch = -1;
1214 }
1215 EXPORT_SYMBOL(omap_dma_unlink_lch);
1216
1217 /*----------------------------------------------------------------------------*/
1218
1219 #ifndef CONFIG_ARCH_OMAP1
1220 /* Create chain of DMA channesls */
1221 static void create_dma_lch_chain(int lch_head, int lch_queue)
1222 {
1223         u32 l;
1224
1225         /* Check if this is the first link in chain */
1226         if (dma_chan[lch_head].next_linked_ch == -1) {
1227                 dma_chan[lch_head].next_linked_ch = lch_queue;
1228                 dma_chan[lch_head].prev_linked_ch = lch_queue;
1229                 dma_chan[lch_queue].next_linked_ch = lch_head;
1230                 dma_chan[lch_queue].prev_linked_ch = lch_head;
1231         }
1232
1233         /* a link exists, link the new channel in circular chain */
1234         else {
1235                 dma_chan[lch_queue].next_linked_ch =
1236                                         dma_chan[lch_head].next_linked_ch;
1237                 dma_chan[lch_queue].prev_linked_ch = lch_head;
1238                 dma_chan[lch_head].next_linked_ch = lch_queue;
1239                 dma_chan[dma_chan[lch_queue].next_linked_ch].prev_linked_ch =
1240                                         lch_queue;
1241         }
1242
1243         l = dma_read(CLNK_CTRL(lch_head));
1244         l &= ~(0x1f);
1245         l |= lch_queue;
1246         dma_write(l, CLNK_CTRL(lch_head));
1247
1248         l = dma_read(CLNK_CTRL(lch_queue));
1249         l &= ~(0x1f);
1250         l |= (dma_chan[lch_queue].next_linked_ch);
1251         dma_write(l, CLNK_CTRL(lch_queue));
1252 }
1253
1254 /**
1255  * @brief omap_request_dma_chain : Request a chain of DMA channels
1256  *
1257  * @param dev_id - Device id using the dma channel
1258  * @param dev_name - Device name
1259  * @param callback - Call back function
1260  * @chain_id -
1261  * @no_of_chans - Number of channels requested
1262  * @chain_mode - Dynamic or static chaining : OMAP_DMA_STATIC_CHAIN
1263  *                                            OMAP_DMA_DYNAMIC_CHAIN
1264  * @params - Channel parameters
1265  *
1266  * @return - Success : 0
1267  *           Failure: -EINVAL/-ENOMEM
1268  */
1269 int omap_request_dma_chain(int dev_id, const char *dev_name,
1270                            void (*callback) (int lch, u16 ch_status,
1271                                              void *data),
1272                            int *chain_id, int no_of_chans, int chain_mode,
1273                            struct omap_dma_channel_params params)
1274 {
1275         int *channels;
1276         int i, err;
1277
1278         /* Is the chain mode valid ? */
1279         if (chain_mode != OMAP_DMA_STATIC_CHAIN
1280                         && chain_mode != OMAP_DMA_DYNAMIC_CHAIN) {
1281                 printk(KERN_ERR "Invalid chain mode requested\n");
1282                 return -EINVAL;
1283         }
1284
1285         if (unlikely((no_of_chans < 1
1286                         || no_of_chans > dma_lch_count))) {
1287                 printk(KERN_ERR "Invalid Number of channels requested\n");
1288                 return -EINVAL;
1289         }
1290
1291         /*
1292          * Allocate a queue to maintain the status of the channels
1293          * in the chain
1294          */
1295         channels = kmalloc(sizeof(*channels) * no_of_chans, GFP_KERNEL);
1296         if (channels == NULL) {
1297                 printk(KERN_ERR "omap_dma: No memory for channel queue\n");
1298                 return -ENOMEM;
1299         }
1300
1301         /* request and reserve DMA channels for the chain */
1302         for (i = 0; i < no_of_chans; i++) {
1303                 err = omap_request_dma(dev_id, dev_name,
1304                                         callback, NULL, &channels[i]);
1305                 if (err < 0) {
1306                         int j;
1307                         for (j = 0; j < i; j++)
1308                                 omap_free_dma(channels[j]);
1309                         kfree(channels);
1310                         printk(KERN_ERR "omap_dma: Request failed %d\n", err);
1311                         return err;
1312                 }
1313                 dma_chan[channels[i]].prev_linked_ch = -1;
1314                 dma_chan[channels[i]].state = DMA_CH_NOTSTARTED;
1315
1316                 /*
1317                  * Allowing client drivers to set common parameters now,
1318                  * so that later only relevant (src_start, dest_start
1319                  * and element count) can be set
1320                  */
1321                 omap_set_dma_params(channels[i], &params);
1322         }
1323
1324         *chain_id = channels[0];
1325         dma_linked_lch[*chain_id].linked_dmach_q = channels;
1326         dma_linked_lch[*chain_id].chain_mode = chain_mode;
1327         dma_linked_lch[*chain_id].chain_state = DMA_CHAIN_NOTSTARTED;
1328         dma_linked_lch[*chain_id].no_of_lchs_linked = no_of_chans;
1329
1330         for (i = 0; i < no_of_chans; i++)
1331                 dma_chan[channels[i]].chain_id = *chain_id;
1332
1333         /* Reset the Queue pointers */
1334         OMAP_DMA_CHAIN_QINIT(*chain_id);
1335
1336         /* Set up the chain */
1337         if (no_of_chans == 1)
1338                 create_dma_lch_chain(channels[0], channels[0]);
1339         else {
1340                 for (i = 0; i < (no_of_chans - 1); i++)
1341                         create_dma_lch_chain(channels[i], channels[i + 1]);
1342         }
1343
1344         return 0;
1345 }
1346 EXPORT_SYMBOL(omap_request_dma_chain);
1347
1348 /**
1349  * @brief omap_modify_dma_chain_param : Modify the chain's params - Modify the
1350  * params after setting it. Dont do this while dma is running!!
1351  *
1352  * @param chain_id - Chained logical channel id.
1353  * @param params
1354  *
1355  * @return - Success : 0
1356  *           Failure : -EINVAL
1357  */
1358 int omap_modify_dma_chain_params(int chain_id,
1359                                 struct omap_dma_channel_params params)
1360 {
1361         int *channels;
1362         u32 i;
1363
1364         /* Check for input params */
1365         if (unlikely((chain_id < 0
1366                         || chain_id >= dma_lch_count))) {
1367                 printk(KERN_ERR "Invalid chain id\n");
1368                 return -EINVAL;
1369         }
1370
1371         /* Check if the chain exists */
1372         if (dma_linked_lch[chain_id].linked_dmach_q == NULL) {
1373                 printk(KERN_ERR "Chain doesn't exists\n");
1374                 return -EINVAL;
1375         }
1376         channels = dma_linked_lch[chain_id].linked_dmach_q;
1377
1378         for (i = 0; i < dma_linked_lch[chain_id].no_of_lchs_linked; i++) {
1379                 /*
1380                  * Allowing client drivers to set common parameters now,
1381                  * so that later only relevant (src_start, dest_start
1382                  * and element count) can be set
1383                  */
1384                 omap_set_dma_params(channels[i], &params);
1385         }
1386
1387         return 0;
1388 }
1389 EXPORT_SYMBOL(omap_modify_dma_chain_params);
1390
1391 /**
1392  * @brief omap_free_dma_chain - Free all the logical channels in a chain.
1393  *
1394  * @param chain_id
1395  *
1396  * @return - Success : 0
1397  *           Failure : -EINVAL
1398  */
1399 int omap_free_dma_chain(int chain_id)
1400 {
1401         int *channels;
1402         u32 i;
1403
1404         /* Check for input params */
1405         if (unlikely((chain_id < 0 || chain_id >= dma_lch_count))) {
1406                 printk(KERN_ERR "Invalid chain id\n");
1407                 return -EINVAL;
1408         }
1409
1410         /* Check if the chain exists */
1411         if (dma_linked_lch[chain_id].linked_dmach_q == NULL) {
1412                 printk(KERN_ERR "Chain doesn't exists\n");
1413                 return -EINVAL;
1414         }
1415
1416         channels = dma_linked_lch[chain_id].linked_dmach_q;
1417         for (i = 0; i < dma_linked_lch[chain_id].no_of_lchs_linked; i++) {
1418                 dma_chan[channels[i]].next_linked_ch = -1;
1419                 dma_chan[channels[i]].prev_linked_ch = -1;
1420                 dma_chan[channels[i]].chain_id = -1;
1421                 dma_chan[channels[i]].state = DMA_CH_NOTSTARTED;
1422                 omap_free_dma(channels[i]);
1423         }
1424
1425         kfree(channels);
1426
1427         dma_linked_lch[chain_id].linked_dmach_q = NULL;
1428         dma_linked_lch[chain_id].chain_mode = -1;
1429         dma_linked_lch[chain_id].chain_state = -1;
1430
1431         return (0);
1432 }
1433 EXPORT_SYMBOL(omap_free_dma_chain);
1434
1435 /**
1436  * @brief omap_dma_chain_status - Check if the chain is in
1437  * active / inactive state.
1438  * @param chain_id
1439  *
1440  * @return - Success : OMAP_DMA_CHAIN_ACTIVE/OMAP_DMA_CHAIN_INACTIVE
1441  *           Failure : -EINVAL
1442  */
1443 int omap_dma_chain_status(int chain_id)
1444 {
1445         /* Check for input params */
1446         if (unlikely((chain_id < 0 || chain_id >= dma_lch_count))) {
1447                 printk(KERN_ERR "Invalid chain id\n");
1448                 return -EINVAL;
1449         }
1450
1451         /* Check if the chain exists */
1452         if (dma_linked_lch[chain_id].linked_dmach_q == NULL) {
1453                 printk(KERN_ERR "Chain doesn't exists\n");
1454                 return -EINVAL;
1455         }
1456         pr_debug("CHAINID=%d, qcnt=%d\n", chain_id,
1457                         dma_linked_lch[chain_id].q_count);
1458
1459         if (OMAP_DMA_CHAIN_QEMPTY(chain_id))
1460                 return OMAP_DMA_CHAIN_INACTIVE;
1461
1462         return OMAP_DMA_CHAIN_ACTIVE;
1463 }
1464 EXPORT_SYMBOL(omap_dma_chain_status);
1465
1466 /**
1467  * @brief omap_dma_chain_a_transfer - Get a free channel from a chain,
1468  * set the params and start the transfer.
1469  *
1470  * @param chain_id
1471  * @param src_start - buffer start address
1472  * @param dest_start - Dest address
1473  * @param elem_count
1474  * @param frame_count
1475  * @param callbk_data - channel callback parameter data.
1476  *
1477  * @return  - Success : 0
1478  *            Failure: -EINVAL/-EBUSY
1479  */
1480 int omap_dma_chain_a_transfer(int chain_id, int src_start, int dest_start,
1481                         int elem_count, int frame_count, void *callbk_data)
1482 {
1483         int *channels;
1484         u32 l, lch;
1485         int start_dma = 0;
1486
1487         /*
1488          * if buffer size is less than 1 then there is
1489          * no use of starting the chain
1490          */
1491         if (elem_count < 1) {
1492                 printk(KERN_ERR "Invalid buffer size\n");
1493                 return -EINVAL;
1494         }
1495
1496         /* Check for input params */
1497         if (unlikely((chain_id < 0
1498                         || chain_id >= dma_lch_count))) {
1499                 printk(KERN_ERR "Invalid chain id\n");
1500                 return -EINVAL;
1501         }
1502
1503         /* Check if the chain exists */
1504         if (dma_linked_lch[chain_id].linked_dmach_q == NULL) {
1505                 printk(KERN_ERR "Chain doesn't exist\n");
1506                 return -EINVAL;
1507         }
1508
1509         /* Check if all the channels in chain are in use */
1510         if (OMAP_DMA_CHAIN_QFULL(chain_id))
1511                 return -EBUSY;
1512
1513         /* Frame count may be negative in case of indexed transfers */
1514         channels = dma_linked_lch[chain_id].linked_dmach_q;
1515
1516         /* Get a free channel */
1517         lch = channels[dma_linked_lch[chain_id].q_tail];
1518
1519         /* Store the callback data */
1520         dma_chan[lch].data = callbk_data;
1521
1522         /* Increment the q_tail */
1523         OMAP_DMA_CHAIN_INCQTAIL(chain_id);
1524
1525         /* Set the params to the free channel */
1526         if (src_start != 0)
1527                 dma_write(src_start, CSSA(lch));
1528         if (dest_start != 0)
1529                 dma_write(dest_start, CDSA(lch));
1530
1531         /* Write the buffer size */
1532         dma_write(elem_count, CEN(lch));
1533         dma_write(frame_count, CFN(lch));
1534
1535         /*
1536          * If the chain is dynamically linked,
1537          * then we may have to start the chain if its not active
1538          */
1539         if (dma_linked_lch[chain_id].chain_mode == OMAP_DMA_DYNAMIC_CHAIN) {
1540
1541                 /*
1542                  * In Dynamic chain, if the chain is not started,
1543                  * queue the channel
1544                  */
1545                 if (dma_linked_lch[chain_id].chain_state ==
1546                                                 DMA_CHAIN_NOTSTARTED) {
1547                         /* Enable the link in previous channel */
1548                         if (dma_chan[dma_chan[lch].prev_linked_ch].state ==
1549                                                                 DMA_CH_QUEUED)
1550                                 enable_lnk(dma_chan[lch].prev_linked_ch);
1551                         dma_chan[lch].state = DMA_CH_QUEUED;
1552                 }
1553
1554                 /*
1555                  * Chain is already started, make sure its active,
1556                  * if not then start the chain
1557                  */
1558                 else {
1559                         start_dma = 1;
1560
1561                         if (dma_chan[dma_chan[lch].prev_linked_ch].state ==
1562                                                         DMA_CH_STARTED) {
1563                                 enable_lnk(dma_chan[lch].prev_linked_ch);
1564                                 dma_chan[lch].state = DMA_CH_QUEUED;
1565                                 start_dma = 0;
1566                                 if (0 == ((1 << 7) & dma_read(
1567                                         CCR(dma_chan[lch].prev_linked_ch)))) {
1568                                         disable_lnk(dma_chan[lch].
1569                                                     prev_linked_ch);
1570                                         pr_debug("\n prev ch is stopped\n");
1571                                         start_dma = 1;
1572                                 }
1573                         }
1574
1575                         else if (dma_chan[dma_chan[lch].prev_linked_ch].state
1576                                                         == DMA_CH_QUEUED) {
1577                                 enable_lnk(dma_chan[lch].prev_linked_ch);
1578                                 dma_chan[lch].state = DMA_CH_QUEUED;
1579                                 start_dma = 0;
1580                         }
1581                         omap_enable_channel_irq(lch);
1582
1583                         l = dma_read(CCR(lch));
1584
1585                         if ((0 == (l & (1 << 24))))
1586                                 l &= ~(1 << 25);
1587                         else
1588                                 l |= (1 << 25);
1589                         if (start_dma == 1) {
1590                                 if (0 == (l & (1 << 7))) {
1591                                         l |= (1 << 7);
1592                                         dma_chan[lch].state = DMA_CH_STARTED;
1593                                         pr_debug("starting %d\n", lch);
1594                                         dma_write(l, CCR(lch));
1595                                 } else
1596                                         start_dma = 0;
1597                         } else {
1598                                 if (0 == (l & (1 << 7)))
1599                                         dma_write(l, CCR(lch));
1600                         }
1601                         dma_chan[lch].flags |= OMAP_DMA_ACTIVE;
1602                 }
1603         }
1604
1605         return 0;
1606 }
1607 EXPORT_SYMBOL(omap_dma_chain_a_transfer);
1608
1609 /**
1610  * @brief omap_start_dma_chain_transfers - Start the chain
1611  *
1612  * @param chain_id
1613  *
1614  * @return - Success : 0
1615  *           Failure : -EINVAL/-EBUSY
1616  */
1617 int omap_start_dma_chain_transfers(int chain_id)
1618 {
1619         int *channels;
1620         u32 l, i;
1621
1622         if (unlikely((chain_id < 0 || chain_id >= dma_lch_count))) {
1623                 printk(KERN_ERR "Invalid chain id\n");
1624                 return -EINVAL;
1625         }
1626
1627         channels = dma_linked_lch[chain_id].linked_dmach_q;
1628
1629         if (dma_linked_lch[channels[0]].chain_state == DMA_CHAIN_STARTED) {
1630                 printk(KERN_ERR "Chain is already started\n");
1631                 return -EBUSY;
1632         }
1633
1634         if (dma_linked_lch[chain_id].chain_mode == OMAP_DMA_STATIC_CHAIN) {
1635                 for (i = 0; i < dma_linked_lch[chain_id].no_of_lchs_linked;
1636                                                                         i++) {
1637                         enable_lnk(channels[i]);
1638                         omap_enable_channel_irq(channels[i]);
1639                 }
1640         } else {
1641                 omap_enable_channel_irq(channels[0]);
1642         }
1643
1644         l = dma_read(CCR(channels[0]));
1645         l |= (1 << 7);
1646         dma_linked_lch[chain_id].chain_state = DMA_CHAIN_STARTED;
1647         dma_chan[channels[0]].state = DMA_CH_STARTED;
1648
1649         if ((0 == (l & (1 << 24))))
1650                 l &= ~(1 << 25);
1651         else
1652                 l |= (1 << 25);
1653         dma_write(l, CCR(channels[0]));
1654
1655         dma_chan[channels[0]].flags |= OMAP_DMA_ACTIVE;
1656
1657         return 0;
1658 }
1659 EXPORT_SYMBOL(omap_start_dma_chain_transfers);
1660
1661 /**
1662  * @brief omap_stop_dma_chain_transfers - Stop the dma transfer of a chain.
1663  *
1664  * @param chain_id
1665  *
1666  * @return - Success : 0
1667  *           Failure : EINVAL
1668  */
1669 int omap_stop_dma_chain_transfers(int chain_id)
1670 {
1671         int *channels;
1672         u32 l, i;
1673         u32 sys_cf;
1674
1675         /* Check for input params */
1676         if (unlikely((chain_id < 0 || chain_id >= dma_lch_count))) {
1677                 printk(KERN_ERR "Invalid chain id\n");
1678                 return -EINVAL;
1679         }
1680
1681         /* Check if the chain exists */
1682         if (dma_linked_lch[chain_id].linked_dmach_q == NULL) {
1683                 printk(KERN_ERR "Chain doesn't exists\n");
1684                 return -EINVAL;
1685         }
1686         channels = dma_linked_lch[chain_id].linked_dmach_q;
1687
1688         /*
1689          * DMA Errata:
1690          * Special programming model needed to disable DMA before end of block
1691          */
1692         sys_cf = dma_read(OCP_SYSCONFIG);
1693         l = sys_cf;
1694         /* Middle mode reg set no Standby */
1695         l &= ~((1 << 12)|(1 << 13));
1696         dma_write(l, OCP_SYSCONFIG);
1697
1698         for (i = 0; i < dma_linked_lch[chain_id].no_of_lchs_linked; i++) {
1699
1700                 /* Stop the Channel transmission */
1701                 l = dma_read(CCR(channels[i]));
1702                 l &= ~(1 << 7);
1703                 dma_write(l, CCR(channels[i]));
1704
1705                 /* Disable the link in all the channels */
1706                 disable_lnk(channels[i]);
1707                 dma_chan[channels[i]].state = DMA_CH_NOTSTARTED;
1708
1709         }
1710         dma_linked_lch[chain_id].chain_state = DMA_CHAIN_NOTSTARTED;
1711
1712         /* Reset the Queue pointers */
1713         OMAP_DMA_CHAIN_QINIT(chain_id);
1714
1715         /* Errata - put in the old value */
1716         dma_write(sys_cf, OCP_SYSCONFIG);
1717
1718         return 0;
1719 }
1720 EXPORT_SYMBOL(omap_stop_dma_chain_transfers);
1721
1722 /* Get the index of the ongoing DMA in chain */
1723 /**
1724  * @brief omap_get_dma_chain_index - Get the element and frame index
1725  * of the ongoing DMA in chain
1726  *
1727  * @param chain_id
1728  * @param ei - Element index
1729  * @param fi - Frame index
1730  *
1731  * @return - Success : 0
1732  *           Failure : -EINVAL
1733  */
1734 int omap_get_dma_chain_index(int chain_id, int *ei, int *fi)
1735 {
1736         int lch;
1737         int *channels;
1738
1739         /* Check for input params */
1740         if (unlikely((chain_id < 0 || chain_id >= dma_lch_count))) {
1741                 printk(KERN_ERR "Invalid chain id\n");
1742                 return -EINVAL;
1743         }
1744
1745         /* Check if the chain exists */
1746         if (dma_linked_lch[chain_id].linked_dmach_q == NULL) {
1747                 printk(KERN_ERR "Chain doesn't exists\n");
1748                 return -EINVAL;
1749         }
1750         if ((!ei) || (!fi))
1751                 return -EINVAL;
1752
1753         channels = dma_linked_lch[chain_id].linked_dmach_q;
1754
1755         /* Get the current channel */
1756         lch = channels[dma_linked_lch[chain_id].q_head];
1757
1758         *ei = dma_read(CCEN(lch));
1759         *fi = dma_read(CCFN(lch));
1760
1761         return 0;
1762 }
1763 EXPORT_SYMBOL(omap_get_dma_chain_index);
1764
1765 /**
1766  * @brief omap_get_dma_chain_dst_pos - Get the destination position of the
1767  * ongoing DMA in chain
1768  *
1769  * @param chain_id
1770  *
1771  * @return - Success : Destination position
1772  *           Failure : -EINVAL
1773  */
1774 int omap_get_dma_chain_dst_pos(int chain_id)
1775 {
1776         int lch;
1777         int *channels;
1778
1779         /* Check for input params */
1780         if (unlikely((chain_id < 0 || chain_id >= dma_lch_count))) {
1781                 printk(KERN_ERR "Invalid chain id\n");
1782                 return -EINVAL;
1783         }
1784
1785         /* Check if the chain exists */
1786         if (dma_linked_lch[chain_id].linked_dmach_q == NULL) {
1787                 printk(KERN_ERR "Chain doesn't exists\n");
1788                 return -EINVAL;
1789         }
1790
1791         channels = dma_linked_lch[chain_id].linked_dmach_q;
1792
1793         /* Get the current channel */
1794         lch = channels[dma_linked_lch[chain_id].q_head];
1795
1796         return dma_read(CDAC(lch));
1797 }
1798 EXPORT_SYMBOL(omap_get_dma_chain_dst_pos);
1799
1800 /**
1801  * @brief omap_get_dma_chain_src_pos - Get the source position
1802  * of the ongoing DMA in chain
1803  * @param chain_id
1804  *
1805  * @return - Success : Destination position
1806  *           Failure : -EINVAL
1807  */
1808 int omap_get_dma_chain_src_pos(int chain_id)
1809 {
1810         int lch;
1811         int *channels;
1812
1813         /* Check for input params */
1814         if (unlikely((chain_id < 0 || chain_id >= dma_lch_count))) {
1815                 printk(KERN_ERR "Invalid chain id\n");
1816                 return -EINVAL;
1817         }
1818
1819         /* Check if the chain exists */
1820         if (dma_linked_lch[chain_id].linked_dmach_q == NULL) {
1821                 printk(KERN_ERR "Chain doesn't exists\n");
1822                 return -EINVAL;
1823         }
1824
1825         channels = dma_linked_lch[chain_id].linked_dmach_q;
1826
1827         /* Get the current channel */
1828         lch = channels[dma_linked_lch[chain_id].q_head];
1829
1830         return dma_read(CSAC(lch));
1831 }
1832 EXPORT_SYMBOL(omap_get_dma_chain_src_pos);
1833 #endif  /* ifndef CONFIG_ARCH_OMAP1 */
1834
1835 /*----------------------------------------------------------------------------*/
1836
1837 #ifdef CONFIG_ARCH_OMAP1
1838
1839 static int omap1_dma_handle_ch(int ch)
1840 {
1841         u32 csr;
1842
1843         if (enable_1510_mode && ch >= 6) {
1844                 csr = dma_chan[ch].saved_csr;
1845                 dma_chan[ch].saved_csr = 0;
1846         } else
1847                 csr = dma_read(CSR(ch));
1848         if (enable_1510_mode && ch <= 2 && (csr >> 7) != 0) {
1849                 dma_chan[ch + 6].saved_csr = csr >> 7;
1850                 csr &= 0x7f;
1851         }
1852         if ((csr & 0x3f) == 0)
1853                 return 0;
1854         if (unlikely(dma_chan[ch].dev_id == -1)) {
1855                 printk(KERN_WARNING "Spurious interrupt from DMA channel "
1856                        "%d (CSR %04x)\n", ch, csr);
1857                 return 0;
1858         }
1859         if (unlikely(csr & OMAP1_DMA_TOUT_IRQ))
1860                 printk(KERN_WARNING "DMA timeout with device %d\n",
1861                        dma_chan[ch].dev_id);
1862         if (unlikely(csr & OMAP_DMA_DROP_IRQ))
1863                 printk(KERN_WARNING "DMA synchronization event drop occurred "
1864                        "with device %d\n", dma_chan[ch].dev_id);
1865         if (likely(csr & OMAP_DMA_BLOCK_IRQ))
1866                 dma_chan[ch].flags &= ~OMAP_DMA_ACTIVE;
1867         if (likely(dma_chan[ch].callback != NULL))
1868                 dma_chan[ch].callback(ch, csr, dma_chan[ch].data);
1869
1870         return 1;
1871 }
1872
1873 static irqreturn_t omap1_dma_irq_handler(int irq, void *dev_id)
1874 {
1875         int ch = ((int) dev_id) - 1;
1876         int handled = 0;
1877
1878         for (;;) {
1879                 int handled_now = 0;
1880
1881                 handled_now += omap1_dma_handle_ch(ch);
1882                 if (enable_1510_mode && dma_chan[ch + 6].saved_csr)
1883                         handled_now += omap1_dma_handle_ch(ch + 6);
1884                 if (!handled_now)
1885                         break;
1886                 handled += handled_now;
1887         }
1888
1889         return handled ? IRQ_HANDLED : IRQ_NONE;
1890 }
1891
1892 #else
1893 #define omap1_dma_irq_handler   NULL
1894 #endif
1895
1896 #ifdef CONFIG_ARCH_OMAP2PLUS
1897
1898 static int omap2_dma_handle_ch(int ch)
1899 {
1900         u32 status = dma_read(CSR(ch));
1901
1902         if (!status) {
1903                 if (printk_ratelimit())
1904                         printk(KERN_WARNING "Spurious DMA IRQ for lch %d\n",
1905                                 ch);
1906                 dma_write(1 << ch, IRQSTATUS_L0);
1907                 return 0;
1908         }
1909         if (unlikely(dma_chan[ch].dev_id == -1)) {
1910                 if (printk_ratelimit())
1911                         printk(KERN_WARNING "IRQ %04x for non-allocated DMA"
1912                                         "channel %d\n", status, ch);
1913                 return 0;
1914         }
1915         if (unlikely(status & OMAP_DMA_DROP_IRQ))
1916                 printk(KERN_INFO
1917                        "DMA synchronization event drop occurred with device "
1918                        "%d\n", dma_chan[ch].dev_id);
1919         if (unlikely(status & OMAP2_DMA_TRANS_ERR_IRQ)) {
1920                 printk(KERN_INFO "DMA transaction error with device %d\n",
1921                        dma_chan[ch].dev_id);
1922                 if (cpu_class_is_omap2()) {
1923                         /*
1924                          * Errata: sDMA Channel is not disabled
1925                          * after a transaction error. So we explicitely
1926                          * disable the channel
1927                          */
1928                         u32 ccr;
1929
1930                         ccr = dma_read(CCR(ch));
1931                         ccr &= ~OMAP_DMA_CCR_EN;
1932                         dma_write(ccr, CCR(ch));
1933                         dma_chan[ch].flags &= ~OMAP_DMA_ACTIVE;
1934                 }
1935         }
1936         if (unlikely(status & OMAP2_DMA_SECURE_ERR_IRQ))
1937                 printk(KERN_INFO "DMA secure error with device %d\n",
1938                        dma_chan[ch].dev_id);
1939         if (unlikely(status & OMAP2_DMA_MISALIGNED_ERR_IRQ))
1940                 printk(KERN_INFO "DMA misaligned error with device %d\n",
1941                        dma_chan[ch].dev_id);
1942
1943         dma_write(OMAP2_DMA_CSR_CLEAR_MASK, CSR(ch));
1944         dma_write(1 << ch, IRQSTATUS_L0);
1945
1946         /* If the ch is not chained then chain_id will be -1 */
1947         if (dma_chan[ch].chain_id != -1) {
1948                 int chain_id = dma_chan[ch].chain_id;
1949                 dma_chan[ch].state = DMA_CH_NOTSTARTED;
1950                 if (dma_read(CLNK_CTRL(ch)) & (1 << 15))
1951                         dma_chan[dma_chan[ch].next_linked_ch].state =
1952                                                         DMA_CH_STARTED;
1953                 if (dma_linked_lch[chain_id].chain_mode ==
1954                                                 OMAP_DMA_DYNAMIC_CHAIN)
1955                         disable_lnk(ch);
1956
1957                 if (!OMAP_DMA_CHAIN_QEMPTY(chain_id))
1958                         OMAP_DMA_CHAIN_INCQHEAD(chain_id);
1959
1960                 status = dma_read(CSR(ch));
1961         }
1962
1963         dma_write(status, CSR(ch));
1964
1965         if (likely(dma_chan[ch].callback != NULL))
1966                 dma_chan[ch].callback(ch, status, dma_chan[ch].data);
1967
1968         return 0;
1969 }
1970
1971 /* STATUS register count is from 1-32 while our is 0-31 */
1972 static irqreturn_t omap2_dma_irq_handler(int irq, void *dev_id)
1973 {
1974         u32 val, enable_reg;
1975         int i;
1976
1977         val = dma_read(IRQSTATUS_L0);
1978         if (val == 0) {
1979                 if (printk_ratelimit())
1980                         printk(KERN_WARNING "Spurious DMA IRQ\n");
1981                 return IRQ_HANDLED;
1982         }
1983         enable_reg = dma_read(IRQENABLE_L0);
1984         val &= enable_reg; /* Dispatch only relevant interrupts */
1985         for (i = 0; i < dma_lch_count && val != 0; i++) {
1986                 if (val & 1)
1987                         omap2_dma_handle_ch(i);
1988                 val >>= 1;
1989         }
1990
1991         return IRQ_HANDLED;
1992 }
1993
1994 static struct irqaction omap24xx_dma_irq = {
1995         .name = "DMA",
1996         .handler = omap2_dma_irq_handler,
1997         .flags = IRQF_DISABLED
1998 };
1999
2000 #else
2001 static struct irqaction omap24xx_dma_irq;
2002 #endif
2003
2004 /*----------------------------------------------------------------------------*/
2005
2006 void omap_dma_global_context_save(void)
2007 {
2008         omap_dma_global_context.dma_irqenable_l0 =
2009                 dma_read(IRQENABLE_L0);
2010         omap_dma_global_context.dma_ocp_sysconfig =
2011                 dma_read(OCP_SYSCONFIG);
2012         omap_dma_global_context.dma_gcr = dma_read(GCR);
2013 }
2014
2015 void omap_dma_global_context_restore(void)
2016 {
2017         int ch;
2018
2019         dma_write(omap_dma_global_context.dma_gcr, GCR);
2020         dma_write(omap_dma_global_context.dma_ocp_sysconfig,
2021                 OCP_SYSCONFIG);
2022         dma_write(omap_dma_global_context.dma_irqenable_l0,
2023                 IRQENABLE_L0);
2024
2025         /*
2026          * A bug in ROM code leaves IRQ status for channels 0 and 1 uncleared
2027          * after secure sram context save and restore. Hence we need to
2028          * manually clear those IRQs to avoid spurious interrupts. This
2029          * affects only secure devices.
2030          */
2031         if (cpu_is_omap34xx() && (omap_type() != OMAP2_DEVICE_TYPE_GP))
2032                 dma_write(0x3 , IRQSTATUS_L0);
2033
2034         for (ch = 0; ch < dma_chan_count; ch++)
2035                 if (dma_chan[ch].dev_id != -1)
2036                         omap_clear_dma(ch);
2037 }
2038
2039 /*----------------------------------------------------------------------------*/
2040
2041 static int __init omap_init_dma(void)
2042 {
2043         unsigned long base;
2044         int ch, r;
2045
2046         if (cpu_class_is_omap1()) {
2047                 base = OMAP1_DMA_BASE;
2048                 dma_lch_count = OMAP1_LOGICAL_DMA_CH_COUNT;
2049         } else if (cpu_is_omap24xx()) {
2050                 base = OMAP24XX_DMA4_BASE;
2051                 dma_lch_count = OMAP_DMA4_LOGICAL_DMA_CH_COUNT;
2052         } else if (cpu_is_omap34xx()) {
2053                 base = OMAP34XX_DMA4_BASE;
2054                 dma_lch_count = OMAP_DMA4_LOGICAL_DMA_CH_COUNT;
2055         } else if (cpu_is_omap44xx()) {
2056                 base = OMAP44XX_DMA4_BASE;
2057                 dma_lch_count = OMAP_DMA4_LOGICAL_DMA_CH_COUNT;
2058         } else {
2059                 pr_err("DMA init failed for unsupported omap\n");
2060                 return -ENODEV;
2061         }
2062
2063         omap_dma_base = ioremap(base, SZ_4K);
2064         BUG_ON(!omap_dma_base);
2065
2066         if (cpu_class_is_omap2() && omap_dma_reserve_channels
2067                         && (omap_dma_reserve_channels <= dma_lch_count))
2068                 dma_lch_count = omap_dma_reserve_channels;
2069
2070         dma_chan = kzalloc(sizeof(struct omap_dma_lch) * dma_lch_count,
2071                                 GFP_KERNEL);
2072         if (!dma_chan) {
2073                 r = -ENOMEM;
2074                 goto out_unmap;
2075         }
2076
2077         if (cpu_class_is_omap2()) {
2078                 dma_linked_lch = kzalloc(sizeof(struct dma_link_info) *
2079                                                 dma_lch_count, GFP_KERNEL);
2080                 if (!dma_linked_lch) {
2081                         r = -ENOMEM;
2082                         goto out_free;
2083                 }
2084         }
2085
2086         if (cpu_is_omap15xx()) {
2087                 printk(KERN_INFO "DMA support for OMAP15xx initialized\n");
2088                 dma_chan_count = 9;
2089                 enable_1510_mode = 1;
2090         } else if (cpu_is_omap16xx() || cpu_is_omap7xx()) {
2091                 printk(KERN_INFO "OMAP DMA hardware version %d\n",
2092                        dma_read(HW_ID));
2093                 printk(KERN_INFO "DMA capabilities: %08x:%08x:%04x:%04x:%04x\n",
2094                        (dma_read(CAPS_0_U) << 16) |
2095                        dma_read(CAPS_0_L),
2096                        (dma_read(CAPS_1_U) << 16) |
2097                        dma_read(CAPS_1_L),
2098                        dma_read(CAPS_2), dma_read(CAPS_3),
2099                        dma_read(CAPS_4));
2100                 if (!enable_1510_mode) {
2101                         u16 w;
2102
2103                         /* Disable OMAP 3.0/3.1 compatibility mode. */
2104                         w = dma_read(GSCR);
2105                         w |= 1 << 3;
2106                         dma_write(w, GSCR);
2107                         dma_chan_count = 16;
2108                 } else
2109                         dma_chan_count = 9;
2110         } else if (cpu_class_is_omap2()) {
2111                 u8 revision = dma_read(REVISION) & 0xff;
2112                 printk(KERN_INFO "OMAP DMA hardware revision %d.%d\n",
2113                        revision >> 4, revision & 0xf);
2114                 dma_chan_count = dma_lch_count;
2115         } else {
2116                 dma_chan_count = 0;
2117                 return 0;
2118         }
2119
2120         spin_lock_init(&dma_chan_lock);
2121
2122         for (ch = 0; ch < dma_chan_count; ch++) {
2123                 omap_clear_dma(ch);
2124                 if (cpu_class_is_omap2())
2125                         omap2_disable_irq_lch(ch);
2126
2127                 dma_chan[ch].dev_id = -1;
2128                 dma_chan[ch].next_lch = -1;
2129
2130                 if (ch >= 6 && enable_1510_mode)
2131                         continue;
2132
2133                 if (cpu_class_is_omap1()) {
2134                         /*
2135                          * request_irq() doesn't like dev_id (ie. ch) being
2136                          * zero, so we have to kludge around this.
2137                          */
2138                         r = request_irq(omap1_dma_irq[ch],
2139                                         omap1_dma_irq_handler, 0, "DMA",
2140                                         (void *) (ch + 1));
2141                         if (r != 0) {
2142                                 int i;
2143
2144                                 printk(KERN_ERR "unable to request IRQ %d "
2145                                        "for DMA (error %d)\n",
2146                                        omap1_dma_irq[ch], r);
2147                                 for (i = 0; i < ch; i++)
2148                                         free_irq(omap1_dma_irq[i],
2149                                                  (void *) (i + 1));
2150                                 goto out_free;
2151                         }
2152                 }
2153         }
2154
2155         if (cpu_is_omap2430() || cpu_is_omap34xx() || cpu_is_omap44xx())
2156                 omap_dma_set_global_params(DMA_DEFAULT_ARB_RATE,
2157                                 DMA_DEFAULT_FIFO_DEPTH, 0);
2158
2159         if (cpu_class_is_omap2()) {
2160                 int irq;
2161                 if (cpu_is_omap44xx())
2162                         irq = OMAP44XX_IRQ_SDMA_0;
2163                 else
2164                         irq = INT_24XX_SDMA_IRQ0;
2165                 setup_irq(irq, &omap24xx_dma_irq);
2166         }
2167
2168         if (cpu_is_omap34xx() || cpu_is_omap44xx()) {
2169                 /* Enable smartidle idlemodes and autoidle */
2170                 u32 v = dma_read(OCP_SYSCONFIG);
2171                 v &= ~(DMA_SYSCONFIG_MIDLEMODE_MASK |
2172                                 DMA_SYSCONFIG_SIDLEMODE_MASK |
2173                                 DMA_SYSCONFIG_AUTOIDLE);
2174                 v |= (DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_SMARTIDLE) |
2175                         DMA_SYSCONFIG_SIDLEMODE(DMA_IDLEMODE_SMARTIDLE) |
2176                         DMA_SYSCONFIG_AUTOIDLE);
2177                 dma_write(v , OCP_SYSCONFIG);
2178                 /* reserve dma channels 0 and 1 in high security devices */
2179                 if (cpu_is_omap34xx() &&
2180                         (omap_type() != OMAP2_DEVICE_TYPE_GP)) {
2181                         printk(KERN_INFO "Reserving DMA channels 0 and 1 for "
2182                                         "HS ROM code\n");
2183                         dma_chan[0].dev_id = 0;
2184                         dma_chan[1].dev_id = 1;
2185                 }
2186         }
2187
2188         return 0;
2189
2190 out_free:
2191         kfree(dma_chan);
2192
2193 out_unmap:
2194         iounmap(omap_dma_base);
2195
2196         return r;
2197 }
2198
2199 arch_initcall(omap_init_dma);
2200
2201 /*
2202  * Reserve the omap SDMA channels using cmdline bootarg
2203  * "omap_dma_reserve_ch=". The valid range is 1 to 32
2204  */
2205 static int __init omap_dma_cmdline_reserve_ch(char *str)
2206 {
2207         if (get_option(&str, &omap_dma_reserve_channels) != 1)
2208                 omap_dma_reserve_channels = 0;
2209         return 1;
2210 }
2211
2212 __setup("omap_dma_reserve_ch=", omap_dma_cmdline_reserve_ch);
2213
2214