arch/blackfin/kernel/bfin_dma_5xx.c

   1 /*
   2  * File:         arch/blackfin/kernel/bfin_dma_5xx.c
   3  * Based on:
   4  * Author:
   5  *
   6  * Created:
   7  * Description:  This file contains the simple DMA Implementation for Blackfin
   8  *
   9  * Modified:
  10  *               Copyright 2004-2006 Analog Devices Inc.
  11  *
  12  * Bugs:         Enter bugs at http://blackfin.uclinux.org/
  13  *
  14  * This program is free software; you can redistribute it and/or modify
  15  * it under the terms of the GNU General Public License as published by
  16  * the Free Software Foundation; either version 2 of the License, or
  17  * (at your option) any later version.
  18  *
  19  * This program is distributed in the hope that it will be useful,
  20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  22  * GNU General Public License for more details.
  23  *
  24  * You should have received a copy of the GNU General Public License
  25  * along with this program; if not, see the file COPYING, or write
  26  * to the Free Software Foundation, Inc.,
  27  * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  28  */
  29
  30 #include <linux/errno.h>
  31 #include <linux/module.h>
  32 #include <linux/sched.h>
  33 #include <linux/interrupt.h>
  34 #include <linux/kernel.h>
  35 #include <linux/param.h>
  36
  37 #include <asm/blackfin.h>
  38 #include <asm/dma.h>
  39 #include <asm/cacheflush.h>
  40
  41 /* Remove unused code not exported by symbol or internally called */
  42 #define REMOVE_DEAD_CODE
  43
  44 /**************************************************************************
  45  * Global Variables
  46 ***************************************************************************/
  47
  48 static struct dma_channel dma_ch[MAX_BLACKFIN_DMA_CHANNEL];
  49
  50 /*------------------------------------------------------------------------------
  51  *       Set the Buffer Clear bit in the Configuration register of specific DMA
  52  *       channel. This will stop the descriptor based DMA operation.
  53  *-----------------------------------------------------------------------------*/
  54 static void clear_dma_buffer(unsigned int channel)
  55 {
  56         dma_ch[channel].regs->cfg |= RESTART;
  57         SSYNC();
  58         dma_ch[channel].regs->cfg &= ~RESTART;
  59         SSYNC();
  60 }
  61
  62 static int __init blackfin_dma_init(void)
  63 {
  64         int i;
  65
  66         printk(KERN_INFO "Blackfin DMA Controller\n");
  67
  68         for (i = 0; i < MAX_BLACKFIN_DMA_CHANNEL; i++) {
  69                 dma_ch[i].chan_status = DMA_CHANNEL_FREE;
  70                 dma_ch[i].regs = dma_io_base_addr[i];
  71                 mutex_init(&(dma_ch[i].dmalock));
  72         }
  73         /* Mark MEMDMA Channel 0 as requested since we're using it internally */
  74         dma_ch[CH_MEM_STREAM0_DEST].chan_status = DMA_CHANNEL_REQUESTED;
  75         dma_ch[CH_MEM_STREAM0_SRC].chan_status = DMA_CHANNEL_REQUESTED;
  76
  77 #if defined(CONFIG_DEB_DMA_URGENT)
  78         bfin_write_EBIU_DDRQUE(bfin_read_EBIU_DDRQUE()
  79                          | DEB1_URGENT | DEB2_URGENT | DEB3_URGENT);
  80 #endif
  81         return 0;
  82 }
  83
  84 arch_initcall(blackfin_dma_init);
  85
  86 /*------------------------------------------------------------------------------
  87  *      Request the specific DMA channel from the system.
  88  *-----------------------------------------------------------------------------*/
  89 int request_dma(unsigned int channel, char *device_id)
  90 {
  91
  92         pr_debug("request_dma() : BEGIN \n");
  93         mutex_lock(&(dma_ch[channel].dmalock));
  94
  95         if ((dma_ch[channel].chan_status == DMA_CHANNEL_REQUESTED)
  96             || (dma_ch[channel].chan_status == DMA_CHANNEL_ENABLED)) {
  97                 mutex_unlock(&(dma_ch[channel].dmalock));
  98                 pr_debug("DMA CHANNEL IN USE  \n");
  99                 return -EBUSY;
 100         } else {
 101                 dma_ch[channel].chan_status = DMA_CHANNEL_REQUESTED;
 102                 pr_debug("DMA CHANNEL IS ALLOCATED  \n");
 103         }
 104
 105         mutex_unlock(&(dma_ch[channel].dmalock));
 106
 107 #ifdef CONFIG_BF54x
 108         if (channel >= CH_UART2_RX && channel <= CH_UART3_TX) {
 109                 if (strncmp(device_id, "BFIN_UART", 9) == 0) {
 110                         dma_ch[channel].regs->peripheral_map &= 0x0FFF;
 111                         dma_ch[channel].regs->peripheral_map |=
 112                                 ((channel - CH_UART2_RX + 0xC)<<12);
 113                 } else {
 114                         dma_ch[channel].regs->peripheral_map &= 0x0FFF;
 115                         dma_ch[channel].regs->peripheral_map |=
 116                                 ((channel - CH_UART2_RX + 0x6)<<12);
 117                 }
 118         }
 119 #endif
 120
 121         dma_ch[channel].device_id = device_id;
 122         dma_ch[channel].irq_callback = NULL;
 123
 124         /* This is to be enabled by putting a restriction -
 125          * you have to request DMA, before doing any operations on
 126          * descriptor/channel
 127          */
 128         pr_debug("request_dma() : END  \n");
 129         return channel;
 130 }
 131 EXPORT_SYMBOL(request_dma);
 132
 133 int set_dma_callback(unsigned int channel, dma_interrupt_t callback, void *data)
 134 {
 135         int ret_irq = 0;
 136
 137         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 138                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 139
 140         if (callback != NULL) {
 141                 int ret_val;
 142                 ret_irq = channel2irq(channel);
 143
 144                 dma_ch[channel].data = data;
 145
 146                 ret_val =
 147                     request_irq(ret_irq, (void *)callback, IRQF_DISABLED,
 148                                 dma_ch[channel].device_id, data);
 149                 if (ret_val) {
 150                         printk(KERN_NOTICE
 151                                "Request irq in DMA engine failed.\n");
 152                         return -EPERM;
 153                 }
 154                 dma_ch[channel].irq_callback = callback;
 155         }
 156         return 0;
 157 }
 158 EXPORT_SYMBOL(set_dma_callback);
 159
 160 void free_dma(unsigned int channel)
 161 {
 162         int ret_irq;
 163
 164         pr_debug("freedma() : BEGIN \n");
 165         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 166                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 167
 168         /* Halt the DMA */
 169         disable_dma(channel);
 170         clear_dma_buffer(channel);
 171
 172         if (dma_ch[channel].irq_callback != NULL) {
 173                 ret_irq = channel2irq(channel);
 174                 free_irq(ret_irq, dma_ch[channel].data);
 175         }
 176
 177         /* Clear the DMA Variable in the Channel */
 178         mutex_lock(&(dma_ch[channel].dmalock));
 179         dma_ch[channel].chan_status = DMA_CHANNEL_FREE;
 180         mutex_unlock(&(dma_ch[channel].dmalock));
 181
 182         pr_debug("freedma() : END \n");
 183 }
 184 EXPORT_SYMBOL(free_dma);
 185
 186 void dma_enable_irq(unsigned int channel)
 187 {
 188         int ret_irq;
 189
 190         pr_debug("dma_enable_irq() : BEGIN \n");
 191         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 192                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 193
 194         ret_irq = channel2irq(channel);
 195         enable_irq(ret_irq);
 196 }
 197 EXPORT_SYMBOL(dma_enable_irq);
 198
 199 void dma_disable_irq(unsigned int channel)
 200 {
 201         int ret_irq;
 202
 203         pr_debug("dma_disable_irq() : BEGIN \n");
 204         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 205                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 206
 207         ret_irq = channel2irq(channel);
 208         disable_irq(ret_irq);
 209 }
 210 EXPORT_SYMBOL(dma_disable_irq);
 211
 212 int dma_channel_active(unsigned int channel)
 213 {
 214         if (dma_ch[channel].chan_status == DMA_CHANNEL_FREE) {
 215                 return 0;
 216         } else {
 217                 return 1;
 218         }
 219 }
 220 EXPORT_SYMBOL(dma_channel_active);
 221
 222 /*------------------------------------------------------------------------------
 223 *       stop the specific DMA channel.
 224 *-----------------------------------------------------------------------------*/
 225 void disable_dma(unsigned int channel)
 226 {
 227         pr_debug("stop_dma() : BEGIN \n");
 228
 229         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 230                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 231
 232         dma_ch[channel].regs->cfg &= ~DMAEN;    /* Clean the enable bit */
 233         SSYNC();
 234         dma_ch[channel].chan_status = DMA_CHANNEL_REQUESTED;
 235         /* Needs to be enabled Later */
 236         pr_debug("stop_dma() : END \n");
 237         return;
 238 }
 239 EXPORT_SYMBOL(disable_dma);
 240
 241 void enable_dma(unsigned int channel)
 242 {
 243         pr_debug("enable_dma() : BEGIN \n");
 244
 245         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 246                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 247
 248         dma_ch[channel].chan_status = DMA_CHANNEL_ENABLED;
 249         dma_ch[channel].regs->curr_x_count = 0;
 250         dma_ch[channel].regs->curr_y_count = 0;
 251
 252         dma_ch[channel].regs->cfg |= DMAEN;     /* Set the enable bit */
 253         SSYNC();
 254         pr_debug("enable_dma() : END \n");
 255         return;
 256 }
 257 EXPORT_SYMBOL(enable_dma);
 258
 259 /*------------------------------------------------------------------------------
 260 *               Set the Start Address register for the specific DMA channel
 261 *               This function can be used for register based DMA,
 262 *               to setup the start address
 263 *               addr:           Starting address of the DMA Data to be transferred.
 264 *-----------------------------------------------------------------------------*/
 265 void set_dma_start_addr(unsigned int channel, unsigned long addr)
 266 {
 267         pr_debug("set_dma_start_addr() : BEGIN \n");
 268
 269         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 270                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 271
 272         dma_ch[channel].regs->start_addr = addr;
 273         SSYNC();
 274         pr_debug("set_dma_start_addr() : END\n");
 275 }
 276 EXPORT_SYMBOL(set_dma_start_addr);
 277
 278 void set_dma_next_desc_addr(unsigned int channel, unsigned long addr)
 279 {
 280         pr_debug("set_dma_next_desc_addr() : BEGIN \n");
 281
 282         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 283                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 284
 285         dma_ch[channel].regs->next_desc_ptr = addr;
 286         SSYNC();
 287         pr_debug("set_dma_next_desc_addr() : END\n");
 288 }
 289 EXPORT_SYMBOL(set_dma_next_desc_addr);
 290
 291 void set_dma_curr_desc_addr(unsigned int channel, unsigned long addr)
 292 {
 293         pr_debug("set_dma_curr_desc_addr() : BEGIN \n");
 294
 295         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 296                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 297
 298         dma_ch[channel].regs->curr_desc_ptr = addr;
 299         SSYNC();
 300         pr_debug("set_dma_curr_desc_addr() : END\n");
 301 }
 302 EXPORT_SYMBOL(set_dma_curr_desc_addr);
 303
 304 void set_dma_x_count(unsigned int channel, unsigned short x_count)
 305 {
 306         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 307                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 308
 309         dma_ch[channel].regs->x_count = x_count;
 310         SSYNC();
 311 }
 312 EXPORT_SYMBOL(set_dma_x_count);
 313
 314 void set_dma_y_count(unsigned int channel, unsigned short y_count)
 315 {
 316         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 317                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 318
 319         dma_ch[channel].regs->y_count = y_count;
 320         SSYNC();
 321 }
 322 EXPORT_SYMBOL(set_dma_y_count);
 323
 324 void set_dma_x_modify(unsigned int channel, short x_modify)
 325 {
 326         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 327                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 328
 329         dma_ch[channel].regs->x_modify = x_modify;
 330         SSYNC();
 331 }
 332 EXPORT_SYMBOL(set_dma_x_modify);
 333
 334 void set_dma_y_modify(unsigned int channel, short y_modify)
 335 {
 336         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 337                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 338
 339         dma_ch[channel].regs->y_modify = y_modify;
 340         SSYNC();
 341 }
 342 EXPORT_SYMBOL(set_dma_y_modify);
 343
 344 void set_dma_config(unsigned int channel, unsigned short config)
 345 {
 346         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 347                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 348
 349         dma_ch[channel].regs->cfg = config;
 350         SSYNC();
 351 }
 352 EXPORT_SYMBOL(set_dma_config);
 353
 354 unsigned short
 355 set_bfin_dma_config(char direction, char flow_mode,
 356                     char intr_mode, char dma_mode, char width, char syncmode)
 357 {
 358         unsigned short config;
 359
 360         config =
 361             ((direction << 1) | (width << 2) | (dma_mode << 4) |
 362              (intr_mode << 6) | (flow_mode << 12) | (syncmode << 5));
 363         return config;
 364 }
 365 EXPORT_SYMBOL(set_bfin_dma_config);
 366
 367 void set_dma_sg(unsigned int channel, struct dmasg *sg, int nr_sg)
 368 {
 369         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 370                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 371
 372         dma_ch[channel].regs->cfg |= ((nr_sg & 0x0F) << 8);
 373
 374         dma_ch[channel].regs->next_desc_ptr = (unsigned int)sg;
 375
 376         SSYNC();
 377 }
 378 EXPORT_SYMBOL(set_dma_sg);
 379
 380 void set_dma_curr_addr(unsigned int channel, unsigned long addr)
 381 {
 382         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 383                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 384
 385         dma_ch[channel].regs->curr_addr_ptr = addr;
 386         SSYNC();
 387 }
 388 EXPORT_SYMBOL(set_dma_curr_addr);
 389
 390 /*------------------------------------------------------------------------------
 391  *      Get the DMA status of a specific DMA channel from the system.
 392  *-----------------------------------------------------------------------------*/
 393 unsigned short get_dma_curr_irqstat(unsigned int channel)
 394 {
 395         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 396                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 397
 398         return dma_ch[channel].regs->irq_status;
 399 }
 400 EXPORT_SYMBOL(get_dma_curr_irqstat);
 401
 402 /*------------------------------------------------------------------------------
 403  *      Clear the DMA_DONE bit in DMA status. Stop the DMA completion interrupt.
 404  *-----------------------------------------------------------------------------*/
 405 void clear_dma_irqstat(unsigned int channel)
 406 {
 407         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 408                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 409         dma_ch[channel].regs->irq_status |= 3;
 410 }
 411 EXPORT_SYMBOL(clear_dma_irqstat);
 412
 413 /*------------------------------------------------------------------------------
 414  *      Get current DMA xcount of a specific DMA channel from the system.
 415  *-----------------------------------------------------------------------------*/
 416 unsigned short get_dma_curr_xcount(unsigned int channel)
 417 {
 418         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 419                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 420
 421         return dma_ch[channel].regs->curr_x_count;
 422 }
 423 EXPORT_SYMBOL(get_dma_curr_xcount);
 424
 425 /*------------------------------------------------------------------------------
 426  *      Get current DMA ycount of a specific DMA channel from the system.
 427  *-----------------------------------------------------------------------------*/
 428 unsigned short get_dma_curr_ycount(unsigned int channel)
 429 {
 430         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 431                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 432
 433         return dma_ch[channel].regs->curr_y_count;
 434 }
 435 EXPORT_SYMBOL(get_dma_curr_ycount);
 436
 437 unsigned long get_dma_next_desc_ptr(unsigned int channel)
 438 {
 439         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 440               && channel < MAX_BLACKFIN_DMA_CHANNEL));
 441
 442         return dma_ch[channel].regs->next_desc_ptr;
 443 }
 444 EXPORT_SYMBOL(get_dma_next_desc_ptr);
 445
 446 unsigned long get_dma_curr_desc_ptr(unsigned int channel)
 447 {
 448         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 449               && channel < MAX_BLACKFIN_DMA_CHANNEL));
 450
 451         return dma_ch[channel].regs->curr_desc_ptr;
 452 }
 453 EXPORT_SYMBOL(get_dma_curr_desc_ptr);
 454
 455 unsigned long get_dma_curr_addr(unsigned int channel)
 456 {
 457         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 458               && channel < MAX_BLACKFIN_DMA_CHANNEL));
 459
 460         return dma_ch[channel].regs->curr_addr_ptr;
 461 }
 462 EXPORT_SYMBOL(get_dma_curr_addr);
 463
 464 static void *__dma_memcpy(void *dest, const void *src, size_t size)
 465 {
 466         int direction;  /* 1 - address decrease, 0 - address increase */
 467         int flag_align; /* 1 - address aligned,  0 - address unaligned */
 468         int flag_2D;    /* 1 - 2D DMA needed,    0 - 1D DMA needed */
 469         unsigned long flags;
 470
 471         if (size <= 0)
 472                 return NULL;
 473
 474         local_irq_save(flags);
 475
 476         if ((unsigned long)src < memory_end)
 477                 blackfin_dcache_flush_range((unsigned int)src,
 478                                             (unsigned int)(src + size));
 479
 480         if ((unsigned long)dest < memory_end)
 481                 blackfin_dcache_invalidate_range((unsigned int)dest,
 482                                                  (unsigned int)(dest + size));
 483
 484         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 485
 486         if ((unsigned long)src < (unsigned long)dest)
 487                 direction = 1;
 488         else
 489                 direction = 0;
 490
 491         if ((((unsigned long)dest % 2) == 0) && (((unsigned long)src % 2) == 0)
 492             && ((size % 2) == 0))
 493                 flag_align = 1;
 494         else
 495                 flag_align = 0;
 496
 497         if (size > 0x10000)     /* size > 64K */
 498                 flag_2D = 1;
 499         else
 500                 flag_2D = 0;
 501
 502         /* Setup destination and source start address */
 503         if (direction) {
 504                 if (flag_align) {
 505                         bfin_write_MDMA_D0_START_ADDR(dest + size - 2);
 506                         bfin_write_MDMA_S0_START_ADDR(src + size - 2);
 507                 } else {
 508                         bfin_write_MDMA_D0_START_ADDR(dest + size - 1);
 509                         bfin_write_MDMA_S0_START_ADDR(src + size - 1);
 510                 }
 511         } else {
 512                 bfin_write_MDMA_D0_START_ADDR(dest);
 513                 bfin_write_MDMA_S0_START_ADDR(src);
 514         }
 515
 516         /* Setup destination and source xcount */
 517         if (flag_2D) {
 518                 if (flag_align) {
 519                         bfin_write_MDMA_D0_X_COUNT(1024 / 2);
 520                         bfin_write_MDMA_S0_X_COUNT(1024 / 2);
 521                 } else {
 522                         bfin_write_MDMA_D0_X_COUNT(1024);
 523                         bfin_write_MDMA_S0_X_COUNT(1024);
 524                 }
 525                 bfin_write_MDMA_D0_Y_COUNT(size >> 10);
 526                 bfin_write_MDMA_S0_Y_COUNT(size >> 10);
 527         } else {
 528                 if (flag_align) {
 529                         bfin_write_MDMA_D0_X_COUNT(size / 2);
 530                         bfin_write_MDMA_S0_X_COUNT(size / 2);
 531                 } else {
 532                         bfin_write_MDMA_D0_X_COUNT(size);
 533                         bfin_write_MDMA_S0_X_COUNT(size);
 534                 }
 535         }
 536
 537         /* Setup destination and source xmodify and ymodify */
 538         if (direction) {
 539                 if (flag_align) {
 540                         bfin_write_MDMA_D0_X_MODIFY(-2);
 541                         bfin_write_MDMA_S0_X_MODIFY(-2);
 542                         if (flag_2D) {
 543                                 bfin_write_MDMA_D0_Y_MODIFY(-2);
 544                                 bfin_write_MDMA_S0_Y_MODIFY(-2);
 545                         }
 546                 } else {
 547                         bfin_write_MDMA_D0_X_MODIFY(-1);
 548                         bfin_write_MDMA_S0_X_MODIFY(-1);
 549                         if (flag_2D) {
 550                                 bfin_write_MDMA_D0_Y_MODIFY(-1);
 551                                 bfin_write_MDMA_S0_Y_MODIFY(-1);
 552                         }
 553                 }
 554         } else {
 555                 if (flag_align) {
 556                         bfin_write_MDMA_D0_X_MODIFY(2);
 557                         bfin_write_MDMA_S0_X_MODIFY(2);
 558                         if (flag_2D) {
 559                                 bfin_write_MDMA_D0_Y_MODIFY(2);
 560                                 bfin_write_MDMA_S0_Y_MODIFY(2);
 561                         }
 562                 } else {
 563                         bfin_write_MDMA_D0_X_MODIFY(1);
 564                         bfin_write_MDMA_S0_X_MODIFY(1);
 565                         if (flag_2D) {
 566                                 bfin_write_MDMA_D0_Y_MODIFY(1);
 567                                 bfin_write_MDMA_S0_Y_MODIFY(1);
 568                         }
 569                 }
 570         }
 571
 572         /* Enable source DMA */
 573         if (flag_2D) {
 574                 if (flag_align) {
 575                         bfin_write_MDMA_S0_CONFIG(DMAEN | DMA2D | WDSIZE_16);
 576                         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | DMA2D | WDSIZE_16);
 577                 } else {
 578                         bfin_write_MDMA_S0_CONFIG(DMAEN | DMA2D);
 579                         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | DMA2D);
 580                 }
 581         } else {
 582                 if (flag_align) {
 583                         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_16);
 584                         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_16);
 585                 } else {
 586                         bfin_write_MDMA_S0_CONFIG(DMAEN);
 587                         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN);
 588                 }
 589         }
 590
 591         SSYNC();
 592
 593         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE))
 594                 ;
 595
 596         bfin_write_MDMA_D0_IRQ_STATUS(bfin_read_MDMA_D0_IRQ_STATUS() |
 597                                       (DMA_DONE | DMA_ERR));
 598
 599         bfin_write_MDMA_S0_CONFIG(0);
 600         bfin_write_MDMA_D0_CONFIG(0);
 601
 602         local_irq_restore(flags);
 603
 604         return dest;
 605 }
 606
 607 void *dma_memcpy(void *dest, const void *src, size_t size)
 608 {
 609         size_t bulk;
 610         size_t rest;
 611         void * addr;
 612
 613         bulk = (size >> 16) << 16;
 614         rest = size - bulk;
 615         if (bulk)
 616                 __dma_memcpy(dest, src, bulk);
 617         addr = __dma_memcpy(dest+bulk, src+bulk, rest);
 618         return addr;
 619 }
 620 EXPORT_SYMBOL(dma_memcpy);
 621
 622 void *safe_dma_memcpy(void *dest, const void *src, size_t size)
 623 {
 624         void *addr;
 625         addr = dma_memcpy(dest, src, size);
 626         return addr;
 627 }
 628 EXPORT_SYMBOL(safe_dma_memcpy);
 629
 630 void dma_outsb(unsigned long addr, const void *buf, unsigned short len)
 631 {
 632         unsigned long flags;
 633
 634         local_irq_save(flags);
 635
 636         blackfin_dcache_flush_range((unsigned int)buf,
 637                          (unsigned int)(buf) + len);
 638
 639         bfin_write_MDMA_D0_START_ADDR(addr);
 640         bfin_write_MDMA_D0_X_COUNT(len);
 641         bfin_write_MDMA_D0_X_MODIFY(0);
 642         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 643
 644         bfin_write_MDMA_S0_START_ADDR(buf);
 645         bfin_write_MDMA_S0_X_COUNT(len);
 646         bfin_write_MDMA_S0_X_MODIFY(1);
 647         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 648
 649         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_8);
 650         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_8);
 651
 652         SSYNC();
 653
 654         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 655
 656         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 657
 658         bfin_write_MDMA_S0_CONFIG(0);
 659         bfin_write_MDMA_D0_CONFIG(0);
 660         local_irq_restore(flags);
 661
 662 }
 663 EXPORT_SYMBOL(dma_outsb);
 664
 665
 666 void dma_insb(unsigned long addr, void *buf, unsigned short len)
 667 {
 668         unsigned long flags;
 669
 670         blackfin_dcache_invalidate_range((unsigned int)buf,
 671                          (unsigned int)(buf) + len);
 672
 673         local_irq_save(flags);
 674         bfin_write_MDMA_D0_START_ADDR(buf);
 675         bfin_write_MDMA_D0_X_COUNT(len);
 676         bfin_write_MDMA_D0_X_MODIFY(1);
 677         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 678
 679         bfin_write_MDMA_S0_START_ADDR(addr);
 680         bfin_write_MDMA_S0_X_COUNT(len);
 681         bfin_write_MDMA_S0_X_MODIFY(0);
 682         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 683
 684         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_8);
 685         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_8);
 686
 687         SSYNC();
 688
 689         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 690
 691         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 692
 693         bfin_write_MDMA_S0_CONFIG(0);
 694         bfin_write_MDMA_D0_CONFIG(0);
 695         local_irq_restore(flags);
 696
 697 }
 698 EXPORT_SYMBOL(dma_insb);
 699
 700 void dma_outsw(unsigned long addr, const void  *buf, unsigned short len)
 701 {
 702         unsigned long flags;
 703
 704         local_irq_save(flags);
 705
 706         blackfin_dcache_flush_range((unsigned int)buf,
 707                          (unsigned int)(buf) + len * sizeof(short));
 708
 709         bfin_write_MDMA_D0_START_ADDR(addr);
 710         bfin_write_MDMA_D0_X_COUNT(len);
 711         bfin_write_MDMA_D0_X_MODIFY(0);
 712         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 713
 714         bfin_write_MDMA_S0_START_ADDR(buf);
 715         bfin_write_MDMA_S0_X_COUNT(len);
 716         bfin_write_MDMA_S0_X_MODIFY(2);
 717         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 718
 719         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_16);
 720         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_16);
 721
 722         SSYNC();
 723
 724         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 725
 726         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 727
 728         bfin_write_MDMA_S0_CONFIG(0);
 729         bfin_write_MDMA_D0_CONFIG(0);
 730         local_irq_restore(flags);
 731
 732 }
 733 EXPORT_SYMBOL(dma_outsw);
 734
 735 void dma_insw(unsigned long addr, void *buf, unsigned short len)
 736 {
 737         unsigned long flags;
 738
 739         blackfin_dcache_invalidate_range((unsigned int)buf,
 740                          (unsigned int)(buf) + len * sizeof(short));
 741
 742         local_irq_save(flags);
 743
 744         bfin_write_MDMA_D0_START_ADDR(buf);
 745         bfin_write_MDMA_D0_X_COUNT(len);
 746         bfin_write_MDMA_D0_X_MODIFY(2);
 747         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 748
 749         bfin_write_MDMA_S0_START_ADDR(addr);
 750         bfin_write_MDMA_S0_X_COUNT(len);
 751         bfin_write_MDMA_S0_X_MODIFY(0);
 752         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 753
 754         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_16);
 755         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_16);
 756
 757         SSYNC();
 758
 759         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 760
 761         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 762
 763         bfin_write_MDMA_S0_CONFIG(0);
 764         bfin_write_MDMA_D0_CONFIG(0);
 765         local_irq_restore(flags);
 766
 767 }
 768 EXPORT_SYMBOL(dma_insw);
 769
 770 void dma_outsl(unsigned long addr, const void *buf, unsigned short len)
 771 {
 772         unsigned long flags;
 773
 774         local_irq_save(flags);
 775
 776         blackfin_dcache_flush_range((unsigned int)buf,
 777                          (unsigned int)(buf) + len * sizeof(long));
 778
 779         bfin_write_MDMA_D0_START_ADDR(addr);
 780         bfin_write_MDMA_D0_X_COUNT(len);
 781         bfin_write_MDMA_D0_X_MODIFY(0);
 782         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 783
 784         bfin_write_MDMA_S0_START_ADDR(buf);
 785         bfin_write_MDMA_S0_X_COUNT(len);
 786         bfin_write_MDMA_S0_X_MODIFY(4);
 787         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 788
 789         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_32);
 790         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_32);
 791
 792         SSYNC();
 793
 794         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 795
 796         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 797
 798         bfin_write_MDMA_S0_CONFIG(0);
 799         bfin_write_MDMA_D0_CONFIG(0);
 800         local_irq_restore(flags);
 801
 802 }
 803 EXPORT_SYMBOL(dma_outsl);
 804
 805 void dma_insl(unsigned long addr, void *buf, unsigned short len)
 806 {
 807         unsigned long flags;
 808
 809         blackfin_dcache_invalidate_range((unsigned int)buf,
 810                          (unsigned int)(buf) + len * sizeof(long));
 811
 812         local_irq_save(flags);
 813
 814         bfin_write_MDMA_D0_START_ADDR(buf);
 815         bfin_write_MDMA_D0_X_COUNT(len);
 816         bfin_write_MDMA_D0_X_MODIFY(4);
 817         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 818
 819         bfin_write_MDMA_S0_START_ADDR(addr);
 820         bfin_write_MDMA_S0_X_COUNT(len);
 821         bfin_write_MDMA_S0_X_MODIFY(0);
 822         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 823
 824         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_32);
 825         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_32);
 826
 827         SSYNC();
 828
 829         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 830
 831         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 832
 833         bfin_write_MDMA_S0_CONFIG(0);
 834         bfin_write_MDMA_D0_CONFIG(0);
 835         local_irq_restore(flags);
 836
 837 }
 838 EXPORT_SYMBOL(dma_insl);