arch/arm/mach-bcmring/dma.c

   1 /*****************************************************************************
   2 * Copyright 2004 - 2008 Broadcom Corporation.  All rights reserved.
   3 *
   4 * Unless you and Broadcom execute a separate written software license
   5 * agreement governing use of this software, this software is licensed to you
   6 * under the terms of the GNU General Public License version 2, available at
   7 * http://www.broadcom.com/licenses/GPLv2.php (the "GPL").
   8 *
   9 * Notwithstanding the above, under no circumstances may you combine this
  10 * software in any way with any other Broadcom software provided under a
  11 * license other than the GPL, without Broadcom's express prior written
  12 * consent.
  13 *****************************************************************************/
  14
  15 /****************************************************************************/
  16 /**
  17 *   @file   dma.c
  18 *
  19 *   @brief  Implements the DMA interface.
  20 */
  21 /****************************************************************************/
  22
  23 /* ---- Include Files ---------------------------------------------------- */
  24
  25 #include <linux/module.h>
  26 #include <linux/device.h>
  27 #include <linux/dma-mapping.h>
  28 #include <linux/interrupt.h>
  29 #include <linux/sched.h>
  30 #include <linux/irqreturn.h>
  31 #include <linux/proc_fs.h>
  32 #include <linux/slab.h>
  33
  34 #include <mach/timer.h>
  35
  36 #include <linux/mm.h>
  37 #include <linux/pfn.h>
  38 #include <linux/atomic.h>
  39 #include <linux/sched.h>
  40 #include <mach/dma.h>
  41
  42 /* I don't quite understand why dc4 fails when this is set to 1 and DMA is enabled */
  43 /* especially since dc4 doesn't use kmalloc'd memory. */
  44
  45 #define ALLOW_MAP_OF_KMALLOC_MEMORY 0
  46
  47 /* ---- Public Variables ------------------------------------------------- */
  48
  49 /* ---- Private Constants and Types -------------------------------------- */
  50
  51 #define MAKE_HANDLE(controllerIdx, channelIdx)    (((controllerIdx) << 4) | (channelIdx))
  52
  53 #define CONTROLLER_FROM_HANDLE(handle)    (((handle) >> 4) & 0x0f)
  54 #define CHANNEL_FROM_HANDLE(handle)       ((handle) & 0x0f)
  55
  56 #define DMA_MAP_DEBUG   0
  57
  58 #if DMA_MAP_DEBUG
  59 #   define  DMA_MAP_PRINT(fmt, args...)   printk("%s: " fmt, __func__,  ## args)
  60 #else
  61 #   define  DMA_MAP_PRINT(fmt, args...)
  62 #endif
  63
  64 /* ---- Private Variables ------------------------------------------------ */
  65
  66 static DMA_Global_t gDMA;
  67 static struct proc_dir_entry *gDmaDir;
  68
  69 static atomic_t gDmaStatMemTypeKmalloc = ATOMIC_INIT(0);
  70 static atomic_t gDmaStatMemTypeVmalloc = ATOMIC_INIT(0);
  71 static atomic_t gDmaStatMemTypeUser = ATOMIC_INIT(0);
  72 static atomic_t gDmaStatMemTypeCoherent = ATOMIC_INIT(0);
  73
  74 #include "dma_device.c"
  75
  76 /* ---- Private Function Prototypes -------------------------------------- */
  77
  78 /* ---- Functions  ------------------------------------------------------- */
  79
  80 /****************************************************************************/
  81 /**
  82 *   Displays information for /proc/dma/mem-type
  83 */
  84 /****************************************************************************/
  85
  86 static int dma_proc_read_mem_type(char *buf, char **start, off_t offset,
  87                                   int count, int *eof, void *data)
  88 {
  89         int len = 0;
  90
  91         len += sprintf(buf + len, "dma_map_mem statistics\n");
  92         len +=
  93             sprintf(buf + len, "coherent: %d\n",
  94                     atomic_read(&gDmaStatMemTypeCoherent));
  95         len +=
  96             sprintf(buf + len, "kmalloc:  %d\n",
  97                     atomic_read(&gDmaStatMemTypeKmalloc));
  98         len +=
  99             sprintf(buf + len, "vmalloc:  %d\n",
 100                     atomic_read(&gDmaStatMemTypeVmalloc));
 101         len +=
 102             sprintf(buf + len, "user:     %d\n",
 103                     atomic_read(&gDmaStatMemTypeUser));
 104
 105         return len;
 106 }
 107
 108 /****************************************************************************/
 109 /**
 110 *   Displays information for /proc/dma/channels
 111 */
 112 /****************************************************************************/
 113
 114 static int dma_proc_read_channels(char *buf, char **start, off_t offset,
 115                                   int count, int *eof, void *data)
 116 {
 117         int controllerIdx;
 118         int channelIdx;
 119         int limit = count - 200;
 120         int len = 0;
 121         DMA_Channel_t *channel;
 122
 123         if (down_interruptible(&gDMA.lock) < 0) {
 124                 return -ERESTARTSYS;
 125         }
 126
 127         for (controllerIdx = 0; controllerIdx < DMA_NUM_CONTROLLERS;
 128              controllerIdx++) {
 129                 for (channelIdx = 0; channelIdx < DMA_NUM_CHANNELS;
 130                      channelIdx++) {
 131                         if (len >= limit) {
 132                                 break;
 133                         }
 134
 135                         channel =
 136                             &gDMA.controller[controllerIdx].channel[channelIdx];
 137
 138                         len +=
 139                             sprintf(buf + len, "%d:%d ", controllerIdx,
 140                                     channelIdx);
 141
 142                         if ((channel->flags & DMA_CHANNEL_FLAG_IS_DEDICATED) !=
 143                             0) {
 144                                 len +=
 145                                     sprintf(buf + len, "Dedicated for %s ",
 146                                             DMA_gDeviceAttribute[channel->
 147                                                                  devType].name);
 148                         } else {
 149                                 len += sprintf(buf + len, "Shared ");
 150                         }
 151
 152                         if ((channel->flags & DMA_CHANNEL_FLAG_NO_ISR) != 0) {
 153                                 len += sprintf(buf + len, "No ISR ");
 154                         }
 155
 156                         if ((channel->flags & DMA_CHANNEL_FLAG_LARGE_FIFO) != 0) {
 157                                 len += sprintf(buf + len, "Fifo: 128 ");
 158                         } else {
 159                                 len += sprintf(buf + len, "Fifo: 64  ");
 160                         }
 161
 162                         if ((channel->flags & DMA_CHANNEL_FLAG_IN_USE) != 0) {
 163                                 len +=
 164                                     sprintf(buf + len, "InUse by %s",
 165                                             DMA_gDeviceAttribute[channel->
 166                                                                  devType].name);
 167 #if (DMA_DEBUG_TRACK_RESERVATION)
 168                                 len +=
 169                                     sprintf(buf + len, " (%s:%d)",
 170                                             channel->fileName,
 171                                             channel->lineNum);
 172 #endif
 173                         } else {
 174                                 len += sprintf(buf + len, "Avail ");
 175                         }
 176
 177                         if (channel->lastDevType != DMA_DEVICE_NONE) {
 178                                 len +=
 179                                     sprintf(buf + len, "Last use: %s ",
 180                                             DMA_gDeviceAttribute[channel->
 181                                                                  lastDevType].
 182                                             name);
 183                         }
 184
 185                         len += sprintf(buf + len, "\n");
 186                 }
 187         }
 188         up(&gDMA.lock);
 189         *eof = 1;
 190
 191         return len;
 192 }
 193
 194 /****************************************************************************/
 195 /**
 196 *   Displays information for /proc/dma/devices
 197 */
 198 /****************************************************************************/
 199
 200 static int dma_proc_read_devices(char *buf, char **start, off_t offset,
 201                                  int count, int *eof, void *data)
 202 {
 203         int limit = count - 200;
 204         int len = 0;
 205         int devIdx;
 206
 207         if (down_interruptible(&gDMA.lock) < 0) {
 208                 return -ERESTARTSYS;
 209         }
 210
 211         for (devIdx = 0; devIdx < DMA_NUM_DEVICE_ENTRIES; devIdx++) {
 212                 DMA_DeviceAttribute_t *devAttr = &DMA_gDeviceAttribute[devIdx];
 213
 214                 if (devAttr->name == NULL) {
 215                         continue;
 216                 }
 217
 218                 if (len >= limit) {
 219                         break;
 220                 }
 221
 222                 len += sprintf(buf + len, "%-12s ", devAttr->name);
 223
 224                 if ((devAttr->flags & DMA_DEVICE_FLAG_IS_DEDICATED) != 0) {
 225                         len +=
 226                             sprintf(buf + len, "Dedicated %d:%d ",
 227                                     devAttr->dedicatedController,
 228                                     devAttr->dedicatedChannel);
 229                 } else {
 230                         len += sprintf(buf + len, "Shared DMA:");
 231                         if ((devAttr->flags & DMA_DEVICE_FLAG_ON_DMA0) != 0) {
 232                                 len += sprintf(buf + len, "0");
 233                         }
 234                         if ((devAttr->flags & DMA_DEVICE_FLAG_ON_DMA1) != 0) {
 235                                 len += sprintf(buf + len, "1");
 236                         }
 237                         len += sprintf(buf + len, " ");
 238                 }
 239                 if ((devAttr->flags & DMA_DEVICE_FLAG_NO_ISR) != 0) {
 240                         len += sprintf(buf + len, "NoISR ");
 241                 }
 242                 if ((devAttr->flags & DMA_DEVICE_FLAG_ALLOW_LARGE_FIFO) != 0) {
 243                         len += sprintf(buf + len, "Allow-128 ");
 244                 }
 245
 246                 len +=
 247                     sprintf(buf + len,
 248                             "Xfer #: %Lu Ticks: %Lu Bytes: %Lu DescLen: %u\n",
 249                             devAttr->numTransfers, devAttr->transferTicks,
 250                             devAttr->transferBytes,
 251                             devAttr->ring.bytesAllocated);
 252
 253         }
 254
 255         up(&gDMA.lock);
 256         *eof = 1;
 257
 258         return len;
 259 }
 260
 261 /****************************************************************************/
 262 /**
 263 *   Determines if a DMA_Device_t is "valid".
 264 *
 265 *   @return
 266 *       TRUE        - dma device is valid
 267 *       FALSE       - dma device isn't valid
 268 */
 269 /****************************************************************************/
 270
 271 static inline int IsDeviceValid(DMA_Device_t device)
 272 {
 273         return (device >= 0) && (device < DMA_NUM_DEVICE_ENTRIES);
 274 }
 275
 276 /****************************************************************************/
 277 /**
 278 *   Translates a DMA handle into a pointer to a channel.
 279 *
 280 *   @return
 281 *       non-NULL    - pointer to DMA_Channel_t
 282 *       NULL        - DMA Handle was invalid
 283 */
 284 /****************************************************************************/
 285
 286 static inline DMA_Channel_t *HandleToChannel(DMA_Handle_t handle)
 287 {
 288         int controllerIdx;
 289         int channelIdx;
 290
 291         controllerIdx = CONTROLLER_FROM_HANDLE(handle);
 292         channelIdx = CHANNEL_FROM_HANDLE(handle);
 293
 294         if ((controllerIdx > DMA_NUM_CONTROLLERS)
 295             || (channelIdx > DMA_NUM_CHANNELS)) {
 296                 return NULL;
 297         }
 298         return &gDMA.controller[controllerIdx].channel[channelIdx];
 299 }
 300
 301 /****************************************************************************/
 302 /**
 303 *   Interrupt handler which is called to process DMA interrupts.
 304 */
 305 /****************************************************************************/
 306
 307 static irqreturn_t dma_interrupt_handler(int irq, void *dev_id)
 308 {
 309         DMA_Channel_t *channel;
 310         DMA_DeviceAttribute_t *devAttr;
 311         int irqStatus;
 312
 313         channel = (DMA_Channel_t *) dev_id;
 314
 315         /* Figure out why we were called, and knock down the interrupt */
 316
 317         irqStatus = dmacHw_getInterruptStatus(channel->dmacHwHandle);
 318         dmacHw_clearInterrupt(channel->dmacHwHandle);
 319
 320         if ((channel->devType < 0)
 321             || (channel->devType > DMA_NUM_DEVICE_ENTRIES)) {
 322                 printk(KERN_ERR "dma_interrupt_handler: Invalid devType: %d\n",
 323                        channel->devType);
 324                 return IRQ_NONE;
 325         }
 326         devAttr = &DMA_gDeviceAttribute[channel->devType];
 327
 328         /* Update stats */
 329
 330         if ((irqStatus & dmacHw_INTERRUPT_STATUS_TRANS) != 0) {
 331                 devAttr->transferTicks +=
 332                     (timer_get_tick_count() - devAttr->transferStartTime);
 333         }
 334
 335         if ((irqStatus & dmacHw_INTERRUPT_STATUS_ERROR) != 0) {
 336                 printk(KERN_ERR
 337                        "dma_interrupt_handler: devType :%d DMA error (%s)\n",
 338                        channel->devType, devAttr->name);
 339         } else {
 340                 devAttr->numTransfers++;
 341                 devAttr->transferBytes += devAttr->numBytes;
 342         }
 343
 344         /* Call any installed handler */
 345
 346         if (devAttr->devHandler != NULL) {
 347                 devAttr->devHandler(channel->devType, irqStatus,
 348                                     devAttr->userData);
 349         }
 350
 351         return IRQ_HANDLED;
 352 }
 353
 354 /****************************************************************************/
 355 /**
 356 *   Allocates memory to hold a descriptor ring. The descriptor ring then
 357 *   needs to be populated by making one or more calls to
 358 *   dna_add_descriptors.
 359 *
 360 *   The returned descriptor ring will be automatically initialized.
 361 *
 362 *   @return
 363 *       0           Descriptor ring was allocated successfully
 364 *       -EINVAL     Invalid parameters passed in
 365 *       -ENOMEM     Unable to allocate memory for the desired number of descriptors.
 366 */
 367 /****************************************************************************/
 368
 369 int dma_alloc_descriptor_ring(DMA_DescriptorRing_t *ring,       /* Descriptor ring to populate */
 370                               int numDescriptors        /* Number of descriptors that need to be allocated. */
 371     ) {
 372         size_t bytesToAlloc = dmacHw_descriptorLen(numDescriptors);
 373
 374         if ((ring == NULL) || (numDescriptors <= 0)) {
 375                 return -EINVAL;
 376         }
 377
 378         ring->physAddr = 0;
 379         ring->descriptorsAllocated = 0;
 380         ring->bytesAllocated = 0;
 381
 382         ring->virtAddr = dma_alloc_writecombine(NULL,
 383                                                      bytesToAlloc,
 384                                                      &ring->physAddr,
 385                                                      GFP_KERNEL);
 386         if (ring->virtAddr == NULL) {
 387                 return -ENOMEM;
 388         }
 389
 390         ring->bytesAllocated = bytesToAlloc;
 391         ring->descriptorsAllocated = numDescriptors;
 392
 393         return dma_init_descriptor_ring(ring, numDescriptors);
 394 }
 395
 396 EXPORT_SYMBOL(dma_alloc_descriptor_ring);
 397
 398 /****************************************************************************/
 399 /**
 400 *   Releases the memory which was previously allocated for a descriptor ring.
 401 */
 402 /****************************************************************************/
 403
 404 void dma_free_descriptor_ring(DMA_DescriptorRing_t *ring        /* Descriptor to release */
 405     ) {
 406         if (ring->virtAddr != NULL) {
 407                 dma_free_writecombine(NULL,
 408                                       ring->bytesAllocated,
 409                                       ring->virtAddr, ring->physAddr);
 410         }
 411
 412         ring->bytesAllocated = 0;
 413         ring->descriptorsAllocated = 0;
 414         ring->virtAddr = NULL;
 415         ring->physAddr = 0;
 416 }
 417
 418 EXPORT_SYMBOL(dma_free_descriptor_ring);
 419
 420 /****************************************************************************/
 421 /**
 422 *   Initializes a descriptor ring, so that descriptors can be added to it.
 423 *   Once a descriptor ring has been allocated, it may be reinitialized for
 424 *   use with additional/different regions of memory.
 425 *
 426 *   Note that if 7 descriptors are allocated, it's perfectly acceptable to
 427 *   initialize the ring with a smaller number of descriptors. The amount
 428 *   of memory allocated for the descriptor ring will not be reduced, and
 429 *   the descriptor ring may be reinitialized later
 430 *
 431 *   @return
 432 *       0           Descriptor ring was initialized successfully
 433 *       -ENOMEM     The descriptor which was passed in has insufficient space
 434 *                   to hold the desired number of descriptors.
 435 */
 436 /****************************************************************************/
 437
 438 int dma_init_descriptor_ring(DMA_DescriptorRing_t *ring,        /* Descriptor ring to initialize */
 439                              int numDescriptors /* Number of descriptors to initialize. */
 440     ) {
 441         if (ring->virtAddr == NULL) {
 442                 return -EINVAL;
 443         }
 444         if (dmacHw_initDescriptor(ring->virtAddr,
 445                                   ring->physAddr,
 446                                   ring->bytesAllocated, numDescriptors) < 0) {
 447                 printk(KERN_ERR
 448                        "dma_init_descriptor_ring: dmacHw_initDescriptor failed\n");
 449                 return -ENOMEM;
 450         }
 451
 452         return 0;
 453 }
 454
 455 EXPORT_SYMBOL(dma_init_descriptor_ring);
 456
 457 /****************************************************************************/
 458 /**
 459 *   Determines the number of descriptors which would be required for a
 460 *   transfer of the indicated memory region.
 461 *
 462 *   This function also needs to know which DMA device this transfer will
 463 *   be destined for, so that the appropriate DMA configuration can be retrieved.
 464 *   DMA parameters such as transfer width, and whether this is a memory-to-memory
 465 *   or memory-to-peripheral, etc can all affect the actual number of descriptors
 466 *   required.
 467 *
 468 *   @return
 469 *       > 0     Returns the number of descriptors required for the indicated transfer
 470 *       -ENODEV - Device handed in is invalid.
 471 *       -EINVAL Invalid parameters
 472 *       -ENOMEM Memory exhausted
 473 */
 474 /****************************************************************************/
 475
 476 int dma_calculate_descriptor_count(DMA_Device_t device, /* DMA Device that this will be associated with */
 477                                    dma_addr_t srcData,  /* Place to get data to write to device */
 478                                    dma_addr_t dstData,  /* Pointer to device data address */
 479                                    size_t numBytes      /* Number of bytes to transfer to the device */
 480     ) {
 481         int numDescriptors;
 482         DMA_DeviceAttribute_t *devAttr;
 483
 484         if (!IsDeviceValid(device)) {
 485                 return -ENODEV;
 486         }
 487         devAttr = &DMA_gDeviceAttribute[device];
 488
 489         numDescriptors = dmacHw_calculateDescriptorCount(&devAttr->config,
 490                                                               (void *)srcData,
 491                                                               (void *)dstData,
 492                                                               numBytes);
 493         if (numDescriptors < 0) {
 494                 printk(KERN_ERR
 495                        "dma_calculate_descriptor_count: dmacHw_calculateDescriptorCount failed\n");
 496                 return -EINVAL;
 497         }
 498
 499         return numDescriptors;
 500 }
 501
 502 EXPORT_SYMBOL(dma_calculate_descriptor_count);
 503
 504 /****************************************************************************/
 505 /**
 506 *   Adds a region of memory to the descriptor ring. Note that it may take
 507 *   multiple descriptors for each region of memory. It is the callers
 508 *   responsibility to allocate a sufficiently large descriptor ring.
 509 *
 510 *   @return
 511 *       0       Descriptors were added successfully
 512 *       -ENODEV Device handed in is invalid.
 513 *       -EINVAL Invalid parameters
 514 *       -ENOMEM Memory exhausted
 515 */
 516 /****************************************************************************/
 517
 518 int dma_add_descriptors(DMA_DescriptorRing_t *ring,     /* Descriptor ring to add descriptors to */
 519                         DMA_Device_t device,    /* DMA Device that descriptors are for */
 520                         dma_addr_t srcData,     /* Place to get data (memory or device) */
 521                         dma_addr_t dstData,     /* Place to put data (memory or device) */
 522                         size_t numBytes /* Number of bytes to transfer to the device */
 523     ) {
 524         int rc;
 525         DMA_DeviceAttribute_t *devAttr;
 526
 527         if (!IsDeviceValid(device)) {
 528                 return -ENODEV;
 529         }
 530         devAttr = &DMA_gDeviceAttribute[device];
 531
 532         rc = dmacHw_setDataDescriptor(&devAttr->config,
 533                                       ring->virtAddr,
 534                                       (void *)srcData,
 535                                       (void *)dstData, numBytes);
 536         if (rc < 0) {
 537                 printk(KERN_ERR
 538                        "dma_add_descriptors: dmacHw_setDataDescriptor failed with code: %d\n",
 539                        rc);
 540                 return -ENOMEM;
 541         }
 542
 543         return 0;
 544 }
 545
 546 EXPORT_SYMBOL(dma_add_descriptors);
 547
 548 /****************************************************************************/
 549 /**
 550 *   Sets the descriptor ring associated with a device.
 551 *
 552 *   Once set, the descriptor ring will be associated with the device, even
 553 *   across channel request/free calls. Passing in a NULL descriptor ring
 554 *   will release any descriptor ring currently associated with the device.
 555 *
 556 *   Note: If you call dma_transfer, or one of the other dma_alloc_ functions
 557 *         the descriptor ring may be released and reallocated.
 558 *
 559 *   Note: This function will release the descriptor memory for any current
 560 *         descriptor ring associated with this device.
 561 *
 562 *   @return
 563 *       0       Descriptors were added successfully
 564 *       -ENODEV Device handed in is invalid.
 565 */
 566 /****************************************************************************/
 567
 568 int dma_set_device_descriptor_ring(DMA_Device_t device, /* Device to update the descriptor ring for. */
 569                                    DMA_DescriptorRing_t *ring   /* Descriptor ring to add descriptors to */
 570     ) {
 571         DMA_DeviceAttribute_t *devAttr;
 572
 573         if (!IsDeviceValid(device)) {
 574                 return -ENODEV;
 575         }
 576         devAttr = &DMA_gDeviceAttribute[device];
 577
 578         /* Free the previously allocated descriptor ring */
 579
 580         dma_free_descriptor_ring(&devAttr->ring);
 581
 582         if (ring != NULL) {
 583                 /* Copy in the new one */
 584
 585                 devAttr->ring = *ring;
 586         }
 587
 588         /* Set things up so that if dma_transfer is called then this descriptor */
 589         /* ring will get freed. */
 590
 591         devAttr->prevSrcData = 0;
 592         devAttr->prevDstData = 0;
 593         devAttr->prevNumBytes = 0;
 594
 595         return 0;
 596 }
 597
 598 EXPORT_SYMBOL(dma_set_device_descriptor_ring);
 599
 600 /****************************************************************************/
 601 /**
 602 *   Retrieves the descriptor ring associated with a device.
 603 *
 604 *   @return
 605 *       0       Descriptors were added successfully
 606 *       -ENODEV Device handed in is invalid.
 607 */
 608 /****************************************************************************/
 609
 610 int dma_get_device_descriptor_ring(DMA_Device_t device, /* Device to retrieve the descriptor ring for. */
 611                                    DMA_DescriptorRing_t *ring   /* Place to store retrieved ring */
 612     ) {
 613         DMA_DeviceAttribute_t *devAttr;
 614
 615         memset(ring, 0, sizeof(*ring));
 616
 617         if (!IsDeviceValid(device)) {
 618                 return -ENODEV;
 619         }
 620         devAttr = &DMA_gDeviceAttribute[device];
 621
 622         *ring = devAttr->ring;
 623
 624         return 0;
 625 }
 626
 627 EXPORT_SYMBOL(dma_get_device_descriptor_ring);
 628
 629 /****************************************************************************/
 630 /**
 631 *   Configures a DMA channel.
 632 *
 633 *   @return
 634 *       >= 0    - Initialization was successful.
 635 *
 636 *       -EBUSY  - Device is currently being used.
 637 *       -ENODEV - Device handed in is invalid.
 638 */
 639 /****************************************************************************/
 640
 641 static int ConfigChannel(DMA_Handle_t handle)
 642 {
 643         DMA_Channel_t *channel;
 644         DMA_DeviceAttribute_t *devAttr;
 645         int controllerIdx;
 646
 647         channel = HandleToChannel(handle);
 648         if (channel == NULL) {
 649                 return -ENODEV;
 650         }
 651         devAttr = &DMA_gDeviceAttribute[channel->devType];
 652         controllerIdx = CONTROLLER_FROM_HANDLE(handle);
 653
 654         if ((devAttr->flags & DMA_DEVICE_FLAG_PORT_PER_DMAC) != 0) {
 655                 if (devAttr->config.transferType ==
 656                     dmacHw_TRANSFER_TYPE_MEM_TO_PERIPHERAL) {
 657                         devAttr->config.dstPeripheralPort =
 658                             devAttr->dmacPort[controllerIdx];
 659                 } else if (devAttr->config.transferType ==
 660                            dmacHw_TRANSFER_TYPE_PERIPHERAL_TO_MEM) {
 661                         devAttr->config.srcPeripheralPort =
 662                             devAttr->dmacPort[controllerIdx];
 663                 }
 664         }
 665
 666         if (dmacHw_configChannel(channel->dmacHwHandle, &devAttr->config) != 0) {
 667                 printk(KERN_ERR "ConfigChannel: dmacHw_configChannel failed\n");
 668                 return -EIO;
 669         }
 670
 671         return 0;
 672 }
 673
 674 /****************************************************************************/
 675 /**
 676 *   Initializes all of the data structures associated with the DMA.
 677 *   @return
 678 *       >= 0    - Initialization was successful.
 679 *
 680 *       -EBUSY  - Device is currently being used.
 681 *       -ENODEV - Device handed in is invalid.
 682 */
 683 /****************************************************************************/
 684
 685 int dma_init(void)
 686 {
 687         int rc = 0;
 688         int controllerIdx;
 689         int channelIdx;
 690         DMA_Device_t devIdx;
 691         DMA_Channel_t *channel;
 692         DMA_Handle_t dedicatedHandle;
 693
 694         memset(&gDMA, 0, sizeof(gDMA));
 695
 696         sema_init(&gDMA.lock, 0);
 697         init_waitqueue_head(&gDMA.freeChannelQ);
 698
 699         /* Initialize the Hardware */
 700
 701         dmacHw_initDma();
 702
 703         /* Start off by marking all of the DMA channels as shared. */
 704
 705         for (controllerIdx = 0; controllerIdx < DMA_NUM_CONTROLLERS;
 706              controllerIdx++) {
 707                 for (channelIdx = 0; channelIdx < DMA_NUM_CHANNELS;
 708                      channelIdx++) {
 709                         channel =
 710                             &gDMA.controller[controllerIdx].channel[channelIdx];
 711
 712                         channel->flags = 0;
 713                         channel->devType = DMA_DEVICE_NONE;
 714                         channel->lastDevType = DMA_DEVICE_NONE;
 715
 716 #if (DMA_DEBUG_TRACK_RESERVATION)
 717                         channel->fileName = "";
 718                         channel->lineNum = 0;
 719 #endif
 720
 721                         channel->dmacHwHandle =
 722                             dmacHw_getChannelHandle(dmacHw_MAKE_CHANNEL_ID
 723                                                     (controllerIdx,
 724                                                      channelIdx));
 725                         dmacHw_initChannel(channel->dmacHwHandle);
 726                 }
 727         }
 728
 729         /* Record any special attributes that channels may have */
 730
 731         gDMA.controller[0].channel[0].flags |= DMA_CHANNEL_FLAG_LARGE_FIFO;
 732         gDMA.controller[0].channel[1].flags |= DMA_CHANNEL_FLAG_LARGE_FIFO;
 733         gDMA.controller[1].channel[0].flags |= DMA_CHANNEL_FLAG_LARGE_FIFO;
 734         gDMA.controller[1].channel[1].flags |= DMA_CHANNEL_FLAG_LARGE_FIFO;
 735
 736         /* Now walk through and record the dedicated channels. */
 737
 738         for (devIdx = 0; devIdx < DMA_NUM_DEVICE_ENTRIES; devIdx++) {
 739                 DMA_DeviceAttribute_t *devAttr = &DMA_gDeviceAttribute[devIdx];
 740
 741                 if (((devAttr->flags & DMA_DEVICE_FLAG_NO_ISR) != 0)
 742                     && ((devAttr->flags & DMA_DEVICE_FLAG_IS_DEDICATED) == 0)) {
 743                         printk(KERN_ERR
 744                                "DMA Device: %s Can only request NO_ISR for dedicated devices\n",
 745                                devAttr->name);
 746                         rc = -EINVAL;
 747                         goto out;
 748                 }
 749
 750                 if ((devAttr->flags & DMA_DEVICE_FLAG_IS_DEDICATED) != 0) {
 751                         /* This is a dedicated device. Mark the channel as being reserved. */
 752
 753                         if (devAttr->dedicatedController >= DMA_NUM_CONTROLLERS) {
 754                                 printk(KERN_ERR
 755                                        "DMA Device: %s DMA Controller %d is out of range\n",
 756                                        devAttr->name,
 757                                        devAttr->dedicatedController);
 758                                 rc = -EINVAL;
 759                                 goto out;
 760                         }
 761
 762                         if (devAttr->dedicatedChannel >= DMA_NUM_CHANNELS) {
 763                                 printk(KERN_ERR
 764                                        "DMA Device: %s DMA Channel %d is out of range\n",
 765                                        devAttr->name,
 766                                        devAttr->dedicatedChannel);
 767                                 rc = -EINVAL;
 768                                 goto out;
 769                         }
 770
 771                         dedicatedHandle =
 772                             MAKE_HANDLE(devAttr->dedicatedController,
 773                                         devAttr->dedicatedChannel);
 774                         channel = HandleToChannel(dedicatedHandle);
 775
 776                         if ((channel->flags & DMA_CHANNEL_FLAG_IS_DEDICATED) !=
 777                             0) {
 778                                 printk
 779                                     ("DMA Device: %s attempting to use same DMA Controller:Channel (%d:%d) as %s\n",
 780                                      devAttr->name,
 781                                      devAttr->dedicatedController,
 782                                      devAttr->dedicatedChannel,
 783                                      DMA_gDeviceAttribute[channel->devType].
 784                                      name);
 785                                 rc = -EBUSY;
 786                                 goto out;
 787                         }
 788
 789                         channel->flags |= DMA_CHANNEL_FLAG_IS_DEDICATED;
 790                         channel->devType = devIdx;
 791
 792                         if (devAttr->flags & DMA_DEVICE_FLAG_NO_ISR) {
 793                                 channel->flags |= DMA_CHANNEL_FLAG_NO_ISR;
 794                         }
 795
 796                         /* For dedicated channels, we can go ahead and configure the DMA channel now */
 797                         /* as well. */
 798
 799                         ConfigChannel(dedicatedHandle);
 800                 }
 801         }
 802
 803         /* Go through and register the interrupt handlers */
 804
 805         for (controllerIdx = 0; controllerIdx < DMA_NUM_CONTROLLERS;
 806              controllerIdx++) {
 807                 for (channelIdx = 0; channelIdx < DMA_NUM_CHANNELS;
 808                      channelIdx++) {
 809                         channel =
 810                             &gDMA.controller[controllerIdx].channel[channelIdx];
 811
 812                         if ((channel->flags & DMA_CHANNEL_FLAG_NO_ISR) == 0) {
 813                                 snprintf(channel->name, sizeof(channel->name),
 814                                          "dma %d:%d %s", controllerIdx,
 815                                          channelIdx,
 816                                          channel->devType ==
 817                                          DMA_DEVICE_NONE ? "" :
 818                                          DMA_gDeviceAttribute[channel->devType].
 819                                          name);
 820
 821                                 rc =
 822                                      request_irq(IRQ_DMA0C0 +
 823                                                  (controllerIdx *
 824                                                   DMA_NUM_CHANNELS) +
 825                                                  channelIdx,
 826                                                  dma_interrupt_handler,
 827                                                  IRQF_DISABLED, channel->name,
 828                                                  channel);
 829                                 if (rc != 0) {
 830                                         printk(KERN_ERR
 831                                                "request_irq for IRQ_DMA%dC%d failed\n",
 832                                                controllerIdx, channelIdx);
 833                                 }
 834                         }
 835                 }
 836         }
 837
 838         /* Create /proc/dma/channels and /proc/dma/devices */
 839
 840         gDmaDir = proc_mkdir("dma", NULL);
 841
 842         if (gDmaDir == NULL) {
 843                 printk(KERN_ERR "Unable to create /proc/dma\n");
 844         } else {
 845                 create_proc_read_entry("channels", 0, gDmaDir,
 846                                        dma_proc_read_channels, NULL);
 847                 create_proc_read_entry("devices", 0, gDmaDir,
 848                                        dma_proc_read_devices, NULL);
 849                 create_proc_read_entry("mem-type", 0, gDmaDir,
 850                                        dma_proc_read_mem_type, NULL);
 851         }
 852
 853 out:
 854
 855         up(&gDMA.lock);
 856
 857         return rc;
 858 }
 859
 860 /****************************************************************************/
 861 /**
 862 *   Reserves a channel for use with @a dev. If the device is setup to use
 863 *   a shared channel, then this function will block until a free channel
 864 *   becomes available.
 865 *
 866 *   @return
 867 *       >= 0    - A valid DMA Handle.
 868 *       -EBUSY  - Device is currently being used.
 869 *       -ENODEV - Device handed in is invalid.
 870 */
 871 /****************************************************************************/
 872
 873 #if (DMA_DEBUG_TRACK_RESERVATION)
 874 DMA_Handle_t dma_request_channel_dbg
 875     (DMA_Device_t dev, const char *fileName, int lineNum)
 876 #else
 877 DMA_Handle_t dma_request_channel(DMA_Device_t dev)
 878 #endif
 879 {
 880         DMA_Handle_t handle;
 881         DMA_DeviceAttribute_t *devAttr;
 882         DMA_Channel_t *channel;
 883         int controllerIdx;
 884         int controllerIdx2;
 885         int channelIdx;
 886
 887         if (down_interruptible(&gDMA.lock) < 0) {
 888                 return -ERESTARTSYS;
 889         }
 890
 891         if ((dev < 0) || (dev >= DMA_NUM_DEVICE_ENTRIES)) {
 892                 handle = -ENODEV;
 893                 goto out;
 894         }
 895         devAttr = &DMA_gDeviceAttribute[dev];
 896
 897 #if (DMA_DEBUG_TRACK_RESERVATION)
 898         {
 899                 char *s;
 900
 901                 s = strrchr(fileName, '/');
 902                 if (s != NULL) {
 903                         fileName = s + 1;
 904                 }
 905         }
 906 #endif
 907         if ((devAttr->flags & DMA_DEVICE_FLAG_IN_USE) != 0) {
 908                 /* This device has already been requested and not been freed */
 909
 910                 printk(KERN_ERR "%s: device %s is already requested\n",
 911                        __func__, devAttr->name);
 912                 handle = -EBUSY;
 913                 goto out;
 914         }
 915
 916         if ((devAttr->flags & DMA_DEVICE_FLAG_IS_DEDICATED) != 0) {
 917                 /* This device has a dedicated channel. */
 918
 919                 channel =
 920                     &gDMA.controller[devAttr->dedicatedController].
 921                     channel[devAttr->dedicatedChannel];
 922                 if ((channel->flags & DMA_CHANNEL_FLAG_IN_USE) != 0) {
 923                         handle = -EBUSY;
 924                         goto out;
 925                 }
 926
 927                 channel->flags |= DMA_CHANNEL_FLAG_IN_USE;
 928                 devAttr->flags |= DMA_DEVICE_FLAG_IN_USE;
 929
 930 #if (DMA_DEBUG_TRACK_RESERVATION)
 931                 channel->fileName = fileName;
 932                 channel->lineNum = lineNum;
 933 #endif
 934                 handle =
 935                     MAKE_HANDLE(devAttr->dedicatedController,
 936                                 devAttr->dedicatedChannel);
 937                 goto out;
 938         }
 939
 940         /* This device needs to use one of the shared channels. */
 941
 942         handle = DMA_INVALID_HANDLE;
 943         while (handle == DMA_INVALID_HANDLE) {
 944                 /* Scan through the shared channels and see if one is available */
 945
 946                 for (controllerIdx2 = 0; controllerIdx2 < DMA_NUM_CONTROLLERS;
 947                      controllerIdx2++) {
 948                         /* Check to see if we should try on controller 1 first. */
 949
 950                         controllerIdx = controllerIdx2;
 951                         if ((devAttr->
 952                              flags & DMA_DEVICE_FLAG_ALLOC_DMA1_FIRST) != 0) {
 953                                 controllerIdx = 1 - controllerIdx;
 954                         }
 955
 956                         /* See if the device is available on the controller being tested */
 957
 958                         if ((devAttr->
 959                              flags & (DMA_DEVICE_FLAG_ON_DMA0 << controllerIdx))
 960                             != 0) {
 961                                 for (channelIdx = 0;
 962                                      channelIdx < DMA_NUM_CHANNELS;
 963                                      channelIdx++) {
 964                                         channel =
 965                                             &gDMA.controller[controllerIdx].
 966                                             channel[channelIdx];
 967
 968                                         if (((channel->
 969                                               flags &
 970                                               DMA_CHANNEL_FLAG_IS_DEDICATED) ==
 971                                              0)
 972                                             &&
 973                                             ((channel->
 974                                               flags & DMA_CHANNEL_FLAG_IN_USE)
 975                                              == 0)) {
 976                                                 if (((channel->
 977                                                       flags &
 978                                                       DMA_CHANNEL_FLAG_LARGE_FIFO)
 979                                                      != 0)
 980                                                     &&
 981                                                     ((devAttr->
 982                                                       flags &
 983                                                       DMA_DEVICE_FLAG_ALLOW_LARGE_FIFO)
 984                                                      == 0)) {
 985                                                         /* This channel is a large fifo - don't tie it up */
 986                                                         /* with devices that we don't want using it. */
 987
 988                                                         continue;
 989                                                 }
 990
 991                                                 channel->flags |=
 992                                                     DMA_CHANNEL_FLAG_IN_USE;
 993                                                 channel->devType = dev;
 994                                                 devAttr->flags |=
 995                                                     DMA_DEVICE_FLAG_IN_USE;
 996
 997 #if (DMA_DEBUG_TRACK_RESERVATION)
 998                                                 channel->fileName = fileName;
 999                                                 channel->lineNum = lineNum;
1000 #endif
1001                                                 handle =
1002                                                     MAKE_HANDLE(controllerIdx,
1003                                                                 channelIdx);
1004
1005                                                 /* Now that we've reserved the channel - we can go ahead and configure it */
1006
1007                                                 if (ConfigChannel(handle) != 0) {
1008                                                         handle = -EIO;
1009                                                         printk(KERN_ERR
1010                                                                "dma_request_channel: ConfigChannel failed\n");
1011                                                 }
1012                                                 goto out;
1013                                         }
1014                                 }
1015                         }
1016                 }
1017
1018                 /* No channels are currently available. Let's wait for one to free up. */
1019
1020                 {
1021                         DEFINE_WAIT(wait);
1022
1023                         prepare_to_wait(&gDMA.freeChannelQ, &wait,
1024                                         TASK_INTERRUPTIBLE);
1025                         up(&gDMA.lock);
1026                         schedule();
1027                         finish_wait(&gDMA.freeChannelQ, &wait);
1028
1029                         if (signal_pending(current)) {
1030                                 /* We don't currently hold gDMA.lock, so we return directly */
1031
1032                                 return -ERESTARTSYS;
1033                         }
1034                 }
1035
1036                 if (down_interruptible(&gDMA.lock)) {
1037                         return -ERESTARTSYS;
1038                 }
1039         }
1040
1041 out:
1042         up(&gDMA.lock);
1043
1044         return handle;
1045 }
1046
1047 /* Create both _dbg and non _dbg functions for modules. */
1048
1049 #if (DMA_DEBUG_TRACK_RESERVATION)
1050 #undef dma_request_channel
1051 DMA_Handle_t dma_request_channel(DMA_Device_t dev)
1052 {
1053         return dma_request_channel_dbg(dev, __FILE__, __LINE__);
1054 }
1055
1056 EXPORT_SYMBOL(dma_request_channel_dbg);
1057 #endif
1058 EXPORT_SYMBOL(dma_request_channel);
1059
1060 /****************************************************************************/
1061 /**
1062 *   Frees a previously allocated DMA Handle.
1063 */
1064 /****************************************************************************/
1065
1066 int dma_free_channel(DMA_Handle_t handle        /* DMA handle. */
1067     ) {
1068         int rc = 0;
1069         DMA_Channel_t *channel;
1070         DMA_DeviceAttribute_t *devAttr;
1071
1072         if (down_interruptible(&gDMA.lock) < 0) {
1073                 return -ERESTARTSYS;
1074         }
1075
1076         channel = HandleToChannel(handle);
1077         if (channel == NULL) {
1078                 rc = -EINVAL;
1079                 goto out;
1080         }
1081
1082         devAttr = &DMA_gDeviceAttribute[channel->devType];
1083
1084         if ((channel->flags & DMA_CHANNEL_FLAG_IS_DEDICATED) == 0) {
1085                 channel->lastDevType = channel->devType;
1086                 channel->devType = DMA_DEVICE_NONE;
1087         }
1088         channel->flags &= ~DMA_CHANNEL_FLAG_IN_USE;
1089         devAttr->flags &= ~DMA_DEVICE_FLAG_IN_USE;
1090
1091 out:
1092         up(&gDMA.lock);
1093
1094         wake_up_interruptible(&gDMA.freeChannelQ);
1095
1096         return rc;
1097 }
1098
1099 EXPORT_SYMBOL(dma_free_channel);
1100
1101 /****************************************************************************/
1102 /**
1103 *   Determines if a given device has been configured as using a shared
1104 *   channel.
1105 *
1106 *   @return
1107 *       0           Device uses a dedicated channel
1108 *       > zero      Device uses a shared channel
1109 *       < zero      Error code
1110 */
1111 /****************************************************************************/
1112
1113 int dma_device_is_channel_shared(DMA_Device_t device    /* Device to check. */
1114     ) {
1115         DMA_DeviceAttribute_t *devAttr;
1116
1117         if (!IsDeviceValid(device)) {
1118                 return -ENODEV;
1119         }
1120         devAttr = &DMA_gDeviceAttribute[device];
1121
1122         return ((devAttr->flags & DMA_DEVICE_FLAG_IS_DEDICATED) == 0);
1123 }
1124
1125 EXPORT_SYMBOL(dma_device_is_channel_shared);
1126
1127 /****************************************************************************/
1128 /**
1129 *   Allocates buffers for the descriptors. This is normally done automatically
1130 *   but needs to be done explicitly when initiating a dma from interrupt
1131 *   context.
1132 *
1133 *   @return
1134 *       0       Descriptors were allocated successfully
1135 *       -EINVAL Invalid device type for this kind of transfer
1136 *               (i.e. the device is _MEM_TO_DEV and not _DEV_TO_MEM)
1137 *       -ENOMEM Memory exhausted
1138 */
1139 /****************************************************************************/
1140
1141 int dma_alloc_descriptors(DMA_Handle_t handle,  /* DMA Handle */
1142                           dmacHw_TRANSFER_TYPE_e transferType,  /* Type of transfer being performed */
1143                           dma_addr_t srcData,   /* Place to get data to write to device */
1144                           dma_addr_t dstData,   /* Pointer to device data address */
1145                           size_t numBytes       /* Number of bytes to transfer to the device */
1146     ) {
1147         DMA_Channel_t *channel;
1148         DMA_DeviceAttribute_t *devAttr;
1149         int numDescriptors;
1150         size_t ringBytesRequired;
1151         int rc = 0;
1152
1153         channel = HandleToChannel(handle);
1154         if (channel == NULL) {
1155                 return -ENODEV;
1156         }
1157
1158         devAttr = &DMA_gDeviceAttribute[channel->devType];
1159
1160         if (devAttr->config.transferType != transferType) {
1161                 return -EINVAL;
1162         }
1163
1164         /* Figure out how many descriptors we need. */
1165
1166         /* printk("srcData: 0x%08x dstData: 0x%08x, numBytes: %d\n", */
1167         /*        srcData, dstData, numBytes); */
1168
1169         numDescriptors = dmacHw_calculateDescriptorCount(&devAttr->config,
1170                                                               (void *)srcData,
1171                                                               (void *)dstData,
1172                                                               numBytes);
1173         if (numDescriptors < 0) {
1174                 printk(KERN_ERR "%s: dmacHw_calculateDescriptorCount failed\n",
1175                        __func__);
1176                 return -EINVAL;
1177         }
1178
1179         /* Check to see if we can reuse the existing descriptor ring, or if we need to allocate */
1180         /* a new one. */
1181
1182         ringBytesRequired = dmacHw_descriptorLen(numDescriptors);
1183
1184         /* printk("ringBytesRequired: %d\n", ringBytesRequired); */
1185
1186         if (ringBytesRequired > devAttr->ring.bytesAllocated) {
1187                 /* Make sure that this code path is never taken from interrupt context. */
1188                 /* It's OK for an interrupt to initiate a DMA transfer, but the descriptor */
1189                 /* allocation needs to have already been done. */
1190
1191                 might_sleep();
1192
1193                 /* Free the old descriptor ring and allocate a new one. */
1194
1195                 dma_free_descriptor_ring(&devAttr->ring);
1196
1197                 /* And allocate a new one. */
1198
1199                 rc =
1200                      dma_alloc_descriptor_ring(&devAttr->ring,
1201                                                numDescriptors);
1202                 if (rc < 0) {
1203                         printk(KERN_ERR
1204                                "%s: dma_alloc_descriptor_ring(%d) failed\n",
1205                                __func__, numDescriptors);
1206                         return rc;
1207                 }
1208                 /* Setup the descriptor for this transfer */
1209
1210                 if (dmacHw_initDescriptor(devAttr->ring.virtAddr,
1211                                           devAttr->ring.physAddr,
1212                                           devAttr->ring.bytesAllocated,
1213                                           numDescriptors) < 0) {
1214                         printk(KERN_ERR "%s: dmacHw_initDescriptor failed\n",
1215                                __func__);
1216                         return -EINVAL;
1217                 }
1218         } else {
1219                 /* We've already got enough ring buffer allocated. All we need to do is reset */
1220                 /* any control information, just in case the previous DMA was stopped. */
1221
1222                 dmacHw_resetDescriptorControl(devAttr->ring.virtAddr);
1223         }
1224
1225         /* dma_alloc/free both set the prevSrc/DstData to 0. If they happen to be the same */
1226         /* as last time, then we don't need to call setDataDescriptor again. */
1227
1228         if (dmacHw_setDataDescriptor(&devAttr->config,
1229                                      devAttr->ring.virtAddr,
1230                                      (void *)srcData,
1231                                      (void *)dstData, numBytes) < 0) {
1232                 printk(KERN_ERR "%s: dmacHw_setDataDescriptor failed\n",
1233                        __func__);
1234                 return -EINVAL;
1235         }
1236
1237         /* Remember the critical information for this transfer so that we can eliminate */
1238         /* another call to dma_alloc_descriptors if the caller reuses the same buffers */
1239
1240         devAttr->prevSrcData = srcData;
1241         devAttr->prevDstData = dstData;
1242         devAttr->prevNumBytes = numBytes;
1243
1244         return 0;
1245 }
1246
1247 EXPORT_SYMBOL(dma_alloc_descriptors);
1248
1249 /****************************************************************************/
1250 /**
1251 *   Allocates and sets up descriptors for a double buffered circular buffer.
1252 *
1253 *   This is primarily intended to be used for things like the ingress samples
1254 *   from a microphone.
1255 *
1256 *   @return
1257 *       > 0     Number of descriptors actually allocated.
1258 *       -EINVAL Invalid device type for this kind of transfer
1259 *               (i.e. the device is _MEM_TO_DEV and not _DEV_TO_MEM)
1260 *       -ENOMEM Memory exhausted
1261 */
1262 /****************************************************************************/
1263
1264 int dma_alloc_double_dst_descriptors(DMA_Handle_t handle,       /* DMA Handle */
1265                                      dma_addr_t srcData,        /* Physical address of source data */
1266                                      dma_addr_t dstData1,       /* Physical address of first destination buffer */
1267                                      dma_addr_t dstData2,       /* Physical address of second destination buffer */
1268                                      size_t numBytes    /* Number of bytes in each destination buffer */
1269     ) {
1270         DMA_Channel_t *channel;
1271         DMA_DeviceAttribute_t *devAttr;
1272         int numDst1Descriptors;
1273         int numDst2Descriptors;
1274         int numDescriptors;
1275         size_t ringBytesRequired;
1276         int rc = 0;
1277
1278         channel = HandleToChannel(handle);
1279         if (channel == NULL) {
1280                 return -ENODEV;
1281         }
1282
1283         devAttr = &DMA_gDeviceAttribute[channel->devType];
1284
1285         /* Figure out how many descriptors we need. */
1286
1287         /* printk("srcData: 0x%08x dstData: 0x%08x, numBytes: %d\n", */
1288         /*        srcData, dstData, numBytes); */
1289
1290         numDst1Descriptors =
1291              dmacHw_calculateDescriptorCount(&devAttr->config, (void *)srcData,
1292                                              (void *)dstData1, numBytes);
1293         if (numDst1Descriptors < 0) {
1294                 return -EINVAL;
1295         }
1296         numDst2Descriptors =
1297              dmacHw_calculateDescriptorCount(&devAttr->config, (void *)srcData,
1298                                              (void *)dstData2, numBytes);
1299         if (numDst2Descriptors < 0) {
1300                 return -EINVAL;
1301         }
1302         numDescriptors = numDst1Descriptors + numDst2Descriptors;
1303         /* printk("numDescriptors: %d\n", numDescriptors); */
1304
1305         /* Check to see if we can reuse the existing descriptor ring, or if we need to allocate */
1306         /* a new one. */
1307
1308         ringBytesRequired = dmacHw_descriptorLen(numDescriptors);
1309
1310         /* printk("ringBytesRequired: %d\n", ringBytesRequired); */
1311
1312         if (ringBytesRequired > devAttr->ring.bytesAllocated) {
1313                 /* Make sure that this code path is never taken from interrupt context. */
1314                 /* It's OK for an interrupt to initiate a DMA transfer, but the descriptor */
1315                 /* allocation needs to have already been done. */
1316
1317                 might_sleep();
1318
1319                 /* Free the old descriptor ring and allocate a new one. */
1320
1321                 dma_free_descriptor_ring(&devAttr->ring);
1322
1323                 /* And allocate a new one. */
1324
1325                 rc =
1326                      dma_alloc_descriptor_ring(&devAttr->ring,
1327                                                numDescriptors);
1328                 if (rc < 0) {
1329                         printk(KERN_ERR
1330                                "%s: dma_alloc_descriptor_ring(%d) failed\n",
1331                                __func__, ringBytesRequired);
1332                         return rc;
1333                 }
1334         }
1335
1336         /* Setup the descriptor for this transfer. Since this function is used with */
1337         /* CONTINUOUS DMA operations, we need to reinitialize every time, otherwise */
1338         /* setDataDescriptor will keep trying to append onto the end. */
1339
1340         if (dmacHw_initDescriptor(devAttr->ring.virtAddr,
1341                                   devAttr->ring.physAddr,
1342                                   devAttr->ring.bytesAllocated,
1343                                   numDescriptors) < 0) {
1344                 printk(KERN_ERR "%s: dmacHw_initDescriptor failed\n", __func__);
1345                 return -EINVAL;
1346         }
1347
1348         /* dma_alloc/free both set the prevSrc/DstData to 0. If they happen to be the same */
1349         /* as last time, then we don't need to call setDataDescriptor again. */
1350
1351         if (dmacHw_setDataDescriptor(&devAttr->config,
1352                                      devAttr->ring.virtAddr,
1353                                      (void *)srcData,
1354                                      (void *)dstData1, numBytes) < 0) {
1355                 printk(KERN_ERR "%s: dmacHw_setDataDescriptor 1 failed\n",
1356                        __func__);
1357                 return -EINVAL;
1358         }
1359         if (dmacHw_setDataDescriptor(&devAttr->config,
1360                                      devAttr->ring.virtAddr,
1361                                      (void *)srcData,
1362                                      (void *)dstData2, numBytes) < 0) {
1363                 printk(KERN_ERR "%s: dmacHw_setDataDescriptor 2 failed\n",
1364                        __func__);
1365                 return -EINVAL;
1366         }
1367
1368         /* You should use dma_start_transfer rather than dma_transfer_xxx so we don't */
1369         /* try to make the 'prev' variables right. */
1370
1371         devAttr->prevSrcData = 0;
1372         devAttr->prevDstData = 0;
1373         devAttr->prevNumBytes = 0;
1374
1375         return numDescriptors;
1376 }
1377
1378 EXPORT_SYMBOL(dma_alloc_double_dst_descriptors);
1379
1380 /****************************************************************************/
1381 /**
1382 *   Initiates a transfer when the descriptors have already been setup.
1383 *
1384 *   This is a special case, and normally, the dma_transfer_xxx functions should
1385 *   be used.
1386 *
1387 *   @return
1388 *       0       Transfer was started successfully
1389 *       -ENODEV Invalid handle
1390 */
1391 /****************************************************************************/
1392
1393 int dma_start_transfer(DMA_Handle_t handle)
1394 {
1395         DMA_Channel_t *channel;
1396         DMA_DeviceAttribute_t *devAttr;
1397
1398         channel = HandleToChannel(handle);
1399         if (channel == NULL) {
1400                 return -ENODEV;
1401         }
1402         devAttr = &DMA_gDeviceAttribute[channel->devType];
1403
1404         dmacHw_initiateTransfer(channel->dmacHwHandle, &devAttr->config,
1405                                 devAttr->ring.virtAddr);
1406
1407         /* Since we got this far, everything went successfully */
1408
1409         return 0;
1410 }
1411
1412 EXPORT_SYMBOL(dma_start_transfer);
1413
1414 /****************************************************************************/
1415 /**
1416 *   Stops a previously started DMA transfer.
1417 *
1418 *   @return
1419 *       0       Transfer was stopped successfully
1420 *       -ENODEV Invalid handle
1421 */
1422 /****************************************************************************/
1423
1424 int dma_stop_transfer(DMA_Handle_t handle)
1425 {
1426         DMA_Channel_t *channel;
1427
1428         channel = HandleToChannel(handle);
1429         if (channel == NULL) {
1430                 return -ENODEV;
1431         }
1432
1433         dmacHw_stopTransfer(channel->dmacHwHandle);
1434
1435         return 0;
1436 }
1437
1438 EXPORT_SYMBOL(dma_stop_transfer);
1439
1440 /****************************************************************************/
1441 /**
1442 *   Waits for a DMA to complete by polling. This function is only intended
1443 *   to be used for testing. Interrupts should be used for most DMA operations.
1444 */
1445 /****************************************************************************/
1446
1447 int dma_wait_transfer_done(DMA_Handle_t handle)
1448 {
1449         DMA_Channel_t *channel;
1450         dmacHw_TRANSFER_STATUS_e status;
1451
1452         channel = HandleToChannel(handle);
1453         if (channel == NULL) {
1454                 return -ENODEV;
1455         }
1456
1457         while ((status =
1458                 dmacHw_transferCompleted(channel->dmacHwHandle)) ==
1459                dmacHw_TRANSFER_STATUS_BUSY) {
1460                 ;
1461         }
1462
1463         if (status == dmacHw_TRANSFER_STATUS_ERROR) {
1464                 printk(KERN_ERR "%s: DMA transfer failed\n", __func__);
1465                 return -EIO;
1466         }
1467         return 0;
1468 }
1469
1470 EXPORT_SYMBOL(dma_wait_transfer_done);
1471
1472 /****************************************************************************/
1473 /**
1474 *   Initiates a DMA, allocating the descriptors as required.
1475 *
1476 *   @return
1477 *       0       Transfer was started successfully
1478 *       -EINVAL Invalid device type for this kind of transfer
1479 *               (i.e. the device is _DEV_TO_MEM and not _MEM_TO_DEV)
1480 */
1481 /****************************************************************************/
1482
1483 int dma_transfer(DMA_Handle_t handle,   /* DMA Handle */
1484                  dmacHw_TRANSFER_TYPE_e transferType,   /* Type of transfer being performed */
1485                  dma_addr_t srcData,    /* Place to get data to write to device */
1486                  dma_addr_t dstData,    /* Pointer to device data address */
1487                  size_t numBytes        /* Number of bytes to transfer to the device */
1488     ) {
1489         DMA_Channel_t *channel;
1490         DMA_DeviceAttribute_t *devAttr;
1491         int rc = 0;
1492
1493         channel = HandleToChannel(handle);
1494         if (channel == NULL) {
1495                 return -ENODEV;
1496         }
1497
1498         devAttr = &DMA_gDeviceAttribute[channel->devType];
1499
1500         if (devAttr->config.transferType != transferType) {
1501                 return -EINVAL;
1502         }
1503
1504         /* We keep track of the information about the previous request for this */
1505         /* device, and if the attributes match, then we can use the descriptors we setup */
1506         /* the last time, and not have to reinitialize everything. */
1507
1508         {
1509                 rc =
1510                      dma_alloc_descriptors(handle, transferType, srcData,
1511                                            dstData, numBytes);
1512                 if (rc != 0) {
1513                         return rc;
1514                 }
1515         }
1516
1517         /* And kick off the transfer */
1518
1519         devAttr->numBytes = numBytes;
1520         devAttr->transferStartTime = timer_get_tick_count();
1521
1522         dmacHw_initiateTransfer(channel->dmacHwHandle, &devAttr->config,
1523                                 devAttr->ring.virtAddr);
1524
1525         /* Since we got this far, everything went successfully */
1526
1527         return 0;
1528 }
1529
1530 EXPORT_SYMBOL(dma_transfer);
1531
1532 /****************************************************************************/
1533 /**
1534 *   Set the callback function which will be called when a transfer completes.
1535 *   If a NULL callback function is set, then no callback will occur.
1536 *
1537 *   @note   @a devHandler will be called from IRQ context.
1538 *
1539 *   @return
1540 *       0       - Success
1541 *       -ENODEV - Device handed in is invalid.
1542 */
1543 /****************************************************************************/
1544
1545 int dma_set_device_handler(DMA_Device_t dev,    /* Device to set the callback for. */
1546                            DMA_DeviceHandler_t devHandler,      /* Function to call when the DMA completes */
1547                            void *userData       /* Pointer which will be passed to devHandler. */
1548     ) {
1549         DMA_DeviceAttribute_t *devAttr;
1550         unsigned long flags;
1551
1552         if (!IsDeviceValid(dev)) {
1553                 return -ENODEV;
1554         }
1555         devAttr = &DMA_gDeviceAttribute[dev];
1556
1557         local_irq_save(flags);
1558
1559         devAttr->userData = userData;
1560         devAttr->devHandler = devHandler;
1561
1562         local_irq_restore(flags);
1563
1564         return 0;
1565 }
1566
1567 EXPORT_SYMBOL(dma_set_device_handler);
1568
1569 /****************************************************************************/
1570 /**
1571 *   Initializes a memory mapping structure
1572 */
1573 /****************************************************************************/
1574
1575 int dma_init_mem_map(DMA_MemMap_t *memMap)
1576 {
1577         memset(memMap, 0, sizeof(*memMap));
1578
1579         sema_init(&memMap->lock, 1);
1580
1581         return 0;
1582 }
1583
1584 EXPORT_SYMBOL(dma_init_mem_map);
1585
1586 /****************************************************************************/
1587 /**
1588 *   Releases any memory currently being held by a memory mapping structure.
1589 */
1590 /****************************************************************************/
1591
1592 int dma_term_mem_map(DMA_MemMap_t *memMap)
1593 {
1594         down(&memMap->lock);    /* Just being paranoid */
1595
1596         /* Free up any allocated memory */
1597
1598         up(&memMap->lock);
1599         memset(memMap, 0, sizeof(*memMap));
1600
1601         return 0;
1602 }
1603
1604 EXPORT_SYMBOL(dma_term_mem_map);
1605
1606 /****************************************************************************/
1607 /**
1608 *   Looks at a memory address and categorizes it.
1609 *
1610 *   @return One of the values from the DMA_MemType_t enumeration.
1611 */
1612 /****************************************************************************/
1613
1614 DMA_MemType_t dma_mem_type(void *addr)
1615 {
1616         unsigned long addrVal = (unsigned long)addr;
1617
1618         if (addrVal >= VMALLOC_END) {
1619                 /* NOTE: DMA virtual memory space starts at 0xFFxxxxxx */
1620
1621                 /* dma_alloc_xxx pages are physically and virtually contiguous */
1622
1623                 return DMA_MEM_TYPE_DMA;
1624         }
1625
1626         /* Technically, we could add one more classification. Addresses between VMALLOC_END */
1627         /* and the beginning of the DMA virtual address could be considered to be I/O space. */
1628         /* Right now, nobody cares about this particular classification, so we ignore it. */
1629
1630         if (is_vmalloc_addr(addr)) {
1631                 /* Address comes from the vmalloc'd region. Pages are virtually */
1632                 /* contiguous but NOT physically contiguous */
1633
1634                 return DMA_MEM_TYPE_VMALLOC;
1635         }
1636
1637         if (addrVal >= PAGE_OFFSET) {
1638                 /* PAGE_OFFSET is typically 0xC0000000 */
1639
1640                 /* kmalloc'd pages are physically contiguous */
1641
1642                 return DMA_MEM_TYPE_KMALLOC;
1643         }
1644
1645         return DMA_MEM_TYPE_USER;
1646 }
1647
1648 EXPORT_SYMBOL(dma_mem_type);
1649
1650 /****************************************************************************/
1651 /**
1652 *   Looks at a memory address and determines if we support DMA'ing to/from
1653 *   that type of memory.
1654 *
1655 *   @return boolean -
1656 *               return value != 0 means dma supported
1657 *               return value == 0 means dma not supported
1658 */
1659 /****************************************************************************/
1660
1661 int dma_mem_supports_dma(void *addr)
1662 {
1663         DMA_MemType_t memType = dma_mem_type(addr);
1664
1665         return (memType == DMA_MEM_TYPE_DMA)
1666 #if ALLOW_MAP_OF_KMALLOC_MEMORY
1667             || (memType == DMA_MEM_TYPE_KMALLOC)
1668 #endif
1669             || (memType == DMA_MEM_TYPE_USER);
1670 }
1671
1672 EXPORT_SYMBOL(dma_mem_supports_dma);
1673
1674 /****************************************************************************/
1675 /**
1676 *   Maps in a memory region such that it can be used for performing a DMA.
1677 *
1678 *   @return
1679 */
1680 /****************************************************************************/
1681
1682 int dma_map_start(DMA_MemMap_t *memMap, /* Stores state information about the map */
1683                   enum dma_data_direction dir   /* Direction that the mapping will be going */
1684     ) {
1685         int rc;
1686
1687         down(&memMap->lock);
1688
1689         DMA_MAP_PRINT("memMap: %p\n", memMap);
1690
1691         if (memMap->inUse) {
1692                 printk(KERN_ERR "%s: memory map %p is already being used\n",
1693                        __func__, memMap);
1694                 rc = -EBUSY;
1695                 goto out;
1696         }
1697
1698         memMap->inUse = 1;
1699         memMap->dir = dir;
1700         memMap->numRegionsUsed = 0;
1701
1702         rc = 0;
1703
1704 out:
1705
1706         DMA_MAP_PRINT("returning %d", rc);
1707
1708         up(&memMap->lock);
1709
1710         return rc;
1711 }
1712
1713 EXPORT_SYMBOL(dma_map_start);
1714
1715 /****************************************************************************/
1716 /**
1717 *   Adds a segment of memory to a memory map. Each segment is both
1718 *   physically and virtually contiguous.
1719 *
1720 *   @return     0 on success, error code otherwise.
1721 */
1722 /****************************************************************************/
1723
1724 static int dma_map_add_segment(DMA_MemMap_t *memMap,    /* Stores state information about the map */
1725                                DMA_Region_t *region,    /* Region that the segment belongs to */
1726                                void *virtAddr,  /* Virtual address of the segment being added */
1727                                dma_addr_t physAddr,     /* Physical address of the segment being added */
1728                                size_t numBytes  /* Number of bytes of the segment being added */
1729     ) {
1730         DMA_Segment_t *segment;
1731
1732         DMA_MAP_PRINT("memMap:%p va:%p pa:0x%x #:%d\n", memMap, virtAddr,
1733                       physAddr, numBytes);
1734
1735         /* Sanity check */
1736
1737         if (((unsigned long)virtAddr < (unsigned long)region->virtAddr)
1738             || (((unsigned long)virtAddr + numBytes)) >
1739             ((unsigned long)region->virtAddr + region->numBytes)) {
1740                 printk(KERN_ERR
1741                        "%s: virtAddr %p is outside region @ %p len: %d\n",
1742                        __func__, virtAddr, region->virtAddr, region->numBytes);
1743                 return -EINVAL;
1744         }
1745
1746         if (region->numSegmentsUsed > 0) {
1747                 /* Check to see if this segment is physically contiguous with the previous one */
1748
1749                 segment = &region->segment[region->numSegmentsUsed - 1];
1750
1751                 if ((segment->physAddr + segment->numBytes) == physAddr) {
1752                         /* It is - just add on to the end */
1753
1754                         DMA_MAP_PRINT("appending %d bytes to last segment\n",
1755                                       numBytes);
1756
1757                         segment->numBytes += numBytes;
1758
1759                         return 0;
1760                 }
1761         }
1762
1763         /* Reallocate to hold more segments, if required. */
1764
1765         if (region->numSegmentsUsed >= region->numSegmentsAllocated) {
1766                 DMA_Segment_t *newSegment;
1767                 size_t oldSize =
1768                     region->numSegmentsAllocated * sizeof(*newSegment);
1769                 int newAlloc = region->numSegmentsAllocated + 4;
1770                 size_t newSize = newAlloc * sizeof(*newSegment);
1771
1772                 newSegment = kmalloc(newSize, GFP_KERNEL);
1773                 if (newSegment == NULL) {
1774                         return -ENOMEM;
1775                 }
1776                 memcpy(newSegment, region->segment, oldSize);
1777                 memset(&((uint8_t *) newSegment)[oldSize], 0,
1778                        newSize - oldSize);
1779                 kfree(region->segment);
1780
1781                 region->numSegmentsAllocated = newAlloc;
1782                 region->segment = newSegment;
1783         }
1784
1785         segment = &region->segment[region->numSegmentsUsed];
1786         region->numSegmentsUsed++;
1787
1788         segment->virtAddr = virtAddr;
1789         segment->physAddr = physAddr;
1790         segment->numBytes = numBytes;
1791
1792         DMA_MAP_PRINT("returning success\n");
1793
1794         return 0;
1795 }
1796
1797 /****************************************************************************/
1798 /**
1799 *   Adds a region of memory to a memory map. Each region is virtually
1800 *   contiguous, but not necessarily physically contiguous.
1801 *
1802 *   @return     0 on success, error code otherwise.
1803 */
1804 /****************************************************************************/
1805
1806 int dma_map_add_region(DMA_MemMap_t *memMap,    /* Stores state information about the map */
1807                        void *mem,       /* Virtual address that we want to get a map of */
1808                        size_t numBytes  /* Number of bytes being mapped */
1809     ) {
1810         unsigned long addr = (unsigned long)mem;
1811         unsigned int offset;
1812         int rc = 0;
1813         DMA_Region_t *region;
1814         dma_addr_t physAddr;
1815
1816         down(&memMap->lock);
1817
1818         DMA_MAP_PRINT("memMap:%p va:%p #:%d\n", memMap, mem, numBytes);
1819
1820         if (!memMap->inUse) {
1821                 printk(KERN_ERR "%s: Make sure you call dma_map_start first\n",
1822                        __func__);
1823                 rc = -EINVAL;
1824                 goto out;
1825         }
1826
1827         /* Reallocate to hold more regions. */
1828
1829         if (memMap->numRegionsUsed >= memMap->numRegionsAllocated) {
1830                 DMA_Region_t *newRegion;
1831                 size_t oldSize =
1832                     memMap->numRegionsAllocated * sizeof(*newRegion);
1833                 int newAlloc = memMap->numRegionsAllocated + 4;
1834                 size_t newSize = newAlloc * sizeof(*newRegion);
1835
1836                 newRegion = kmalloc(newSize, GFP_KERNEL);
1837                 if (newRegion == NULL) {
1838                         rc = -ENOMEM;
1839                         goto out;
1840                 }
1841                 memcpy(newRegion, memMap->region, oldSize);
1842                 memset(&((uint8_t *) newRegion)[oldSize], 0, newSize - oldSize);
1843
1844                 kfree(memMap->region);
1845
1846                 memMap->numRegionsAllocated = newAlloc;
1847                 memMap->region = newRegion;
1848         }
1849
1850         region = &memMap->region[memMap->numRegionsUsed];
1851         memMap->numRegionsUsed++;
1852
1853         offset = addr & ~PAGE_MASK;
1854
1855         region->memType = dma_mem_type(mem);
1856         region->virtAddr = mem;
1857         region->numBytes = numBytes;
1858         region->numSegmentsUsed = 0;
1859         region->numLockedPages = 0;
1860         region->lockedPages = NULL;
1861
1862         switch (region->memType) {
1863         case DMA_MEM_TYPE_VMALLOC:
1864                 {
1865                         atomic_inc(&gDmaStatMemTypeVmalloc);
1866
1867                         /* printk(KERN_ERR "%s: vmalloc'd pages are not supported\n", __func__); */
1868
1869                         /* vmalloc'd pages are not physically contiguous */
1870
1871                         rc = -EINVAL;
1872                         break;
1873                 }
1874
1875         case DMA_MEM_TYPE_KMALLOC:
1876                 {
1877                         atomic_inc(&gDmaStatMemTypeKmalloc);
1878
1879                         /* kmalloc'd pages are physically contiguous, so they'll have exactly */
1880                         /* one segment */
1881
1882 #if ALLOW_MAP_OF_KMALLOC_MEMORY
1883                         physAddr =
1884                             dma_map_single(NULL, mem, numBytes, memMap->dir);
1885                         rc = dma_map_add_segment(memMap, region, mem, physAddr,
1886                                                  numBytes);
1887 #else
1888                         rc = -EINVAL;
1889 #endif
1890                         break;
1891                 }
1892
1893         case DMA_MEM_TYPE_DMA:
1894                 {
1895                         /* dma_alloc_xxx pages are physically contiguous */
1896
1897                         atomic_inc(&gDmaStatMemTypeCoherent);
1898
1899                         physAddr = (vmalloc_to_pfn(mem) << PAGE_SHIFT) + offset;
1900
1901                         dma_sync_single_for_cpu(NULL, physAddr, numBytes,
1902                                                 memMap->dir);
1903                         rc = dma_map_add_segment(memMap, region, mem, physAddr,
1904                                                  numBytes);
1905                         break;
1906                 }
1907
1908         case DMA_MEM_TYPE_USER:
1909                 {
1910                         size_t firstPageOffset;
1911                         size_t firstPageSize;
1912                         struct page **pages;
1913                         struct task_struct *userTask;
1914
1915                         atomic_inc(&gDmaStatMemTypeUser);
1916
1917 #if 1
1918                         /* If the pages are user pages, then the dma_mem_map_set_user_task function */
1919                         /* must have been previously called. */
1920
1921                         if (memMap->userTask == NULL) {
1922                                 printk(KERN_ERR
1923                                        "%s: must call dma_mem_map_set_user_task when using user-mode memory\n",
1924                                        __func__);
1925                                 return -EINVAL;
1926                         }
1927
1928                         /* User pages need to be locked. */
1929
1930                         firstPageOffset =
1931                             (unsigned long)region->virtAddr & (PAGE_SIZE - 1);
1932                         firstPageSize = PAGE_SIZE - firstPageOffset;
1933
1934                         region->numLockedPages = (firstPageOffset
1935                                                   + region->numBytes +
1936                                                   PAGE_SIZE - 1) / PAGE_SIZE;
1937                         pages =
1938                             kmalloc(region->numLockedPages *
1939                                     sizeof(struct page *), GFP_KERNEL);
1940
1941                         if (pages == NULL) {
1942                                 region->numLockedPages = 0;
1943                                 return -ENOMEM;
1944                         }
1945
1946                         userTask = memMap->userTask;
1947
1948                         down_read(&userTask->mm->mmap_sem);
1949                         rc = get_user_pages(userTask,   /* task */
1950                                             userTask->mm,       /* mm */
1951                                             (unsigned long)region->virtAddr,    /* start */
1952                                             region->numLockedPages,     /* len */
1953                                             memMap->dir == DMA_FROM_DEVICE,     /* write */
1954                                             0,  /* force */
1955                                             pages,      /* pages (array of pointers to page) */
1956                                             NULL);      /* vmas */
1957                         up_read(&userTask->mm->mmap_sem);
1958
1959                         if (rc != region->numLockedPages) {
1960                                 kfree(pages);
1961                                 region->numLockedPages = 0;
1962
1963                                 if (rc >= 0) {
1964                                         rc = -EINVAL;
1965                                 }
1966                         } else {
1967                                 uint8_t *virtAddr = region->virtAddr;
1968                                 size_t bytesRemaining;
1969                                 int pageIdx;
1970
1971                                 rc = 0; /* Since get_user_pages returns +ve number */
1972
1973                                 region->lockedPages = pages;
1974
1975                                 /* We've locked the user pages. Now we need to walk them and figure */
1976                                 /* out the physical addresses. */
1977
1978                                 /* The first page may be partial */
1979
1980                                 dma_map_add_segment(memMap,
1981                                                     region,
1982                                                     virtAddr,
1983                                                     PFN_PHYS(page_to_pfn
1984                                                              (pages[0])) +
1985                                                     firstPageOffset,
1986                                                     firstPageSize);
1987
1988                                 virtAddr += firstPageSize;
1989                                 bytesRemaining =
1990                                     region->numBytes - firstPageSize;
1991
1992                                 for (pageIdx = 1;
1993                                      pageIdx < region->numLockedPages;
1994                                      pageIdx++) {
1995                                         size_t bytesThisPage =
1996                                             (bytesRemaining >
1997                                              PAGE_SIZE ? PAGE_SIZE :
1998                                              bytesRemaining);
1999
2000                                         DMA_MAP_PRINT
2001                                             ("pageIdx:%d pages[pageIdx]=%p pfn=%u phys=%u\n",
2002                                              pageIdx, pages[pageIdx],
2003                                              page_to_pfn(pages[pageIdx]),
2004                                              PFN_PHYS(page_to_pfn
2005                                                       (pages[pageIdx])));
2006
2007                                         dma_map_add_segment(memMap,
2008                                                             region,
2009                                                             virtAddr,
2010                                                             PFN_PHYS(page_to_pfn
2011                                                                      (pages
2012                                                                       [pageIdx])),
2013                                                             bytesThisPage);
2014
2015                                         virtAddr += bytesThisPage;
2016                                         bytesRemaining -= bytesThisPage;
2017                                 }
2018                         }
2019 #else
2020                         printk(KERN_ERR
2021                                "%s: User mode pages are not yet supported\n",
2022                                __func__);
2023
2024                         /* user pages are not physically contiguous */
2025
2026                         rc = -EINVAL;
2027 #endif
2028                         break;
2029                 }
2030
2031         default:
2032                 {
2033                         printk(KERN_ERR "%s: Unsupported memory type: %d\n",
2034                                __func__, region->memType);
2035
2036                         rc = -EINVAL;
2037                         break;
2038                 }
2039         }
2040
2041         if (rc != 0) {
2042                 memMap->numRegionsUsed--;
2043         }
2044
2045 out:
2046
2047         DMA_MAP_PRINT("returning %d\n", rc);
2048
2049         up(&memMap->lock);
2050
2051         return rc;
2052 }
2053
2054 EXPORT_SYMBOL(dma_map_add_segment);
2055
2056 /****************************************************************************/
2057 /**
2058 *   Maps in a memory region such that it can be used for performing a DMA.
2059 *
2060 *   @return     0 on success, error code otherwise.
2061 */
2062 /****************************************************************************/
2063
2064 int dma_map_mem(DMA_MemMap_t *memMap,   /* Stores state information about the map */
2065                 void *mem,      /* Virtual address that we want to get a map of */
2066                 size_t numBytes,        /* Number of bytes being mapped */
2067                 enum dma_data_direction dir     /* Direction that the mapping will be going */
2068     ) {
2069         int rc;
2070
2071         rc = dma_map_start(memMap, dir);
2072         if (rc == 0) {
2073                 rc = dma_map_add_region(memMap, mem, numBytes);
2074                 if (rc < 0) {
2075                         /* Since the add fails, this function will fail, and the caller won't */
2076                         /* call unmap, so we need to do it here. */
2077
2078                         dma_unmap(memMap, 0);
2079                 }
2080         }
2081
2082         return rc;
2083 }
2084
2085 EXPORT_SYMBOL(dma_map_mem);
2086
2087 /****************************************************************************/
2088 /**
2089 *   Setup a descriptor ring for a given memory map.
2090 *
2091 *   It is assumed that the descriptor ring has already been initialized, and
2092 *   this routine will only reallocate a new descriptor ring if the existing
2093 *   one is too small.
2094 *
2095 *   @return     0 on success, error code otherwise.
2096 */
2097 /****************************************************************************/
2098
2099 int dma_map_create_descriptor_ring(DMA_Device_t dev,    /* DMA device (where the ring is stored) */
2100                                    DMA_MemMap_t *memMap,        /* Memory map that will be used */
2101                                    dma_addr_t devPhysAddr       /* Physical address of device */
2102     ) {
2103         int rc;
2104         int numDescriptors;
2105         DMA_DeviceAttribute_t *devAttr;
2106         DMA_Region_t *region;
2107         DMA_Segment_t *segment;
2108         dma_addr_t srcPhysAddr;
2109         dma_addr_t dstPhysAddr;
2110         int regionIdx;
2111         int segmentIdx;
2112
2113         devAttr = &DMA_gDeviceAttribute[dev];
2114
2115         down(&memMap->lock);
2116
2117         /* Figure out how many descriptors we need */
2118
2119         numDescriptors = 0;
2120         for (regionIdx = 0; regionIdx < memMap->numRegionsUsed; regionIdx++) {
2121                 region = &memMap->region[regionIdx];
2122
2123                 for (segmentIdx = 0; segmentIdx < region->numSegmentsUsed;
2124                      segmentIdx++) {
2125                         segment = &region->segment[segmentIdx];
2126
2127                         if (memMap->dir == DMA_TO_DEVICE) {
2128                                 srcPhysAddr = segment->physAddr;
2129                                 dstPhysAddr = devPhysAddr;
2130                         } else {
2131                                 srcPhysAddr = devPhysAddr;
2132                                 dstPhysAddr = segment->physAddr;
2133                         }
2134
2135                         rc =
2136                              dma_calculate_descriptor_count(dev, srcPhysAddr,
2137                                                             dstPhysAddr,
2138                                                             segment->
2139                                                             numBytes);
2140                         if (rc < 0) {
2141                                 printk(KERN_ERR
2142                                        "%s: dma_calculate_descriptor_count failed: %d\n",
2143                                        __func__, rc);
2144                                 goto out;
2145                         }
2146                         numDescriptors += rc;
2147                 }
2148         }
2149
2150         /* Adjust the size of the ring, if it isn't big enough */
2151
2152         if (numDescriptors > devAttr->ring.descriptorsAllocated) {
2153                 dma_free_descriptor_ring(&devAttr->ring);
2154                 rc =
2155                      dma_alloc_descriptor_ring(&devAttr->ring,
2156                                                numDescriptors);
2157                 if (rc < 0) {
2158                         printk(KERN_ERR
2159                                "%s: dma_alloc_descriptor_ring failed: %d\n",
2160                                __func__, rc);
2161                         goto out;
2162                 }
2163         } else {
2164                 rc =
2165                      dma_init_descriptor_ring(&devAttr->ring,
2166                                               numDescriptors);
2167                 if (rc < 0) {
2168                         printk(KERN_ERR
2169                                "%s: dma_init_descriptor_ring failed: %d\n",
2170                                __func__, rc);
2171                         goto out;
2172                 }
2173         }
2174
2175         /* Populate the descriptors */
2176
2177         for (regionIdx = 0; regionIdx < memMap->numRegionsUsed; regionIdx++) {
2178                 region = &memMap->region[regionIdx];
2179
2180                 for (segmentIdx = 0; segmentIdx < region->numSegmentsUsed;
2181                      segmentIdx++) {
2182                         segment = &region->segment[segmentIdx];
2183
2184                         if (memMap->dir == DMA_TO_DEVICE) {
2185                                 srcPhysAddr = segment->physAddr;
2186                                 dstPhysAddr = devPhysAddr;
2187                         } else {
2188                                 srcPhysAddr = devPhysAddr;
2189                                 dstPhysAddr = segment->physAddr;
2190                         }
2191
2192                         rc =
2193                              dma_add_descriptors(&devAttr->ring, dev,
2194                                                  srcPhysAddr, dstPhysAddr,
2195                                                  segment->numBytes);
2196                         if (rc < 0) {
2197                                 printk(KERN_ERR
2198                                        "%s: dma_add_descriptors failed: %d\n",
2199                                        __func__, rc);
2200                                 goto out;
2201                         }
2202                 }
2203         }
2204
2205         rc = 0;
2206
2207 out:
2208
2209         up(&memMap->lock);
2210         return rc;
2211 }
2212
2213 EXPORT_SYMBOL(dma_map_create_descriptor_ring);
2214
2215 /****************************************************************************/
2216 /**
2217 *   Maps in a memory region such that it can be used for performing a DMA.
2218 *
2219 *   @return
2220 */
2221 /****************************************************************************/
2222
2223 int dma_unmap(DMA_MemMap_t *memMap,     /* Stores state information about the map */
2224               int dirtied       /* non-zero if any of the pages were modified */
2225     ) {
2226
2227         int rc = 0;
2228         int regionIdx;
2229         int segmentIdx;
2230         DMA_Region_t *region;
2231         DMA_Segment_t *segment;
2232
2233         down(&memMap->lock);
2234
2235         for (regionIdx = 0; regionIdx < memMap->numRegionsUsed; regionIdx++) {
2236                 region = &memMap->region[regionIdx];
2237
2238                 for (segmentIdx = 0; segmentIdx < region->numSegmentsUsed;
2239                      segmentIdx++) {
2240                         segment = &region->segment[segmentIdx];
2241
2242                         switch (region->memType) {
2243                         case DMA_MEM_TYPE_VMALLOC:
2244                                 {
2245                                         printk(KERN_ERR
2246                                                "%s: vmalloc'd pages are not yet supported\n",
2247                                                __func__);
2248                                         rc = -EINVAL;
2249                                         goto out;
2250                                 }
2251
2252                         case DMA_MEM_TYPE_KMALLOC:
2253                                 {
2254 #if ALLOW_MAP_OF_KMALLOC_MEMORY
2255                                         dma_unmap_single(NULL,
2256                                                          segment->physAddr,
2257                                                          segment->numBytes,
2258                                                          memMap->dir);
2259 #endif
2260                                         break;
2261                                 }
2262
2263                         case DMA_MEM_TYPE_DMA:
2264                                 {
2265                                         dma_sync_single_for_cpu(NULL,
2266                                                                 segment->
2267                                                                 physAddr,
2268                                                                 segment->
2269                                                                 numBytes,
2270                                                                 memMap->dir);
2271                                         break;
2272                                 }
2273
2274                         case DMA_MEM_TYPE_USER:
2275                                 {
2276                                         /* Nothing to do here. */
2277
2278                                         break;
2279                                 }
2280
2281                         default:
2282                                 {
2283                                         printk(KERN_ERR
2284                                                "%s: Unsupported memory type: %d\n",
2285                                                __func__, region->memType);
2286                                         rc = -EINVAL;
2287                                         goto out;
2288                                 }
2289                         }
2290
2291                         segment->virtAddr = NULL;
2292                         segment->physAddr = 0;
2293                         segment->numBytes = 0;
2294                 }
2295
2296                 if (region->numLockedPages > 0) {
2297                         int pageIdx;
2298
2299                         /* Some user pages were locked. We need to go and unlock them now. */
2300
2301                         for (pageIdx = 0; pageIdx < region->numLockedPages;
2302                              pageIdx++) {
2303                                 struct page *page =
2304                                     region->lockedPages[pageIdx];
2305
2306                                 if (memMap->dir == DMA_FROM_DEVICE) {
2307                                         SetPageDirty(page);
2308                                 }
2309                                 page_cache_release(page);
2310                         }
2311                         kfree(region->lockedPages);
2312                         region->numLockedPages = 0;
2313                         region->lockedPages = NULL;
2314                 }
2315
2316                 region->memType = DMA_MEM_TYPE_NONE;
2317                 region->virtAddr = NULL;
2318                 region->numBytes = 0;
2319                 region->numSegmentsUsed = 0;
2320         }
2321         memMap->userTask = NULL;
2322         memMap->numRegionsUsed = 0;
2323         memMap->inUse = 0;
2324
2325 out:
2326         up(&memMap->lock);
2327
2328         return rc;
2329 }
2330
2331 EXPORT_SYMBOL(dma_unmap);