arch/arm/mach-tegra/dma.c

   1 /*
   2  * arch/arm/mach-tegra/dma.c
   3  *
   4  * System DMA driver for NVIDIA Tegra SoCs
   5  *
   6  * Copyright (c) 2008-2009, NVIDIA Corporation.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful, but WITHOUT
  14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  16  * more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, write to the Free Software Foundation, Inc.,
  20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  21  */
  22
  23 #include <linux/io.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/module.h>
  26 #include <linux/spinlock.h>
  27 #include <linux/err.h>
  28 #include <linux/irq.h>
  29 #include <linux/delay.h>
  30 #include <linux/clk.h>
  31 #include <mach/dma.h>
  32 #include <mach/irqs.h>
  33 #include <mach/iomap.h>
  34 #include <mach/suspend.h>
  35
  36 #include "apbio.h"
  37
  38 #define APB_DMA_GEN                             0x000
  39 #define GEN_ENABLE                              (1<<31)
  40
  41 #define APB_DMA_CNTRL                           0x010
  42
  43 #define APB_DMA_IRQ_MASK                        0x01c
  44
  45 #define APB_DMA_IRQ_MASK_SET                    0x020
  46
  47 #define APB_DMA_CHAN_CSR                        0x000
  48 #define CSR_ENB                                 (1<<31)
  49 #define CSR_IE_EOC                              (1<<30)
  50 #define CSR_HOLD                                (1<<29)
  51 #define CSR_DIR                                 (1<<28)
  52 #define CSR_ONCE                                (1<<27)
  53 #define CSR_FLOW                                (1<<21)
  54 #define CSR_REQ_SEL_SHIFT                       16
  55 #define CSR_WCOUNT_SHIFT                        2
  56 #define CSR_WCOUNT_MASK                         0xFFFC
  57
  58 #define APB_DMA_CHAN_STA                                0x004
  59 #define STA_BUSY                                (1<<31)
  60 #define STA_ISE_EOC                             (1<<30)
  61 #define STA_HALT                                (1<<29)
  62 #define STA_PING_PONG                           (1<<28)
  63 #define STA_COUNT_SHIFT                         2
  64 #define STA_COUNT_MASK                          0xFFFC
  65
  66 #define APB_DMA_CHAN_AHB_PTR                            0x010
  67
  68 #define APB_DMA_CHAN_AHB_SEQ                            0x014
  69 #define AHB_SEQ_INTR_ENB                        (1<<31)
  70 #define AHB_SEQ_BUS_WIDTH_SHIFT                 28
  71 #define AHB_SEQ_BUS_WIDTH_MASK                  (0x7<<AHB_SEQ_BUS_WIDTH_SHIFT)
  72 #define AHB_SEQ_BUS_WIDTH_8                     (0<<AHB_SEQ_BUS_WIDTH_SHIFT)
  73 #define AHB_SEQ_BUS_WIDTH_16                    (1<<AHB_SEQ_BUS_WIDTH_SHIFT)
  74 #define AHB_SEQ_BUS_WIDTH_32                    (2<<AHB_SEQ_BUS_WIDTH_SHIFT)
  75 #define AHB_SEQ_BUS_WIDTH_64                    (3<<AHB_SEQ_BUS_WIDTH_SHIFT)
  76 #define AHB_SEQ_BUS_WIDTH_128                   (4<<AHB_SEQ_BUS_WIDTH_SHIFT)
  77 #define AHB_SEQ_DATA_SWAP                       (1<<27)
  78 #define AHB_SEQ_BURST_MASK                      (0x7<<24)
  79 #define AHB_SEQ_BURST_1                         (4<<24)
  80 #define AHB_SEQ_BURST_4                         (5<<24)
  81 #define AHB_SEQ_BURST_8                         (6<<24)
  82 #define AHB_SEQ_DBL_BUF                         (1<<19)
  83 #define AHB_SEQ_WRAP_SHIFT                      16
  84 #define AHB_SEQ_WRAP_MASK                       (0x7<<AHB_SEQ_WRAP_SHIFT)
  85
  86 #define APB_DMA_CHAN_APB_PTR                            0x018
  87
  88 #define APB_DMA_CHAN_APB_SEQ                            0x01c
  89 #define APB_SEQ_BUS_WIDTH_SHIFT                 28
  90 #define APB_SEQ_BUS_WIDTH_MASK                  (0x7<<APB_SEQ_BUS_WIDTH_SHIFT)
  91 #define APB_SEQ_BUS_WIDTH_8                     (0<<APB_SEQ_BUS_WIDTH_SHIFT)
  92 #define APB_SEQ_BUS_WIDTH_16                    (1<<APB_SEQ_BUS_WIDTH_SHIFT)
  93 #define APB_SEQ_BUS_WIDTH_32                    (2<<APB_SEQ_BUS_WIDTH_SHIFT)
  94 #define APB_SEQ_BUS_WIDTH_64                    (3<<APB_SEQ_BUS_WIDTH_SHIFT)
  95 #define APB_SEQ_BUS_WIDTH_128                   (4<<APB_SEQ_BUS_WIDTH_SHIFT)
  96 #define APB_SEQ_DATA_SWAP                       (1<<27)
  97 #define APB_SEQ_WRAP_SHIFT                      16
  98 #define APB_SEQ_WRAP_MASK                       (0x7<<APB_SEQ_WRAP_SHIFT)
  99
 100 #define TEGRA_SYSTEM_DMA_CH_NR                  16
 101 #define TEGRA_SYSTEM_DMA_AVP_CH_NUM             4
 102 #define TEGRA_SYSTEM_DMA_CH_MIN                 0
 103 #define TEGRA_SYSTEM_DMA_CH_MAX \
 104         (TEGRA_SYSTEM_DMA_CH_NR - TEGRA_SYSTEM_DMA_AVP_CH_NUM - 1)
 105
 106 #define NV_DMA_MAX_TRASFER_SIZE 0x10000
 107
 108 static const unsigned int ahb_addr_wrap_table[8] = {
 109         0, 32, 64, 128, 256, 512, 1024, 2048
 110 };
 111
 112 static const unsigned int apb_addr_wrap_table[8] = {
 113         0, 1, 2, 4, 8, 16, 32, 64
 114 };
 115
 116 static const unsigned int bus_width_table[5] = {
 117         8, 16, 32, 64, 128
 118 };
 119
 120 #define TEGRA_DMA_NAME_SIZE 16
 121 struct tegra_dma_channel {
 122         struct list_head        list;
 123         int                     id;
 124         spinlock_t              lock;
 125         char                    name[TEGRA_DMA_NAME_SIZE];
 126         void  __iomem           *addr;
 127         int                     mode;
 128         int                     irq;
 129         int                     req_transfer_count;
 130 };
 131
 132 #define  NV_DMA_MAX_CHANNELS  32
 133
 134 static bool tegra_dma_initialized;
 135 static DEFINE_MUTEX(tegra_dma_lock);
 136 static DEFINE_SPINLOCK(enable_lock);
 137
 138 static DECLARE_BITMAP(channel_usage, NV_DMA_MAX_CHANNELS);
 139 static struct tegra_dma_channel dma_channels[NV_DMA_MAX_CHANNELS];
 140
 141 static void tegra_dma_update_hw(struct tegra_dma_channel *ch,
 142         struct tegra_dma_req *req);
 143 static void tegra_dma_update_hw_partial(struct tegra_dma_channel *ch,
 144         struct tegra_dma_req *req);
 145 static void tegra_dma_stop(struct tegra_dma_channel *ch);
 146
 147 void tegra_dma_flush(struct tegra_dma_channel *ch)
 148 {
 149 }
 150 EXPORT_SYMBOL(tegra_dma_flush);
 151
 152 void tegra_dma_dequeue(struct tegra_dma_channel *ch)
 153 {
 154         struct tegra_dma_req *req;
 155
 156         if (tegra_dma_is_empty(ch))
 157                 return;
 158
 159         req = list_entry(ch->list.next, typeof(*req), node);
 160
 161         tegra_dma_dequeue_req(ch, req);
 162         return;
 163 }
 164
 165 static void tegra_dma_stop(struct tegra_dma_channel *ch)
 166 {
 167         u32 csr;
 168         u32 status;
 169
 170         csr = readl(ch->addr + APB_DMA_CHAN_CSR);
 171         csr &= ~CSR_IE_EOC;
 172         writel(csr, ch->addr + APB_DMA_CHAN_CSR);
 173
 174         csr &= ~CSR_ENB;
 175         writel(csr, ch->addr + APB_DMA_CHAN_CSR);
 176
 177         status = readl(ch->addr + APB_DMA_CHAN_STA);
 178         if (status & STA_ISE_EOC)
 179                 writel(status, ch->addr + APB_DMA_CHAN_STA);
 180 }
 181
 182 static int tegra_dma_cancel(struct tegra_dma_channel *ch)
 183 {
 184         unsigned long irq_flags;
 185
 186         spin_lock_irqsave(&ch->lock, irq_flags);
 187         while (!list_empty(&ch->list))
 188                 list_del(ch->list.next);
 189
 190         tegra_dma_stop(ch);
 191
 192         spin_unlock_irqrestore(&ch->lock, irq_flags);
 193         return 0;
 194 }
 195
 196 static unsigned int get_channel_status(struct tegra_dma_channel *ch,
 197                         struct tegra_dma_req *req, bool is_stop_dma)
 198 {
 199         void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
 200         unsigned int status;
 201
 202         if (is_stop_dma) {
 203                 /*
 204                  * STOP the DMA and get the transfer count.
 205                  * Getting the transfer count is tricky.
 206                  *  - Globally disable DMA on all channels
 207                  *  - Read the channel's status register to know the number
 208                  *    of pending bytes to be transfered.
 209                  *  - Stop the dma channel
 210                  *  - Globally re-enable DMA to resume other transfers
 211                  */
 212                 spin_lock(&enable_lock);
 213                 writel(0, addr + APB_DMA_GEN);
 214                 udelay(20);
 215                 status = readl(ch->addr + APB_DMA_CHAN_STA);
 216                 tegra_dma_stop(ch);
 217                 writel(GEN_ENABLE, addr + APB_DMA_GEN);
 218                 spin_unlock(&enable_lock);
 219                 if (status & STA_ISE_EOC) {
 220                         pr_err("Got Dma Int here clearing");
 221                         writel(status, ch->addr + APB_DMA_CHAN_STA);
 222                 }
 223                 req->status = TEGRA_DMA_REQ_ERROR_ABORTED;
 224         } else {
 225                 status = readl(ch->addr + APB_DMA_CHAN_STA);
 226         }
 227         return status;
 228 }
 229
 230 /* should be called with the channel lock held */
 231 static unsigned int dma_active_count(struct tegra_dma_channel *ch,
 232         struct tegra_dma_req *req, unsigned int status)
 233 {
 234         unsigned int to_transfer;
 235         unsigned int req_transfer_count;
 236         unsigned int bytes_transferred;
 237
 238         to_transfer = ((status & STA_COUNT_MASK) >> STA_COUNT_SHIFT) + 1;
 239         req_transfer_count = ch->req_transfer_count + 1;
 240         bytes_transferred = req_transfer_count;
 241         if (status & STA_BUSY)
 242                 bytes_transferred -= to_transfer;
 243         /*
 244          * In continuous transfer mode, DMA only tracks the count of the
 245          * half DMA buffer. So, if the DMA already finished half the DMA
 246          * then add the half buffer to the completed count.
 247          */
 248         if (ch->mode & TEGRA_DMA_MODE_CONTINOUS) {
 249                 if (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL)
 250                         bytes_transferred += req_transfer_count;
 251                 if (status & STA_ISE_EOC)
 252                         bytes_transferred += req_transfer_count;
 253         }
 254         bytes_transferred *= 4;
 255         return bytes_transferred;
 256 }
 257
 258 int tegra_dma_dequeue_req(struct tegra_dma_channel *ch,
 259         struct tegra_dma_req *_req)
 260 {
 261         unsigned int status;
 262         struct tegra_dma_req *req = NULL;
 263         int found = 0;
 264         unsigned long irq_flags;
 265         int stop = 0;
 266
 267         spin_lock_irqsave(&ch->lock, irq_flags);
 268
 269         if (list_entry(ch->list.next, struct tegra_dma_req, node) == _req)
 270                 stop = 1;
 271
 272         list_for_each_entry(req, &ch->list, node) {
 273                 if (req == _req) {
 274                         list_del(&req->node);
 275                         found = 1;
 276                         break;
 277                 }
 278         }
 279         if (!found) {
 280                 spin_unlock_irqrestore(&ch->lock, irq_flags);
 281                 return 0;
 282         }
 283
 284         if (!stop)
 285                 goto skip_stop_dma;
 286
 287         status = get_channel_status(ch, req, true);
 288         req->bytes_transferred = dma_active_count(ch, req, status);
 289
 290         if (!list_empty(&ch->list)) {
 291                 /* if the list is not empty, queue the next request */
 292                 struct tegra_dma_req *next_req;
 293                 next_req = list_entry(ch->list.next,
 294                         typeof(*next_req), node);
 295                 tegra_dma_update_hw(ch, next_req);
 296         }
 297
 298 skip_stop_dma:
 299         req->status = -TEGRA_DMA_REQ_ERROR_ABORTED;
 300
 301         spin_unlock_irqrestore(&ch->lock, irq_flags);
 302
 303         /* Callback should be called without any lock */
 304         req->complete(req);
 305         return 0;
 306 }
 307 EXPORT_SYMBOL(tegra_dma_dequeue_req);
 308
 309 bool tegra_dma_is_empty(struct tegra_dma_channel *ch)
 310 {
 311         unsigned long irq_flags;
 312         bool is_empty;
 313
 314         spin_lock_irqsave(&ch->lock, irq_flags);
 315         if (list_empty(&ch->list))
 316                 is_empty = true;
 317         else
 318                 is_empty = false;
 319         spin_unlock_irqrestore(&ch->lock, irq_flags);
 320         return is_empty;
 321 }
 322 EXPORT_SYMBOL(tegra_dma_is_empty);
 323
 324 bool tegra_dma_is_req_inflight(struct tegra_dma_channel *ch,
 325         struct tegra_dma_req *_req)
 326 {
 327         unsigned long irq_flags;
 328         struct tegra_dma_req *req;
 329
 330         spin_lock_irqsave(&ch->lock, irq_flags);
 331         list_for_each_entry(req, &ch->list, node) {
 332                 if (req == _req) {
 333                         spin_unlock_irqrestore(&ch->lock, irq_flags);
 334                         return true;
 335                 }
 336         }
 337         spin_unlock_irqrestore(&ch->lock, irq_flags);
 338         return false;
 339 }
 340 EXPORT_SYMBOL(tegra_dma_is_req_inflight);
 341
 342 int tegra_dma_enqueue_req(struct tegra_dma_channel *ch,
 343         struct tegra_dma_req *req)
 344 {
 345         unsigned long irq_flags;
 346         struct tegra_dma_req *_req;
 347         int start_dma = 0;
 348
 349         if (req->size > NV_DMA_MAX_TRASFER_SIZE ||
 350                 req->source_addr & 0x3 || req->dest_addr & 0x3) {
 351                 pr_err("Invalid DMA request for channel %d\n", ch->id);
 352                 return -EINVAL;
 353         }
 354
 355         spin_lock_irqsave(&ch->lock, irq_flags);
 356
 357         list_for_each_entry(_req, &ch->list, node) {
 358                 if (req == _req) {
 359                     spin_unlock_irqrestore(&ch->lock, irq_flags);
 360                     return -EEXIST;
 361                 }
 362         }
 363
 364         req->bytes_transferred = 0;
 365         req->status = 0;
 366         req->buffer_status = 0;
 367         if (list_empty(&ch->list))
 368                 start_dma = 1;
 369
 370         list_add_tail(&req->node, &ch->list);
 371
 372         if (start_dma)
 373                 tegra_dma_update_hw(ch, req);
 374
 375         spin_unlock_irqrestore(&ch->lock, irq_flags);
 376
 377         return 0;
 378 }
 379 EXPORT_SYMBOL(tegra_dma_enqueue_req);
 380
 381 struct tegra_dma_channel *tegra_dma_allocate_channel(int mode)
 382 {
 383         int channel;
 384         struct tegra_dma_channel *ch = NULL;
 385
 386         if (!tegra_dma_initialized)
 387                 return NULL;
 388
 389         mutex_lock(&tegra_dma_lock);
 390
 391         /* first channel is the shared channel */
 392         if (mode & TEGRA_DMA_SHARED) {
 393                 channel = TEGRA_SYSTEM_DMA_CH_MIN;
 394         } else {
 395                 channel = find_first_zero_bit(channel_usage,
 396                         ARRAY_SIZE(dma_channels));
 397                 if (channel >= ARRAY_SIZE(dma_channels))
 398                         goto out;
 399         }
 400         __set_bit(channel, channel_usage);
 401         ch = &dma_channels[channel];
 402         ch->mode = mode;
 403
 404 out:
 405         mutex_unlock(&tegra_dma_lock);
 406         return ch;
 407 }
 408 EXPORT_SYMBOL(tegra_dma_allocate_channel);
 409
 410 void tegra_dma_free_channel(struct tegra_dma_channel *ch)
 411 {
 412         if (ch->mode & TEGRA_DMA_SHARED)
 413                 return;
 414         tegra_dma_cancel(ch);
 415         mutex_lock(&tegra_dma_lock);
 416         __clear_bit(ch->id, channel_usage);
 417         mutex_unlock(&tegra_dma_lock);
 418 }
 419 EXPORT_SYMBOL(tegra_dma_free_channel);
 420
 421 static void tegra_dma_update_hw_partial(struct tegra_dma_channel *ch,
 422         struct tegra_dma_req *req)
 423 {
 424         u32 apb_ptr;
 425         u32 ahb_ptr;
 426
 427         if (req->to_memory) {
 428                 apb_ptr = req->source_addr;
 429                 ahb_ptr = req->dest_addr;
 430         } else {
 431                 apb_ptr = req->dest_addr;
 432                 ahb_ptr = req->source_addr;
 433         }
 434         writel(apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR);
 435         writel(ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR);
 436
 437         req->status = TEGRA_DMA_REQ_INFLIGHT;
 438         return;
 439 }
 440
 441 static void tegra_dma_update_hw(struct tegra_dma_channel *ch,
 442         struct tegra_dma_req *req)
 443 {
 444         int ahb_addr_wrap;
 445         int apb_addr_wrap;
 446         int ahb_bus_width;
 447         int apb_bus_width;
 448         int index;
 449
 450         u32 ahb_seq;
 451         u32 apb_seq;
 452         u32 ahb_ptr;
 453         u32 apb_ptr;
 454         u32 csr;
 455
 456         csr = CSR_IE_EOC | CSR_FLOW;
 457         ahb_seq = AHB_SEQ_INTR_ENB | AHB_SEQ_BURST_1;
 458         apb_seq = 0;
 459
 460         csr |= req->req_sel << CSR_REQ_SEL_SHIFT;
 461
 462         /* One shot mode is always single buffered,
 463          * continuous mode is always double buffered
 464          * */
 465         if (ch->mode & TEGRA_DMA_MODE_ONESHOT) {
 466                 csr |= CSR_ONCE;
 467                 ch->req_transfer_count = (req->size >> 2) - 1;
 468         } else {
 469                 ahb_seq |= AHB_SEQ_DBL_BUF;
 470
 471                 /* In double buffered mode, we set the size to half the
 472                  * requested size and interrupt when half the buffer
 473                  * is full */
 474                 ch->req_transfer_count = (req->size >> 3) - 1;
 475         }
 476
 477         csr |= ch->req_transfer_count << CSR_WCOUNT_SHIFT;
 478
 479         if (req->to_memory) {
 480                 apb_ptr = req->source_addr;
 481                 ahb_ptr = req->dest_addr;
 482
 483                 apb_addr_wrap = req->source_wrap;
 484                 ahb_addr_wrap = req->dest_wrap;
 485                 apb_bus_width = req->source_bus_width;
 486                 ahb_bus_width = req->dest_bus_width;
 487
 488         } else {
 489                 csr |= CSR_DIR;
 490                 apb_ptr = req->dest_addr;
 491                 ahb_ptr = req->source_addr;
 492
 493                 apb_addr_wrap = req->dest_wrap;
 494                 ahb_addr_wrap = req->source_wrap;
 495                 apb_bus_width = req->dest_bus_width;
 496                 ahb_bus_width = req->source_bus_width;
 497         }
 498
 499         apb_addr_wrap >>= 2;
 500         ahb_addr_wrap >>= 2;
 501
 502         /* set address wrap for APB size */
 503         index = 0;
 504         do  {
 505                 if (apb_addr_wrap_table[index] == apb_addr_wrap)
 506                         break;
 507                 index++;
 508         } while (index < ARRAY_SIZE(apb_addr_wrap_table));
 509         BUG_ON(index == ARRAY_SIZE(apb_addr_wrap_table));
 510         apb_seq |= index << APB_SEQ_WRAP_SHIFT;
 511
 512         /* set address wrap for AHB size */
 513         index = 0;
 514         do  {
 515                 if (ahb_addr_wrap_table[index] == ahb_addr_wrap)
 516                         break;
 517                 index++;
 518         } while (index < ARRAY_SIZE(ahb_addr_wrap_table));
 519         BUG_ON(index == ARRAY_SIZE(ahb_addr_wrap_table));
 520         ahb_seq |= index << AHB_SEQ_WRAP_SHIFT;
 521
 522         for (index = 0; index < ARRAY_SIZE(bus_width_table); index++) {
 523                 if (bus_width_table[index] == ahb_bus_width)
 524                         break;
 525         }
 526         BUG_ON(index == ARRAY_SIZE(bus_width_table));
 527         ahb_seq |= index << AHB_SEQ_BUS_WIDTH_SHIFT;
 528
 529         for (index = 0; index < ARRAY_SIZE(bus_width_table); index++) {
 530                 if (bus_width_table[index] == apb_bus_width)
 531                         break;
 532         }
 533         BUG_ON(index == ARRAY_SIZE(bus_width_table));
 534         apb_seq |= index << APB_SEQ_BUS_WIDTH_SHIFT;
 535
 536         writel(csr, ch->addr + APB_DMA_CHAN_CSR);
 537         writel(apb_seq, ch->addr + APB_DMA_CHAN_APB_SEQ);
 538         writel(apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR);
 539         writel(ahb_seq, ch->addr + APB_DMA_CHAN_AHB_SEQ);
 540         writel(ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR);
 541
 542         csr |= CSR_ENB;
 543         writel(csr, ch->addr + APB_DMA_CHAN_CSR);
 544
 545         req->status = TEGRA_DMA_REQ_INFLIGHT;
 546 }
 547
 548 static void handle_oneshot_dma(struct tegra_dma_channel *ch)
 549 {
 550         struct tegra_dma_req *req;
 551         unsigned long irq_flags;
 552
 553         spin_lock_irqsave(&ch->lock, irq_flags);
 554         if (list_empty(&ch->list)) {
 555                 spin_unlock_irqrestore(&ch->lock, irq_flags);
 556                 return;
 557         }
 558
 559         req = list_entry(ch->list.next, typeof(*req), node);
 560         if (req) {
 561                 int bytes_transferred;
 562
 563                 bytes_transferred = ch->req_transfer_count;
 564                 bytes_transferred += 1;
 565                 bytes_transferred <<= 2;
 566
 567                 list_del(&req->node);
 568                 req->bytes_transferred = bytes_transferred;
 569                 req->status = TEGRA_DMA_REQ_SUCCESS;
 570
 571                 spin_unlock_irqrestore(&ch->lock, irq_flags);
 572                 /* Callback should be called without any lock */
 573                 pr_debug("%s: transferred %d bytes\n", __func__,
 574                         req->bytes_transferred);
 575                 req->complete(req);
 576                 spin_lock_irqsave(&ch->lock, irq_flags);
 577         }
 578
 579         if (!list_empty(&ch->list)) {
 580                 req = list_entry(ch->list.next, typeof(*req), node);
 581                 /* the complete function we just called may have enqueued
 582                    another req, in which case dma has already started */
 583                 if (req->status != TEGRA_DMA_REQ_INFLIGHT)
 584                         tegra_dma_update_hw(ch, req);
 585         }
 586         spin_unlock_irqrestore(&ch->lock, irq_flags);
 587 }
 588
 589 static void handle_continuous_dma(struct tegra_dma_channel *ch)
 590 {
 591         struct tegra_dma_req *req;
 592         unsigned long irq_flags;
 593
 594         spin_lock_irqsave(&ch->lock, irq_flags);
 595         if (list_empty(&ch->list)) {
 596                 spin_unlock_irqrestore(&ch->lock, irq_flags);
 597                 return;
 598         }
 599
 600         req = list_entry(ch->list.next, typeof(*req), node);
 601         if (req) {
 602                 if (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_EMPTY) {
 603                         bool is_dma_ping_complete;
 604                         is_dma_ping_complete = (readl(ch->addr + APB_DMA_CHAN_STA)
 605                                                 & STA_PING_PONG) ? true : false;
 606                         if (req->to_memory)
 607                                 is_dma_ping_complete = !is_dma_ping_complete;
 608                         /* Out of sync - Release current buffer */
 609                         if (!is_dma_ping_complete) {
 610                                 int bytes_transferred;
 611
 612                                 bytes_transferred = ch->req_transfer_count;
 613                                 bytes_transferred += 1;
 614                                 bytes_transferred <<= 3;
 615                                 req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL;
 616                                 req->bytes_transferred = bytes_transferred;
 617                                 req->status = TEGRA_DMA_REQ_SUCCESS;
 618                                 tegra_dma_stop(ch);
 619
 620                                 if (!list_is_last(&req->node, &ch->list)) {
 621                                         struct tegra_dma_req *next_req;
 622
 623                                         next_req = list_entry(req->node.next,
 624                                                 typeof(*next_req), node);
 625                                         tegra_dma_update_hw(ch, next_req);
 626                                 }
 627
 628                                 list_del(&req->node);
 629
 630                                 /* DMA lock is NOT held when callbak is called */
 631                                 spin_unlock_irqrestore(&ch->lock, irq_flags);
 632                                 req->complete(req);
 633                                 return;
 634                         }
 635                         /* Load the next request into the hardware, if available
 636                          * */
 637                         if (!list_is_last(&req->node, &ch->list)) {
 638                                 struct tegra_dma_req *next_req;
 639
 640                                 next_req = list_entry(req->node.next,
 641                                         typeof(*next_req), node);
 642                                 tegra_dma_update_hw_partial(ch, next_req);
 643                         }
 644                         req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL;
 645                         req->status = TEGRA_DMA_REQ_SUCCESS;
 646                         /* DMA lock is NOT held when callback is called */
 647                         spin_unlock_irqrestore(&ch->lock, irq_flags);
 648                         if (likely(req->threshold))
 649                                 req->threshold(req);
 650                         return;
 651
 652                 } else if (req->buffer_status ==
 653                         TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL) {
 654                         /* Callback when the buffer is completely full (i.e on
 655                          * the second  interrupt */
 656                         int bytes_transferred;
 657
 658                         bytes_transferred = ch->req_transfer_count;
 659                         bytes_transferred += 1;
 660                         bytes_transferred <<= 3;
 661
 662                         req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL;
 663                         req->bytes_transferred = bytes_transferred;
 664                         req->status = TEGRA_DMA_REQ_SUCCESS;
 665                         list_del(&req->node);
 666
 667                         /* DMA lock is NOT held when callbak is called */
 668                         spin_unlock_irqrestore(&ch->lock, irq_flags);
 669                         req->complete(req);
 670                         return;
 671
 672                 } else {
 673                         BUG();
 674                 }
 675         }
 676         spin_unlock_irqrestore(&ch->lock, irq_flags);
 677 }
 678
 679 static irqreturn_t dma_isr(int irq, void *data)
 680 {
 681         struct tegra_dma_channel *ch = data;
 682         unsigned long status;
 683
 684         status = readl(ch->addr + APB_DMA_CHAN_STA);
 685         if (status & STA_ISE_EOC)
 686                 writel(status, ch->addr + APB_DMA_CHAN_STA);
 687         else {
 688                 pr_warning("Got a spurious ISR for DMA channel %d\n", ch->id);
 689                 return IRQ_HANDLED;
 690         }
 691         return IRQ_WAKE_THREAD;
 692 }
 693
 694 static irqreturn_t dma_thread_fn(int irq, void *data)
 695 {
 696         struct tegra_dma_channel *ch = data;
 697
 698         if (ch->mode & TEGRA_DMA_MODE_ONESHOT)
 699                 handle_oneshot_dma(ch);
 700         else
 701                 handle_continuous_dma(ch);
 702
 703
 704         return IRQ_HANDLED;
 705 }
 706
 707 int __init tegra_dma_init(void)
 708 {
 709         int ret = 0;
 710         int i;
 711         unsigned int irq;
 712         void __iomem *addr;
 713         struct clk *c;
 714
 715         bitmap_fill(channel_usage, NV_DMA_MAX_CHANNELS);
 716
 717         c = clk_get_sys("tegra-apbdma", NULL);
 718         if (IS_ERR(c)) {
 719                 pr_err("Unable to get clock for APB DMA\n");
 720                 ret = PTR_ERR(c);
 721                 goto fail;
 722         }
 723         ret = clk_prepare_enable(c);
 724         if (ret != 0) {
 725                 pr_err("Unable to enable clock for APB DMA\n");
 726                 goto fail;
 727         }
 728
 729         addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
 730         writel(GEN_ENABLE, addr + APB_DMA_GEN);
 731         writel(0, addr + APB_DMA_CNTRL);
 732         writel(0xFFFFFFFFul >> (31 - TEGRA_SYSTEM_DMA_CH_MAX),
 733                addr + APB_DMA_IRQ_MASK_SET);
 734
 735         for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) {
 736                 struct tegra_dma_channel *ch = &dma_channels[i];
 737
 738                 ch->id = i;
 739                 snprintf(ch->name, TEGRA_DMA_NAME_SIZE, "dma_channel_%d", i);
 740
 741                 ch->addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
 742                         TEGRA_APB_DMA_CH0_SIZE * i);
 743
 744                 spin_lock_init(&ch->lock);
 745                 INIT_LIST_HEAD(&ch->list);
 746
 747                 irq = INT_APB_DMA_CH0 + i;
 748                 ret = request_threaded_irq(irq, dma_isr, dma_thread_fn, 0,
 749                         dma_channels[i].name, ch);
 750                 if (ret) {
 751                         pr_err("Failed to register IRQ %d for DMA %d\n",
 752                                 irq, i);
 753                         goto fail;
 754                 }
 755                 ch->irq = irq;
 756
 757                 __clear_bit(i, channel_usage);
 758         }
 759         /* mark the shared channel allocated */
 760         __set_bit(TEGRA_SYSTEM_DMA_CH_MIN, channel_usage);
 761
 762         tegra_dma_initialized = true;
 763
 764         return 0;
 765 fail:
 766         writel(0, addr + APB_DMA_GEN);
 767         for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) {
 768                 struct tegra_dma_channel *ch = &dma_channels[i];
 769                 if (ch->irq)
 770                         free_irq(ch->irq, ch);
 771         }
 772         return ret;
 773 }
 774 postcore_initcall(tegra_dma_init);
 775
 776 #ifdef CONFIG_PM
 777 static u32 apb_dma[5*TEGRA_SYSTEM_DMA_CH_NR + 3];
 778
 779 void tegra_dma_suspend(void)
 780 {
 781         void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
 782         u32 *ctx = apb_dma;
 783         int i;
 784
 785         *ctx++ = readl(addr + APB_DMA_GEN);
 786         *ctx++ = readl(addr + APB_DMA_CNTRL);
 787         *ctx++ = readl(addr + APB_DMA_IRQ_MASK);
 788
 789         for (i = 0; i < TEGRA_SYSTEM_DMA_CH_NR; i++) {
 790                 addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
 791                                   TEGRA_APB_DMA_CH0_SIZE * i);
 792
 793                 *ctx++ = readl(addr + APB_DMA_CHAN_CSR);
 794                 *ctx++ = readl(addr + APB_DMA_CHAN_AHB_PTR);
 795                 *ctx++ = readl(addr + APB_DMA_CHAN_AHB_SEQ);
 796                 *ctx++ = readl(addr + APB_DMA_CHAN_APB_PTR);
 797                 *ctx++ = readl(addr + APB_DMA_CHAN_APB_SEQ);
 798         }
 799 }
 800
 801 void tegra_dma_resume(void)
 802 {
 803         void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
 804         u32 *ctx = apb_dma;
 805         int i;
 806
 807         writel(*ctx++, addr + APB_DMA_GEN);
 808         writel(*ctx++, addr + APB_DMA_CNTRL);
 809         writel(*ctx++, addr + APB_DMA_IRQ_MASK);
 810
 811         for (i = 0; i < TEGRA_SYSTEM_DMA_CH_NR; i++) {
 812                 addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
 813                                   TEGRA_APB_DMA_CH0_SIZE * i);
 814
 815                 writel(*ctx++, addr + APB_DMA_CHAN_CSR);
 816                 writel(*ctx++, addr + APB_DMA_CHAN_AHB_PTR);
 817                 writel(*ctx++, addr + APB_DMA_CHAN_AHB_SEQ);
 818                 writel(*ctx++, addr + APB_DMA_CHAN_APB_PTR);
 819                 writel(*ctx++, addr + APB_DMA_CHAN_APB_SEQ);
 820         }
 821 }
 822
 823 #endif