drivers/dma/iop-adma.c

   1 /*
   2  * offload engine driver for the Intel Xscale series of i/o processors
   3  * Copyright © 2006, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  */
  19
  20 /*
  21  * This driver supports the asynchrounous DMA copy and RAID engines available
  22  * on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
  23  */
  24
  25 #include <linux/init.h>
  26 #include <linux/module.h>
  27 #include <linux/delay.h>
  28 #include <linux/dma-mapping.h>
  29 #include <linux/spinlock.h>
  30 #include <linux/interrupt.h>
  31 #include <linux/platform_device.h>
  32 #include <linux/memory.h>
  33 #include <linux/ioport.h>
  34 #include <linux/raid/pq.h>
  35 #include <linux/slab.h>
  36
  37 #include <mach/adma.h>
  38
  39 #define to_iop_adma_chan(chan) container_of(chan, struct iop_adma_chan, common)
  40 #define to_iop_adma_device(dev) \
  41         container_of(dev, struct iop_adma_device, common)
  42 #define tx_to_iop_adma_slot(tx) \
  43         container_of(tx, struct iop_adma_desc_slot, async_tx)
  44
  45 /**
  46  * iop_adma_free_slots - flags descriptor slots for reuse
  47  * @slot: Slot to free
  48  * Caller must hold &iop_chan->lock while calling this function
  49  */
  50 static void iop_adma_free_slots(struct iop_adma_desc_slot *slot)
  51 {
  52         int stride = slot->slots_per_op;
  53
  54         while (stride--) {
  55                 slot->slots_per_op = 0;
  56                 slot = list_entry(slot->slot_node.next,
  57                                 struct iop_adma_desc_slot,
  58                                 slot_node);
  59         }
  60 }
  61
  62 static void
  63 iop_desc_unmap(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc)
  64 {
  65         struct dma_async_tx_descriptor *tx = &desc->async_tx;
  66         struct iop_adma_desc_slot *unmap = desc->group_head;
  67         struct device *dev = &iop_chan->device->pdev->dev;
  68         u32 len = unmap->unmap_len;
  69         enum dma_ctrl_flags flags = tx->flags;
  70         u32 src_cnt;
  71         dma_addr_t addr;
  72         dma_addr_t dest;
  73
  74         src_cnt = unmap->unmap_src_cnt;
  75         dest = iop_desc_get_dest_addr(unmap, iop_chan);
  76         if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
  77                 enum dma_data_direction dir;
  78
  79                 if (src_cnt > 1) /* is xor? */
  80                         dir = DMA_BIDIRECTIONAL;
  81                 else
  82                         dir = DMA_FROM_DEVICE;
  83
  84                 dma_unmap_page(dev, dest, len, dir);
  85         }
  86
  87         if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
  88                 while (src_cnt--) {
  89                         addr = iop_desc_get_src_addr(unmap, iop_chan, src_cnt);
  90                         if (addr == dest)
  91                                 continue;
  92                         dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
  93                 }
  94         }
  95         desc->group_head = NULL;
  96 }
  97
  98 static void
  99 iop_desc_unmap_pq(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc)
 100 {
 101         struct dma_async_tx_descriptor *tx = &desc->async_tx;
 102         struct iop_adma_desc_slot *unmap = desc->group_head;
 103         struct device *dev = &iop_chan->device->pdev->dev;
 104         u32 len = unmap->unmap_len;
 105         enum dma_ctrl_flags flags = tx->flags;
 106         u32 src_cnt = unmap->unmap_src_cnt;
 107         dma_addr_t pdest = iop_desc_get_dest_addr(unmap, iop_chan);
 108         dma_addr_t qdest = iop_desc_get_qdest_addr(unmap, iop_chan);
 109         int i;
 110
 111         if (tx->flags & DMA_PREP_CONTINUE)
 112                 src_cnt -= 3;
 113
 114         if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP) && !desc->pq_check_result) {
 115                 dma_unmap_page(dev, pdest, len, DMA_BIDIRECTIONAL);
 116                 dma_unmap_page(dev, qdest, len, DMA_BIDIRECTIONAL);
 117         }
 118
 119         if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
 120                 dma_addr_t addr;
 121
 122                 for (i = 0; i < src_cnt; i++) {
 123                         addr = iop_desc_get_src_addr(unmap, iop_chan, i);
 124                         dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
 125                 }
 126                 if (desc->pq_check_result) {
 127                         dma_unmap_page(dev, pdest, len, DMA_TO_DEVICE);
 128                         dma_unmap_page(dev, qdest, len, DMA_TO_DEVICE);
 129                 }
 130         }
 131
 132         desc->group_head = NULL;
 133 }
 134
 135
 136 static dma_cookie_t
 137 iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
 138         struct iop_adma_chan *iop_chan, dma_cookie_t cookie)
 139 {
 140         struct dma_async_tx_descriptor *tx = &desc->async_tx;
 141
 142         BUG_ON(tx->cookie < 0);
 143         if (tx->cookie > 0) {
 144                 cookie = tx->cookie;
 145                 tx->cookie = 0;
 146
 147                 /* call the callback (must not sleep or submit new
 148                  * operations to this channel)
 149                  */
 150                 if (tx->callback)
 151                         tx->callback(tx->callback_param);
 152
 153                 /* unmap dma addresses
 154                  * (unmap_single vs unmap_page?)
 155                  */
 156                 if (desc->group_head && desc->unmap_len) {
 157                         if (iop_desc_is_pq(desc))
 158                                 iop_desc_unmap_pq(iop_chan, desc);
 159                         else
 160                                 iop_desc_unmap(iop_chan, desc);
 161                 }
 162         }
 163
 164         /* run dependent operations */
 165         dma_run_dependencies(tx);
 166
 167         return cookie;
 168 }
 169
 170 static int
 171 iop_adma_clean_slot(struct iop_adma_desc_slot *desc,
 172         struct iop_adma_chan *iop_chan)
 173 {
 174         /* the client is allowed to attach dependent operations
 175          * until 'ack' is set
 176          */
 177         if (!async_tx_test_ack(&desc->async_tx))
 178                 return 0;
 179
 180         /* leave the last descriptor in the chain
 181          * so we can append to it
 182          */
 183         if (desc->chain_node.next == &iop_chan->chain)
 184                 return 1;
 185
 186         dev_dbg(iop_chan->device->common.dev,
 187                 "\tfree slot: %d slots_per_op: %d\n",
 188                 desc->idx, desc->slots_per_op);
 189
 190         list_del(&desc->chain_node);
 191         iop_adma_free_slots(desc);
 192
 193         return 0;
 194 }
 195
 196 static void __iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
 197 {
 198         struct iop_adma_desc_slot *iter, *_iter, *grp_start = NULL;
 199         dma_cookie_t cookie = 0;
 200         u32 current_desc = iop_chan_get_current_descriptor(iop_chan);
 201         int busy = iop_chan_is_busy(iop_chan);
 202         int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
 203
 204         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
 205         /* free completed slots from the chain starting with
 206          * the oldest descriptor
 207          */
 208         list_for_each_entry_safe(iter, _iter, &iop_chan->chain,
 209                                         chain_node) {
 210                 pr_debug("\tcookie: %d slot: %d busy: %d "
 211                         "this_desc: %#x next_desc: %#x ack: %d\n",
 212                         iter->async_tx.cookie, iter->idx, busy,
 213                         iter->async_tx.phys, iop_desc_get_next_desc(iter),
 214                         async_tx_test_ack(&iter->async_tx));
 215                 prefetch(_iter);
 216                 prefetch(&_iter->async_tx);
 217
 218                 /* do not advance past the current descriptor loaded into the
 219                  * hardware channel, subsequent descriptors are either in
 220                  * process or have not been submitted
 221                  */
 222                 if (seen_current)
 223                         break;
 224
 225                 /* stop the search if we reach the current descriptor and the
 226                  * channel is busy, or if it appears that the current descriptor
 227                  * needs to be re-read (i.e. has been appended to)
 228                  */
 229                 if (iter->async_tx.phys == current_desc) {
 230                         BUG_ON(seen_current++);
 231                         if (busy || iop_desc_get_next_desc(iter))
 232                                 break;
 233                 }
 234
 235                 /* detect the start of a group transaction */
 236                 if (!slot_cnt && !slots_per_op) {
 237                         slot_cnt = iter->slot_cnt;
 238                         slots_per_op = iter->slots_per_op;
 239                         if (slot_cnt <= slots_per_op) {
 240                                 slot_cnt = 0;
 241                                 slots_per_op = 0;
 242                         }
 243                 }
 244
 245                 if (slot_cnt) {
 246                         pr_debug("\tgroup++\n");
 247                         if (!grp_start)
 248                                 grp_start = iter;
 249                         slot_cnt -= slots_per_op;
 250                 }
 251
 252                 /* all the members of a group are complete */
 253                 if (slots_per_op != 0 && slot_cnt == 0) {
 254                         struct iop_adma_desc_slot *grp_iter, *_grp_iter;
 255                         int end_of_chain = 0;
 256                         pr_debug("\tgroup end\n");
 257
 258                         /* collect the total results */
 259                         if (grp_start->xor_check_result) {
 260                                 u32 zero_sum_result = 0;
 261                                 slot_cnt = grp_start->slot_cnt;
 262                                 grp_iter = grp_start;
 263
 264                                 list_for_each_entry_from(grp_iter,
 265                                         &iop_chan->chain, chain_node) {
 266                                         zero_sum_result |=
 267                                             iop_desc_get_zero_result(grp_iter);
 268                                             pr_debug("\titer%d result: %d\n",
 269                                             grp_iter->idx, zero_sum_result);
 270                                         slot_cnt -= slots_per_op;
 271                                         if (slot_cnt == 0)
 272                                                 break;
 273                                 }
 274                                 pr_debug("\tgrp_start->xor_check_result: %p\n",
 275                                         grp_start->xor_check_result);
 276                                 *grp_start->xor_check_result = zero_sum_result;
 277                         }
 278
 279                         /* clean up the group */
 280                         slot_cnt = grp_start->slot_cnt;
 281                         grp_iter = grp_start;
 282                         list_for_each_entry_safe_from(grp_iter, _grp_iter,
 283                                 &iop_chan->chain, chain_node) {
 284                                 cookie = iop_adma_run_tx_complete_actions(
 285                                         grp_iter, iop_chan, cookie);
 286
 287                                 slot_cnt -= slots_per_op;
 288                                 end_of_chain = iop_adma_clean_slot(grp_iter,
 289                                         iop_chan);
 290
 291                                 if (slot_cnt == 0 || end_of_chain)
 292                                         break;
 293                         }
 294
 295                         /* the group should be complete at this point */
 296                         BUG_ON(slot_cnt);
 297
 298                         slots_per_op = 0;
 299                         grp_start = NULL;
 300                         if (end_of_chain)
 301                                 break;
 302                         else
 303                                 continue;
 304                 } else if (slots_per_op) /* wait for group completion */
 305                         continue;
 306
 307                 /* write back zero sum results (single descriptor case) */
 308                 if (iter->xor_check_result && iter->async_tx.cookie)
 309                         *iter->xor_check_result =
 310                                 iop_desc_get_zero_result(iter);
 311
 312                 cookie = iop_adma_run_tx_complete_actions(
 313                                         iter, iop_chan, cookie);
 314
 315                 if (iop_adma_clean_slot(iter, iop_chan))
 316                         break;
 317         }
 318
 319         if (cookie > 0) {
 320                 iop_chan->completed_cookie = cookie;
 321                 pr_debug("\tcompleted cookie %d\n", cookie);
 322         }
 323 }
 324
 325 static void
 326 iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
 327 {
 328         spin_lock_bh(&iop_chan->lock);
 329         __iop_adma_slot_cleanup(iop_chan);
 330         spin_unlock_bh(&iop_chan->lock);
 331 }
 332
 333 static void iop_adma_tasklet(unsigned long data)
 334 {
 335         struct iop_adma_chan *iop_chan = (struct iop_adma_chan *) data;
 336
 337         /* lockdep will flag depedency submissions as potentially
 338          * recursive locking, this is not the case as a dependency
 339          * submission will never recurse a channels submit routine.
 340          * There are checks in async_tx.c to prevent this.
 341          */
 342         spin_lock_nested(&iop_chan->lock, SINGLE_DEPTH_NESTING);
 343         __iop_adma_slot_cleanup(iop_chan);
 344         spin_unlock(&iop_chan->lock);
 345 }
 346
 347 static struct iop_adma_desc_slot *
 348 iop_adma_alloc_slots(struct iop_adma_chan *iop_chan, int num_slots,
 349                         int slots_per_op)
 350 {
 351         struct iop_adma_desc_slot *iter, *_iter, *alloc_start = NULL;
 352         LIST_HEAD(chain);
 353         int slots_found, retry = 0;
 354
 355         /* start search from the last allocated descrtiptor
 356          * if a contiguous allocation can not be found start searching
 357          * from the beginning of the list
 358          */
 359 retry:
 360         slots_found = 0;
 361         if (retry == 0)
 362                 iter = iop_chan->last_used;
 363         else
 364                 iter = list_entry(&iop_chan->all_slots,
 365                         struct iop_adma_desc_slot,
 366                         slot_node);
 367
 368         list_for_each_entry_safe_continue(
 369                 iter, _iter, &iop_chan->all_slots, slot_node) {
 370                 prefetch(_iter);
 371                 prefetch(&_iter->async_tx);
 372                 if (iter->slots_per_op) {
 373                         /* give up after finding the first busy slot
 374                          * on the second pass through the list
 375                          */
 376                         if (retry)
 377                                 break;
 378
 379                         slots_found = 0;
 380                         continue;
 381                 }
 382
 383                 /* start the allocation if the slot is correctly aligned */
 384                 if (!slots_found++) {
 385                         if (iop_desc_is_aligned(iter, slots_per_op))
 386                                 alloc_start = iter;
 387                         else {
 388                                 slots_found = 0;
 389                                 continue;
 390                         }
 391                 }
 392
 393                 if (slots_found == num_slots) {
 394                         struct iop_adma_desc_slot *alloc_tail = NULL;
 395                         struct iop_adma_desc_slot *last_used = NULL;
 396                         iter = alloc_start;
 397                         while (num_slots) {
 398                                 int i;
 399                                 dev_dbg(iop_chan->device->common.dev,
 400                                         "allocated slot: %d "
 401                                         "(desc %p phys: %#x) slots_per_op %d\n",
 402                                         iter->idx, iter->hw_desc,
 403                                         iter->async_tx.phys, slots_per_op);
 404
 405                                 /* pre-ack all but the last descriptor */
 406                                 if (num_slots != slots_per_op)
 407                                         async_tx_ack(&iter->async_tx);
 408
 409                                 list_add_tail(&iter->chain_node, &chain);
 410                                 alloc_tail = iter;
 411                                 iter->async_tx.cookie = 0;
 412                                 iter->slot_cnt = num_slots;
 413                                 iter->xor_check_result = NULL;
 414                                 for (i = 0; i < slots_per_op; i++) {
 415                                         iter->slots_per_op = slots_per_op - i;
 416                                         last_used = iter;
 417                                         iter = list_entry(iter->slot_node.next,
 418                                                 struct iop_adma_desc_slot,
 419                                                 slot_node);
 420                                 }
 421                                 num_slots -= slots_per_op;
 422                         }
 423                         alloc_tail->group_head = alloc_start;
 424                         alloc_tail->async_tx.cookie = -EBUSY;
 425                         list_splice(&chain, &alloc_tail->tx_list);
 426                         iop_chan->last_used = last_used;
 427                         iop_desc_clear_next_desc(alloc_start);
 428                         iop_desc_clear_next_desc(alloc_tail);
 429                         return alloc_tail;
 430                 }
 431         }
 432         if (!retry++)
 433                 goto retry;
 434
 435         /* perform direct reclaim if the allocation fails */
 436         __iop_adma_slot_cleanup(iop_chan);
 437
 438         return NULL;
 439 }
 440
 441 static dma_cookie_t
 442 iop_desc_assign_cookie(struct iop_adma_chan *iop_chan,
 443         struct iop_adma_desc_slot *desc)
 444 {
 445         dma_cookie_t cookie = iop_chan->common.cookie;
 446         cookie++;
 447         if (cookie < 0)
 448                 cookie = 1;
 449         iop_chan->common.cookie = desc->async_tx.cookie = cookie;
 450         return cookie;
 451 }
 452
 453 static void iop_adma_check_threshold(struct iop_adma_chan *iop_chan)
 454 {
 455         dev_dbg(iop_chan->device->common.dev, "pending: %d\n",
 456                 iop_chan->pending);
 457
 458         if (iop_chan->pending >= IOP_ADMA_THRESHOLD) {
 459                 iop_chan->pending = 0;
 460                 iop_chan_append(iop_chan);
 461         }
 462 }
 463
 464 static dma_cookie_t
 465 iop_adma_tx_submit(struct dma_async_tx_descriptor *tx)
 466 {
 467         struct iop_adma_desc_slot *sw_desc = tx_to_iop_adma_slot(tx);
 468         struct iop_adma_chan *iop_chan = to_iop_adma_chan(tx->chan);
 469         struct iop_adma_desc_slot *grp_start, *old_chain_tail;
 470         int slot_cnt;
 471         int slots_per_op;
 472         dma_cookie_t cookie;
 473         dma_addr_t next_dma;
 474
 475         grp_start = sw_desc->group_head;
 476         slot_cnt = grp_start->slot_cnt;
 477         slots_per_op = grp_start->slots_per_op;
 478
 479         spin_lock_bh(&iop_chan->lock);
 480         cookie = iop_desc_assign_cookie(iop_chan, sw_desc);
 481
 482         old_chain_tail = list_entry(iop_chan->chain.prev,
 483                 struct iop_adma_desc_slot, chain_node);
 484         list_splice_init(&sw_desc->tx_list,
 485                          &old_chain_tail->chain_node);
 486
 487         /* fix up the hardware chain */
 488         next_dma = grp_start->async_tx.phys;
 489         iop_desc_set_next_desc(old_chain_tail, next_dma);
 490         BUG_ON(iop_desc_get_next_desc(old_chain_tail) != next_dma); /* flush */
 491
 492         /* check for pre-chained descriptors */
 493         iop_paranoia(iop_desc_get_next_desc(sw_desc));
 494
 495         /* increment the pending count by the number of slots
 496          * memcpy operations have a 1:1 (slot:operation) relation
 497          * other operations are heavier and will pop the threshold
 498          * more often.
 499          */
 500         iop_chan->pending += slot_cnt;
 501         iop_adma_check_threshold(iop_chan);
 502         spin_unlock_bh(&iop_chan->lock);
 503
 504         dev_dbg(iop_chan->device->common.dev, "%s cookie: %d slot: %d\n",
 505                 __func__, sw_desc->async_tx.cookie, sw_desc->idx);
 506
 507         return cookie;
 508 }
 509
 510 static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan);
 511 static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan);
 512
 513 /**
 514  * iop_adma_alloc_chan_resources -  returns the number of allocated descriptors
 515  * @chan - allocate descriptor resources for this channel
 516  * @client - current client requesting the channel be ready for requests
 517  *
 518  * Note: We keep the slots for 1 operation on iop_chan->chain at all times.  To
 519  * avoid deadlock, via async_xor, num_descs_in_pool must at a minimum be
 520  * greater than 2x the number slots needed to satisfy a device->max_xor
 521  * request.
 522  * */
 523 static int iop_adma_alloc_chan_resources(struct dma_chan *chan)
 524 {
 525         char *hw_desc;
 526         int idx;
 527         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 528         struct iop_adma_desc_slot *slot = NULL;
 529         int init = iop_chan->slots_allocated ? 0 : 1;
 530         struct iop_adma_platform_data *plat_data =
 531                 iop_chan->device->pdev->dev.platform_data;
 532         int num_descs_in_pool = plat_data->pool_size/IOP_ADMA_SLOT_SIZE;
 533
 534         /* Allocate descriptor slots */
 535         do {
 536                 idx = iop_chan->slots_allocated;
 537                 if (idx == num_descs_in_pool)
 538                         break;
 539
 540                 slot = kzalloc(sizeof(*slot), GFP_KERNEL);
 541                 if (!slot) {
 542                         printk(KERN_INFO "IOP ADMA Channel only initialized"
 543                                 " %d descriptor slots", idx);
 544                         break;
 545                 }
 546                 hw_desc = (char *) iop_chan->device->dma_desc_pool_virt;
 547                 slot->hw_desc = (void *) &hw_desc[idx * IOP_ADMA_SLOT_SIZE];
 548
 549                 dma_async_tx_descriptor_init(&slot->async_tx, chan);
 550                 slot->async_tx.tx_submit = iop_adma_tx_submit;
 551                 INIT_LIST_HEAD(&slot->tx_list);
 552                 INIT_LIST_HEAD(&slot->chain_node);
 553                 INIT_LIST_HEAD(&slot->slot_node);
 554                 hw_desc = (char *) iop_chan->device->dma_desc_pool;
 555                 slot->async_tx.phys =
 556                         (dma_addr_t) &hw_desc[idx * IOP_ADMA_SLOT_SIZE];
 557                 slot->idx = idx;
 558
 559                 spin_lock_bh(&iop_chan->lock);
 560                 iop_chan->slots_allocated++;
 561                 list_add_tail(&slot->slot_node, &iop_chan->all_slots);
 562                 spin_unlock_bh(&iop_chan->lock);
 563         } while (iop_chan->slots_allocated < num_descs_in_pool);
 564
 565         if (idx && !iop_chan->last_used)
 566                 iop_chan->last_used = list_entry(iop_chan->all_slots.next,
 567                                         struct iop_adma_desc_slot,
 568                                         slot_node);
 569
 570         dev_dbg(iop_chan->device->common.dev,
 571                 "allocated %d descriptor slots last_used: %p\n",
 572                 iop_chan->slots_allocated, iop_chan->last_used);
 573
 574         /* initialize the channel and the chain with a null operation */
 575         if (init) {
 576                 if (dma_has_cap(DMA_MEMCPY,
 577                         iop_chan->device->common.cap_mask))
 578                         iop_chan_start_null_memcpy(iop_chan);
 579                 else if (dma_has_cap(DMA_XOR,
 580                         iop_chan->device->common.cap_mask))
 581                         iop_chan_start_null_xor(iop_chan);
 582                 else
 583                         BUG();
 584         }
 585
 586         return (idx > 0) ? idx : -ENOMEM;
 587 }
 588
 589 static struct dma_async_tx_descriptor *
 590 iop_adma_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
 591 {
 592         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 593         struct iop_adma_desc_slot *sw_desc, *grp_start;
 594         int slot_cnt, slots_per_op;
 595
 596         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
 597
 598         spin_lock_bh(&iop_chan->lock);
 599         slot_cnt = iop_chan_interrupt_slot_count(&slots_per_op, iop_chan);
 600         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 601         if (sw_desc) {
 602                 grp_start = sw_desc->group_head;
 603                 iop_desc_init_interrupt(grp_start, iop_chan);
 604                 grp_start->unmap_len = 0;
 605                 sw_desc->async_tx.flags = flags;
 606         }
 607         spin_unlock_bh(&iop_chan->lock);
 608
 609         return sw_desc ? &sw_desc->async_tx : NULL;
 610 }
 611
 612 static struct dma_async_tx_descriptor *
 613 iop_adma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
 614                          dma_addr_t dma_src, size_t len, unsigned long flags)
 615 {
 616         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 617         struct iop_adma_desc_slot *sw_desc, *grp_start;
 618         int slot_cnt, slots_per_op;
 619
 620         if (unlikely(!len))
 621                 return NULL;
 622         BUG_ON(unlikely(len > IOP_ADMA_MAX_BYTE_COUNT));
 623
 624         dev_dbg(iop_chan->device->common.dev, "%s len: %u\n",
 625                 __func__, len);
 626
 627         spin_lock_bh(&iop_chan->lock);
 628         slot_cnt = iop_chan_memcpy_slot_count(len, &slots_per_op);
 629         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 630         if (sw_desc) {
 631                 grp_start = sw_desc->group_head;
 632                 iop_desc_init_memcpy(grp_start, flags);
 633                 iop_desc_set_byte_count(grp_start, iop_chan, len);
 634                 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
 635                 iop_desc_set_memcpy_src_addr(grp_start, dma_src);
 636                 sw_desc->unmap_src_cnt = 1;
 637                 sw_desc->unmap_len = len;
 638                 sw_desc->async_tx.flags = flags;
 639         }
 640         spin_unlock_bh(&iop_chan->lock);
 641
 642         return sw_desc ? &sw_desc->async_tx : NULL;
 643 }
 644
 645 static struct dma_async_tx_descriptor *
 646 iop_adma_prep_dma_memset(struct dma_chan *chan, dma_addr_t dma_dest,
 647                          int value, size_t len, unsigned long flags)
 648 {
 649         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 650         struct iop_adma_desc_slot *sw_desc, *grp_start;
 651         int slot_cnt, slots_per_op;
 652
 653         if (unlikely(!len))
 654                 return NULL;
 655         BUG_ON(unlikely(len > IOP_ADMA_MAX_BYTE_COUNT));
 656
 657         dev_dbg(iop_chan->device->common.dev, "%s len: %u\n",
 658                 __func__, len);
 659
 660         spin_lock_bh(&iop_chan->lock);
 661         slot_cnt = iop_chan_memset_slot_count(len, &slots_per_op);
 662         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 663         if (sw_desc) {
 664                 grp_start = sw_desc->group_head;
 665                 iop_desc_init_memset(grp_start, flags);
 666                 iop_desc_set_byte_count(grp_start, iop_chan, len);
 667                 iop_desc_set_block_fill_val(grp_start, value);
 668                 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
 669                 sw_desc->unmap_src_cnt = 1;
 670                 sw_desc->unmap_len = len;
 671                 sw_desc->async_tx.flags = flags;
 672         }
 673         spin_unlock_bh(&iop_chan->lock);
 674
 675         return sw_desc ? &sw_desc->async_tx : NULL;
 676 }
 677
 678 static struct dma_async_tx_descriptor *
 679 iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest,
 680                       dma_addr_t *dma_src, unsigned int src_cnt, size_t len,
 681                       unsigned long flags)
 682 {
 683         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 684         struct iop_adma_desc_slot *sw_desc, *grp_start;
 685         int slot_cnt, slots_per_op;
 686
 687         if (unlikely(!len))
 688                 return NULL;
 689         BUG_ON(unlikely(len > IOP_ADMA_XOR_MAX_BYTE_COUNT));
 690
 691         dev_dbg(iop_chan->device->common.dev,
 692                 "%s src_cnt: %d len: %u flags: %lx\n",
 693                 __func__, src_cnt, len, flags);
 694
 695         spin_lock_bh(&iop_chan->lock);
 696         slot_cnt = iop_chan_xor_slot_count(len, src_cnt, &slots_per_op);
 697         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 698         if (sw_desc) {
 699                 grp_start = sw_desc->group_head;
 700                 iop_desc_init_xor(grp_start, src_cnt, flags);
 701                 iop_desc_set_byte_count(grp_start, iop_chan, len);
 702                 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
 703                 sw_desc->unmap_src_cnt = src_cnt;
 704                 sw_desc->unmap_len = len;
 705                 sw_desc->async_tx.flags = flags;
 706                 while (src_cnt--)
 707                         iop_desc_set_xor_src_addr(grp_start, src_cnt,
 708                                                   dma_src[src_cnt]);
 709         }
 710         spin_unlock_bh(&iop_chan->lock);
 711
 712         return sw_desc ? &sw_desc->async_tx : NULL;
 713 }
 714
 715 static struct dma_async_tx_descriptor *
 716 iop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src,
 717                           unsigned int src_cnt, size_t len, u32 *result,
 718                           unsigned long flags)
 719 {
 720         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 721         struct iop_adma_desc_slot *sw_desc, *grp_start;
 722         int slot_cnt, slots_per_op;
 723
 724         if (unlikely(!len))
 725                 return NULL;
 726
 727         dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
 728                 __func__, src_cnt, len);
 729
 730         spin_lock_bh(&iop_chan->lock);
 731         slot_cnt = iop_chan_zero_sum_slot_count(len, src_cnt, &slots_per_op);
 732         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 733         if (sw_desc) {
 734                 grp_start = sw_desc->group_head;
 735                 iop_desc_init_zero_sum(grp_start, src_cnt, flags);
 736                 iop_desc_set_zero_sum_byte_count(grp_start, len);
 737                 grp_start->xor_check_result = result;
 738                 pr_debug("\t%s: grp_start->xor_check_result: %p\n",
 739                         __func__, grp_start->xor_check_result);
 740                 sw_desc->unmap_src_cnt = src_cnt;
 741                 sw_desc->unmap_len = len;
 742                 sw_desc->async_tx.flags = flags;
 743                 while (src_cnt--)
 744                         iop_desc_set_zero_sum_src_addr(grp_start, src_cnt,
 745                                                        dma_src[src_cnt]);
 746         }
 747         spin_unlock_bh(&iop_chan->lock);
 748
 749         return sw_desc ? &sw_desc->async_tx : NULL;
 750 }
 751
 752 static struct dma_async_tx_descriptor *
 753 iop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
 754                      unsigned int src_cnt, const unsigned char *scf, size_t len,
 755                      unsigned long flags)
 756 {
 757         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 758         struct iop_adma_desc_slot *sw_desc, *g;
 759         int slot_cnt, slots_per_op;
 760         int continue_srcs;
 761
 762         if (unlikely(!len))
 763                 return NULL;
 764         BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
 765
 766         dev_dbg(iop_chan->device->common.dev,
 767                 "%s src_cnt: %d len: %u flags: %lx\n",
 768                 __func__, src_cnt, len, flags);
 769
 770         if (dmaf_p_disabled_continue(flags))
 771                 continue_srcs = 1+src_cnt;
 772         else if (dmaf_continue(flags))
 773                 continue_srcs = 3+src_cnt;
 774         else
 775                 continue_srcs = 0+src_cnt;
 776
 777         spin_lock_bh(&iop_chan->lock);
 778         slot_cnt = iop_chan_pq_slot_count(len, continue_srcs, &slots_per_op);
 779         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 780         if (sw_desc) {
 781                 int i;
 782
 783                 g = sw_desc->group_head;
 784                 iop_desc_set_byte_count(g, iop_chan, len);
 785
 786                 /* even if P is disabled its destination address (bits
 787                  * [3:0]) must match Q.  It is ok if P points to an
 788                  * invalid address, it won't be written.
 789                  */
 790                 if (flags & DMA_PREP_PQ_DISABLE_P)
 791                         dst[0] = dst[1] & 0x7;
 792
 793                 iop_desc_set_pq_addr(g, dst);
 794                 sw_desc->unmap_src_cnt = src_cnt;
 795                 sw_desc->unmap_len = len;
 796                 sw_desc->async_tx.flags = flags;
 797                 for (i = 0; i < src_cnt; i++)
 798                         iop_desc_set_pq_src_addr(g, i, src[i], scf[i]);
 799
 800                 /* if we are continuing a previous operation factor in
 801                  * the old p and q values, see the comment for dma_maxpq
 802                  * in include/linux/dmaengine.h
 803                  */
 804                 if (dmaf_p_disabled_continue(flags))
 805                         iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
 806                 else if (dmaf_continue(flags)) {
 807                         iop_desc_set_pq_src_addr(g, i++, dst[0], 0);
 808                         iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
 809                         iop_desc_set_pq_src_addr(g, i++, dst[1], 0);
 810                 }
 811                 iop_desc_init_pq(g, i, flags);
 812         }
 813         spin_unlock_bh(&iop_chan->lock);
 814
 815         return sw_desc ? &sw_desc->async_tx : NULL;
 816 }
 817
 818 static struct dma_async_tx_descriptor *
 819 iop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
 820                          unsigned int src_cnt, const unsigned char *scf,
 821                          size_t len, enum sum_check_flags *pqres,
 822                          unsigned long flags)
 823 {
 824         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 825         struct iop_adma_desc_slot *sw_desc, *g;
 826         int slot_cnt, slots_per_op;
 827
 828         if (unlikely(!len))
 829                 return NULL;
 830         BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
 831
 832         dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
 833                 __func__, src_cnt, len);
 834
 835         spin_lock_bh(&iop_chan->lock);
 836         slot_cnt = iop_chan_pq_zero_sum_slot_count(len, src_cnt + 2, &slots_per_op);
 837         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 838         if (sw_desc) {
 839                 /* for validate operations p and q are tagged onto the
 840                  * end of the source list
 841                  */
 842                 int pq_idx = src_cnt;
 843
 844                 g = sw_desc->group_head;
 845                 iop_desc_init_pq_zero_sum(g, src_cnt+2, flags);
 846                 iop_desc_set_pq_zero_sum_byte_count(g, len);
 847                 g->pq_check_result = pqres;
 848                 pr_debug("\t%s: g->pq_check_result: %p\n",
 849                         __func__, g->pq_check_result);
 850                 sw_desc->unmap_src_cnt = src_cnt+2;
 851                 sw_desc->unmap_len = len;
 852                 sw_desc->async_tx.flags = flags;
 853                 while (src_cnt--)
 854                         iop_desc_set_pq_zero_sum_src_addr(g, src_cnt,
 855                                                           src[src_cnt],
 856                                                           scf[src_cnt]);
 857                 iop_desc_set_pq_zero_sum_addr(g, pq_idx, src);
 858         }
 859         spin_unlock_bh(&iop_chan->lock);
 860
 861         return sw_desc ? &sw_desc->async_tx : NULL;
 862 }
 863
 864 static void iop_adma_free_chan_resources(struct dma_chan *chan)
 865 {
 866         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 867         struct iop_adma_desc_slot *iter, *_iter;
 868         int in_use_descs = 0;
 869
 870         iop_adma_slot_cleanup(iop_chan);
 871
 872         spin_lock_bh(&iop_chan->lock);
 873         list_for_each_entry_safe(iter, _iter, &iop_chan->chain,
 874                                         chain_node) {
 875                 in_use_descs++;
 876                 list_del(&iter->chain_node);
 877         }
 878         list_for_each_entry_safe_reverse(
 879                 iter, _iter, &iop_chan->all_slots, slot_node) {
 880                 list_del(&iter->slot_node);
 881                 kfree(iter);
 882                 iop_chan->slots_allocated--;
 883         }
 884         iop_chan->last_used = NULL;
 885
 886         dev_dbg(iop_chan->device->common.dev, "%s slots_allocated %d\n",
 887                 __func__, iop_chan->slots_allocated);
 888         spin_unlock_bh(&iop_chan->lock);
 889
 890         /* one is ok since we left it on there on purpose */
 891         if (in_use_descs > 1)
 892                 printk(KERN_ERR "IOP: Freeing %d in use descriptors!\n",
 893                         in_use_descs - 1);
 894 }
 895
 896 /**
 897  * iop_adma_is_complete - poll the status of an ADMA transaction
 898  * @chan: ADMA channel handle
 899  * @cookie: ADMA transaction identifier
 900  */
 901 static enum dma_status iop_adma_is_complete(struct dma_chan *chan,
 902                                         dma_cookie_t cookie,
 903                                         dma_cookie_t *done,
 904                                         dma_cookie_t *used)
 905 {
 906         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 907         dma_cookie_t last_used;
 908         dma_cookie_t last_complete;
 909         enum dma_status ret;
 910
 911         last_used = chan->cookie;
 912         last_complete = iop_chan->completed_cookie;
 913
 914         if (done)
 915                 *done = last_complete;
 916         if (used)
 917                 *used = last_used;
 918
 919         ret = dma_async_is_complete(cookie, last_complete, last_used);
 920         if (ret == DMA_SUCCESS)
 921                 return ret;
 922
 923         iop_adma_slot_cleanup(iop_chan);
 924
 925         last_used = chan->cookie;
 926         last_complete = iop_chan->completed_cookie;
 927
 928         if (done)
 929                 *done = last_complete;
 930         if (used)
 931                 *used = last_used;
 932
 933         return dma_async_is_complete(cookie, last_complete, last_used);
 934 }
 935
 936 static irqreturn_t iop_adma_eot_handler(int irq, void *data)
 937 {
 938         struct iop_adma_chan *chan = data;
 939
 940         dev_dbg(chan->device->common.dev, "%s\n", __func__);
 941
 942         tasklet_schedule(&chan->irq_tasklet);
 943
 944         iop_adma_device_clear_eot_status(chan);
 945
 946         return IRQ_HANDLED;
 947 }
 948
 949 static irqreturn_t iop_adma_eoc_handler(int irq, void *data)
 950 {
 951         struct iop_adma_chan *chan = data;
 952
 953         dev_dbg(chan->device->common.dev, "%s\n", __func__);
 954
 955         tasklet_schedule(&chan->irq_tasklet);
 956
 957         iop_adma_device_clear_eoc_status(chan);
 958
 959         return IRQ_HANDLED;
 960 }
 961
 962 static irqreturn_t iop_adma_err_handler(int irq, void *data)
 963 {
 964         struct iop_adma_chan *chan = data;
 965         unsigned long status = iop_chan_get_status(chan);
 966
 967         dev_printk(KERN_ERR, chan->device->common.dev,
 968                 "error ( %s%s%s%s%s%s%s)\n",
 969                 iop_is_err_int_parity(status, chan) ? "int_parity " : "",
 970                 iop_is_err_mcu_abort(status, chan) ? "mcu_abort " : "",
 971                 iop_is_err_int_tabort(status, chan) ? "int_tabort " : "",
 972                 iop_is_err_int_mabort(status, chan) ? "int_mabort " : "",
 973                 iop_is_err_pci_tabort(status, chan) ? "pci_tabort " : "",
 974                 iop_is_err_pci_mabort(status, chan) ? "pci_mabort " : "",
 975                 iop_is_err_split_tx(status, chan) ? "split_tx " : "");
 976
 977         iop_adma_device_clear_err_status(chan);
 978
 979         BUG();
 980
 981         return IRQ_HANDLED;
 982 }
 983
 984 static void iop_adma_issue_pending(struct dma_chan *chan)
 985 {
 986         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 987
 988         if (iop_chan->pending) {
 989                 iop_chan->pending = 0;
 990                 iop_chan_append(iop_chan);
 991         }
 992 }
 993
 994 /*
 995  * Perform a transaction to verify the HW works.
 996  */
 997 #define IOP_ADMA_TEST_SIZE 2000
 998
 999 static int __devinit iop_adma_memcpy_self_test(struct iop_adma_device *device)
1000 {
1001         int i;
1002         void *src, *dest;
1003         dma_addr_t src_dma, dest_dma;
1004         struct dma_chan *dma_chan;
1005         dma_cookie_t cookie;
1006         struct dma_async_tx_descriptor *tx;
1007         int err = 0;
1008         struct iop_adma_chan *iop_chan;
1009
1010         dev_dbg(device->common.dev, "%s\n", __func__);
1011
1012         src = kmalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL);
1013         if (!src)
1014                 return -ENOMEM;
1015         dest = kzalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL);
1016         if (!dest) {
1017                 kfree(src);
1018                 return -ENOMEM;
1019         }
1020
1021         /* Fill in src buffer */
1022         for (i = 0; i < IOP_ADMA_TEST_SIZE; i++)
1023                 ((u8 *) src)[i] = (u8)i;
1024
1025         /* Start copy, using first DMA channel */
1026         dma_chan = container_of(device->common.channels.next,
1027                                 struct dma_chan,
1028                                 device_node);
1029         if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
1030                 err = -ENODEV;
1031                 goto out;
1032         }
1033
1034         dest_dma = dma_map_single(dma_chan->device->dev, dest,
1035                                 IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
1036         src_dma = dma_map_single(dma_chan->device->dev, src,
1037                                 IOP_ADMA_TEST_SIZE, DMA_TO_DEVICE);
1038         tx = iop_adma_prep_dma_memcpy(dma_chan, dest_dma, src_dma,
1039                                       IOP_ADMA_TEST_SIZE,
1040                                       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1041
1042         cookie = iop_adma_tx_submit(tx);
1043         iop_adma_issue_pending(dma_chan);
1044         msleep(1);
1045
1046         if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) !=
1047                         DMA_SUCCESS) {
1048                 dev_printk(KERN_ERR, dma_chan->device->dev,
1049                         "Self-test copy timed out, disabling\n");
1050                 err = -ENODEV;
1051                 goto free_resources;
1052         }
1053
1054         iop_chan = to_iop_adma_chan(dma_chan);
1055         dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma,
1056                 IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
1057         if (memcmp(src, dest, IOP_ADMA_TEST_SIZE)) {
1058                 dev_printk(KERN_ERR, dma_chan->device->dev,
1059                         "Self-test copy failed compare, disabling\n");
1060                 err = -ENODEV;
1061                 goto free_resources;
1062         }
1063
1064 free_resources:
1065         iop_adma_free_chan_resources(dma_chan);
1066 out:
1067         kfree(src);
1068         kfree(dest);
1069         return err;
1070 }
1071
1072 #define IOP_ADMA_NUM_SRC_TEST 4 /* must be <= 15 */
1073 static int __devinit
1074 iop_adma_xor_val_self_test(struct iop_adma_device *device)
1075 {
1076         int i, src_idx;
1077         struct page *dest;
1078         struct page *xor_srcs[IOP_ADMA_NUM_SRC_TEST];
1079         struct page *zero_sum_srcs[IOP_ADMA_NUM_SRC_TEST + 1];
1080         dma_addr_t dma_srcs[IOP_ADMA_NUM_SRC_TEST + 1];
1081         dma_addr_t dma_addr, dest_dma;
1082         struct dma_async_tx_descriptor *tx;
1083         struct dma_chan *dma_chan;
1084         dma_cookie_t cookie;
1085         u8 cmp_byte = 0;
1086         u32 cmp_word;
1087         u32 zero_sum_result;
1088         int err = 0;
1089         struct iop_adma_chan *iop_chan;
1090
1091         dev_dbg(device->common.dev, "%s\n", __func__);
1092
1093         for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) {
1094                 xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
1095                 if (!xor_srcs[src_idx]) {
1096                         while (src_idx--)
1097                                 __free_page(xor_srcs[src_idx]);
1098                         return -ENOMEM;
1099                 }
1100         }
1101
1102         dest = alloc_page(GFP_KERNEL);
1103         if (!dest) {
1104                 while (src_idx--)
1105                         __free_page(xor_srcs[src_idx]);
1106                 return -ENOMEM;
1107         }
1108
1109         /* Fill in src buffers */
1110         for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) {
1111                 u8 *ptr = page_address(xor_srcs[src_idx]);
1112                 for (i = 0; i < PAGE_SIZE; i++)
1113                         ptr[i] = (1 << src_idx);
1114         }
1115
1116         for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++)
1117                 cmp_byte ^= (u8) (1 << src_idx);
1118
1119         cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
1120                         (cmp_byte << 8) | cmp_byte;
1121
1122         memset(page_address(dest), 0, PAGE_SIZE);
1123
1124         dma_chan = container_of(device->common.channels.next,
1125                                 struct dma_chan,
1126                                 device_node);
1127         if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
1128                 err = -ENODEV;
1129                 goto out;
1130         }
1131
1132         /* test xor */
1133         dest_dma = dma_map_page(dma_chan->device->dev, dest, 0,
1134                                 PAGE_SIZE, DMA_FROM_DEVICE);
1135         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1136                 dma_srcs[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i],
1137                                            0, PAGE_SIZE, DMA_TO_DEVICE);
1138         tx = iop_adma_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
1139                                    IOP_ADMA_NUM_SRC_TEST, PAGE_SIZE,
1140                                    DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1141
1142         cookie = iop_adma_tx_submit(tx);
1143         iop_adma_issue_pending(dma_chan);
1144         msleep(8);
1145
1146         if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) !=
1147                 DMA_SUCCESS) {
1148                 dev_printk(KERN_ERR, dma_chan->device->dev,
1149                         "Self-test xor timed out, disabling\n");
1150                 err = -ENODEV;
1151                 goto free_resources;
1152         }
1153
1154         iop_chan = to_iop_adma_chan(dma_chan);
1155         dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma,
1156                 PAGE_SIZE, DMA_FROM_DEVICE);
1157         for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
1158                 u32 *ptr = page_address(dest);
1159                 if (ptr[i] != cmp_word) {
1160                         dev_printk(KERN_ERR, dma_chan->device->dev,
1161                                 "Self-test xor failed compare, disabling\n");
1162                         err = -ENODEV;
1163                         goto free_resources;
1164                 }
1165         }
1166         dma_sync_single_for_device(&iop_chan->device->pdev->dev, dest_dma,
1167                 PAGE_SIZE, DMA_TO_DEVICE);
1168
1169         /* skip zero sum if the capability is not present */
1170         if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask))
1171                 goto free_resources;
1172
1173         /* zero sum the sources with the destintation page */
1174         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1175                 zero_sum_srcs[i] = xor_srcs[i];
1176         zero_sum_srcs[i] = dest;
1177
1178         zero_sum_result = 1;
1179
1180         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++)
1181                 dma_srcs[i] = dma_map_page(dma_chan->device->dev,
1182                                            zero_sum_srcs[i], 0, PAGE_SIZE,
1183                                            DMA_TO_DEVICE);
1184         tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
1185                                        IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
1186                                        &zero_sum_result,
1187                                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1188
1189         cookie = iop_adma_tx_submit(tx);
1190         iop_adma_issue_pending(dma_chan);
1191         msleep(8);
1192
1193         if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
1194                 dev_printk(KERN_ERR, dma_chan->device->dev,
1195                         "Self-test zero sum timed out, disabling\n");
1196                 err = -ENODEV;
1197                 goto free_resources;
1198         }
1199
1200         if (zero_sum_result != 0) {
1201                 dev_printk(KERN_ERR, dma_chan->device->dev,
1202                         "Self-test zero sum failed compare, disabling\n");
1203                 err = -ENODEV;
1204                 goto free_resources;
1205         }
1206
1207         /* test memset */
1208         dma_addr = dma_map_page(dma_chan->device->dev, dest, 0,
1209                         PAGE_SIZE, DMA_FROM_DEVICE);
1210         tx = iop_adma_prep_dma_memset(dma_chan, dma_addr, 0, PAGE_SIZE,
1211                                       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1212
1213         cookie = iop_adma_tx_submit(tx);
1214         iop_adma_issue_pending(dma_chan);
1215         msleep(8);
1216
1217         if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
1218                 dev_printk(KERN_ERR, dma_chan->device->dev,
1219                         "Self-test memset timed out, disabling\n");
1220                 err = -ENODEV;
1221                 goto free_resources;
1222         }
1223
1224         for (i = 0; i < PAGE_SIZE/sizeof(u32); i++) {
1225                 u32 *ptr = page_address(dest);
1226                 if (ptr[i]) {
1227                         dev_printk(KERN_ERR, dma_chan->device->dev,
1228                                 "Self-test memset failed compare, disabling\n");
1229                         err = -ENODEV;
1230                         goto free_resources;
1231                 }
1232         }
1233
1234         /* test for non-zero parity sum */
1235         zero_sum_result = 0;
1236         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++)
1237                 dma_srcs[i] = dma_map_page(dma_chan->device->dev,
1238                                            zero_sum_srcs[i], 0, PAGE_SIZE,
1239                                            DMA_TO_DEVICE);
1240         tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
1241                                        IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
1242                                        &zero_sum_result,
1243                                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1244
1245         cookie = iop_adma_tx_submit(tx);
1246         iop_adma_issue_pending(dma_chan);
1247         msleep(8);
1248
1249         if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
1250                 dev_printk(KERN_ERR, dma_chan->device->dev,
1251                         "Self-test non-zero sum timed out, disabling\n");
1252                 err = -ENODEV;
1253                 goto free_resources;
1254         }
1255
1256         if (zero_sum_result != 1) {
1257                 dev_printk(KERN_ERR, dma_chan->device->dev,
1258                         "Self-test non-zero sum failed compare, disabling\n");
1259                 err = -ENODEV;
1260                 goto free_resources;
1261         }
1262
1263 free_resources:
1264         iop_adma_free_chan_resources(dma_chan);
1265 out:
1266         src_idx = IOP_ADMA_NUM_SRC_TEST;
1267         while (src_idx--)
1268                 __free_page(xor_srcs[src_idx]);
1269         __free_page(dest);
1270         return err;
1271 }
1272
1273 #ifdef CONFIG_MD_RAID6_PQ
1274 static int __devinit
1275 iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device)
1276 {
1277         /* combined sources, software pq results, and extra hw pq results */
1278         struct page *pq[IOP_ADMA_NUM_SRC_TEST+2+2];
1279         /* ptr to the extra hw pq buffers defined above */
1280         struct page **pq_hw = &pq[IOP_ADMA_NUM_SRC_TEST+2];
1281         /* address conversion buffers (dma_map / page_address) */
1282         void *pq_sw[IOP_ADMA_NUM_SRC_TEST+2];
1283         dma_addr_t pq_src[IOP_ADMA_NUM_SRC_TEST];
1284         dma_addr_t pq_dest[2];
1285
1286         int i;
1287         struct dma_async_tx_descriptor *tx;
1288         struct dma_chan *dma_chan;
1289         dma_cookie_t cookie;
1290         u32 zero_sum_result;
1291         int err = 0;
1292         struct device *dev;
1293
1294         dev_dbg(device->common.dev, "%s\n", __func__);
1295
1296         for (i = 0; i < ARRAY_SIZE(pq); i++) {
1297                 pq[i] = alloc_page(GFP_KERNEL);
1298                 if (!pq[i]) {
1299                         while (i--)
1300                                 __free_page(pq[i]);
1301                         return -ENOMEM;
1302                 }
1303         }
1304
1305         /* Fill in src buffers */
1306         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) {
1307                 pq_sw[i] = page_address(pq[i]);
1308                 memset(pq_sw[i], 0x11111111 * (1<<i), PAGE_SIZE);
1309         }
1310         pq_sw[i] = page_address(pq[i]);
1311         pq_sw[i+1] = page_address(pq[i+1]);
1312
1313         dma_chan = container_of(device->common.channels.next,
1314                                 struct dma_chan,
1315                                 device_node);
1316         if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
1317                 err = -ENODEV;
1318                 goto out;
1319         }
1320
1321         dev = dma_chan->device->dev;
1322
1323         /* initialize the dests */
1324         memset(page_address(pq_hw[0]), 0 , PAGE_SIZE);
1325         memset(page_address(pq_hw[1]), 0 , PAGE_SIZE);
1326
1327         /* test pq */
1328         pq_dest[0] = dma_map_page(dev, pq_hw[0], 0, PAGE_SIZE, DMA_FROM_DEVICE);
1329         pq_dest[1] = dma_map_page(dev, pq_hw[1], 0, PAGE_SIZE, DMA_FROM_DEVICE);
1330         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1331                 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1332                                          DMA_TO_DEVICE);
1333
1334         tx = iop_adma_prep_dma_pq(dma_chan, pq_dest, pq_src,
1335                                   IOP_ADMA_NUM_SRC_TEST, (u8 *)raid6_gfexp,
1336                                   PAGE_SIZE,
1337                                   DMA_PREP_INTERRUPT |
1338                                   DMA_CTRL_ACK);
1339
1340         cookie = iop_adma_tx_submit(tx);
1341         iop_adma_issue_pending(dma_chan);
1342         msleep(8);
1343
1344         if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) !=
1345                 DMA_SUCCESS) {
1346                 dev_err(dev, "Self-test pq timed out, disabling\n");
1347                 err = -ENODEV;
1348                 goto free_resources;
1349         }
1350
1351         raid6_call.gen_syndrome(IOP_ADMA_NUM_SRC_TEST+2, PAGE_SIZE, pq_sw);
1352
1353         if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST],
1354                    page_address(pq_hw[0]), PAGE_SIZE) != 0) {
1355                 dev_err(dev, "Self-test p failed compare, disabling\n");
1356                 err = -ENODEV;
1357                 goto free_resources;
1358         }
1359         if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST+1],
1360                    page_address(pq_hw[1]), PAGE_SIZE) != 0) {
1361                 dev_err(dev, "Self-test q failed compare, disabling\n");
1362                 err = -ENODEV;
1363                 goto free_resources;
1364         }
1365
1366         /* test correct zero sum using the software generated pq values */
1367         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
1368                 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1369                                          DMA_TO_DEVICE);
1370
1371         zero_sum_result = ~0;
1372         tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
1373                                       pq_src, IOP_ADMA_NUM_SRC_TEST,
1374                                       raid6_gfexp, PAGE_SIZE, &zero_sum_result,
1375                                       DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
1376
1377         cookie = iop_adma_tx_submit(tx);
1378         iop_adma_issue_pending(dma_chan);
1379         msleep(8);
1380
1381         if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) !=
1382                 DMA_SUCCESS) {
1383                 dev_err(dev, "Self-test pq-zero-sum timed out, disabling\n");
1384                 err = -ENODEV;
1385                 goto free_resources;
1386         }
1387
1388         if (zero_sum_result != 0) {
1389                 dev_err(dev, "Self-test pq-zero-sum failed to validate: %x\n",
1390                         zero_sum_result);
1391                 err = -ENODEV;
1392                 goto free_resources;
1393         }
1394
1395         /* test incorrect zero sum */
1396         i = IOP_ADMA_NUM_SRC_TEST;
1397         memset(pq_sw[i] + 100, 0, 100);
1398         memset(pq_sw[i+1] + 200, 0, 200);
1399         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
1400                 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1401                                          DMA_TO_DEVICE);
1402
1403         zero_sum_result = 0;
1404         tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
1405                                       pq_src, IOP_ADMA_NUM_SRC_TEST,
1406                                       raid6_gfexp, PAGE_SIZE, &zero_sum_result,
1407                                       DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
1408
1409         cookie = iop_adma_tx_submit(tx);
1410         iop_adma_issue_pending(dma_chan);
1411         msleep(8);
1412
1413         if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) !=
1414                 DMA_SUCCESS) {
1415                 dev_err(dev, "Self-test !pq-zero-sum timed out, disabling\n");
1416                 err = -ENODEV;
1417                 goto free_resources;
1418         }
1419
1420         if (zero_sum_result != (SUM_CHECK_P_RESULT | SUM_CHECK_Q_RESULT)) {
1421                 dev_err(dev, "Self-test !pq-zero-sum failed to validate: %x\n",
1422                         zero_sum_result);
1423                 err = -ENODEV;
1424                 goto free_resources;
1425         }
1426
1427 free_resources:
1428         iop_adma_free_chan_resources(dma_chan);
1429 out:
1430         i = ARRAY_SIZE(pq);
1431         while (i--)
1432                 __free_page(pq[i]);
1433         return err;
1434 }
1435 #endif
1436
1437 static int __devexit iop_adma_remove(struct platform_device *dev)
1438 {
1439         struct iop_adma_device *device = platform_get_drvdata(dev);
1440         struct dma_chan *chan, *_chan;
1441         struct iop_adma_chan *iop_chan;
1442         struct iop_adma_platform_data *plat_data = dev->dev.platform_data;
1443
1444         dma_async_device_unregister(&device->common);
1445
1446         dma_free_coherent(&dev->dev, plat_data->pool_size,
1447                         device->dma_desc_pool_virt, device->dma_desc_pool);
1448
1449         list_for_each_entry_safe(chan, _chan, &device->common.channels,
1450                                 device_node) {
1451                 iop_chan = to_iop_adma_chan(chan);
1452                 list_del(&chan->device_node);
1453                 kfree(iop_chan);
1454         }
1455         kfree(device);
1456
1457         return 0;
1458 }
1459
1460 static int __devinit iop_adma_probe(struct platform_device *pdev)
1461 {
1462         struct resource *res;
1463         int ret = 0, i;
1464         struct iop_adma_device *adev;
1465         struct iop_adma_chan *iop_chan;
1466         struct dma_device *dma_dev;
1467         struct iop_adma_platform_data *plat_data = pdev->dev.platform_data;
1468
1469         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1470         if (!res)
1471                 return -ENODEV;
1472
1473         if (!devm_request_mem_region(&pdev->dev, res->start,
1474                                 resource_size(res), pdev->name))
1475                 return -EBUSY;
1476
1477         adev = kzalloc(sizeof(*adev), GFP_KERNEL);
1478         if (!adev)
1479                 return -ENOMEM;
1480         dma_dev = &adev->common;
1481
1482         /* allocate coherent memory for hardware descriptors
1483          * note: writecombine gives slightly better performance, but
1484          * requires that we explicitly flush the writes
1485          */
1486         if ((adev->dma_desc_pool_virt = dma_alloc_writecombine(&pdev->dev,
1487                                         plat_data->pool_size,
1488                                         &adev->dma_desc_pool,
1489                                         GFP_KERNEL)) == NULL) {
1490                 ret = -ENOMEM;
1491                 goto err_free_adev;
1492         }
1493
1494         dev_dbg(&pdev->dev, "%s: allocted descriptor pool virt %p phys %p\n",
1495                 __func__, adev->dma_desc_pool_virt,
1496                 (void *) adev->dma_desc_pool);
1497
1498         adev->id = plat_data->hw_id;
1499
1500         /* discover transaction capabilites from the platform data */
1501         dma_dev->cap_mask = plat_data->cap_mask;
1502
1503         adev->pdev = pdev;
1504         platform_set_drvdata(pdev, adev);
1505
1506         INIT_LIST_HEAD(&dma_dev->channels);
1507
1508         /* set base routines */
1509         dma_dev->device_alloc_chan_resources = iop_adma_alloc_chan_resources;
1510         dma_dev->device_free_chan_resources = iop_adma_free_chan_resources;
1511         dma_dev->device_is_tx_complete = iop_adma_is_complete;
1512         dma_dev->device_issue_pending = iop_adma_issue_pending;
1513         dma_dev->dev = &pdev->dev;
1514
1515         /* set prep routines based on capability */
1516         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1517                 dma_dev->device_prep_dma_memcpy = iop_adma_prep_dma_memcpy;
1518         if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask))
1519                 dma_dev->device_prep_dma_memset = iop_adma_prep_dma_memset;
1520         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1521                 dma_dev->max_xor = iop_adma_get_max_xor();
1522                 dma_dev->device_prep_dma_xor = iop_adma_prep_dma_xor;
1523         }
1524         if (dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask))
1525                 dma_dev->device_prep_dma_xor_val =
1526                         iop_adma_prep_dma_xor_val;
1527         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1528                 dma_set_maxpq(dma_dev, iop_adma_get_max_pq(), 0);
1529                 dma_dev->device_prep_dma_pq = iop_adma_prep_dma_pq;
1530         }
1531         if (dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask))
1532                 dma_dev->device_prep_dma_pq_val =
1533                         iop_adma_prep_dma_pq_val;
1534         if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1535                 dma_dev->device_prep_dma_interrupt =
1536                         iop_adma_prep_dma_interrupt;
1537
1538         iop_chan = kzalloc(sizeof(*iop_chan), GFP_KERNEL);
1539         if (!iop_chan) {
1540                 ret = -ENOMEM;
1541                 goto err_free_dma;
1542         }
1543         iop_chan->device = adev;
1544
1545         iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start,
1546                                         resource_size(res));
1547         if (!iop_chan->mmr_base) {
1548                 ret = -ENOMEM;
1549                 goto err_free_iop_chan;
1550         }
1551         tasklet_init(&iop_chan->irq_tasklet, iop_adma_tasklet, (unsigned long)
1552                 iop_chan);
1553
1554         /* clear errors before enabling interrupts */
1555         iop_adma_device_clear_err_status(iop_chan);
1556
1557         for (i = 0; i < 3; i++) {
1558                 irq_handler_t handler[] = { iop_adma_eot_handler,
1559                                         iop_adma_eoc_handler,
1560                                         iop_adma_err_handler };
1561                 int irq = platform_get_irq(pdev, i);
1562                 if (irq < 0) {
1563                         ret = -ENXIO;
1564                         goto err_free_iop_chan;
1565                 } else {
1566                         ret = devm_request_irq(&pdev->dev, irq,
1567                                         handler[i], 0, pdev->name, iop_chan);
1568                         if (ret)
1569                                 goto err_free_iop_chan;
1570                 }
1571         }
1572
1573         spin_lock_init(&iop_chan->lock);
1574         INIT_LIST_HEAD(&iop_chan->chain);
1575         INIT_LIST_HEAD(&iop_chan->all_slots);
1576         iop_chan->common.device = dma_dev;
1577         list_add_tail(&iop_chan->common.device_node, &dma_dev->channels);
1578
1579         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1580                 ret = iop_adma_memcpy_self_test(adev);
1581                 dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret);
1582                 if (ret)
1583                         goto err_free_iop_chan;
1584         }
1585
1586         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask) ||
1587             dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) {
1588                 ret = iop_adma_xor_val_self_test(adev);
1589                 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
1590                 if (ret)
1591                         goto err_free_iop_chan;
1592         }
1593
1594         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask) &&
1595             dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) {
1596                 #ifdef CONFIG_MD_RAID6_PQ
1597                 ret = iop_adma_pq_zero_sum_self_test(adev);
1598                 dev_dbg(&pdev->dev, "pq self test returned %d\n", ret);
1599                 #else
1600                 /* can not test raid6, so do not publish capability */
1601                 dma_cap_clear(DMA_PQ, dma_dev->cap_mask);
1602                 dma_cap_clear(DMA_PQ_VAL, dma_dev->cap_mask);
1603                 ret = 0;
1604                 #endif
1605                 if (ret)
1606                         goto err_free_iop_chan;
1607         }
1608
1609         dev_printk(KERN_INFO, &pdev->dev, "Intel(R) IOP: "
1610           "( %s%s%s%s%s%s%s)\n",
1611           dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "",
1612           dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask) ? "pq_val " : "",
1613           dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1614           dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask) ? "xor_val " : "",
1615           dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)  ? "fill " : "",
1616           dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
1617           dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
1618
1619         dma_async_device_register(dma_dev);
1620         goto out;
1621
1622  err_free_iop_chan:
1623         kfree(iop_chan);
1624  err_free_dma:
1625         dma_free_coherent(&adev->pdev->dev, plat_data->pool_size,
1626                         adev->dma_desc_pool_virt, adev->dma_desc_pool);
1627  err_free_adev:
1628         kfree(adev);
1629  out:
1630         return ret;
1631 }
1632
1633 static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan)
1634 {
1635         struct iop_adma_desc_slot *sw_desc, *grp_start;
1636         dma_cookie_t cookie;
1637         int slot_cnt, slots_per_op;
1638
1639         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
1640
1641         spin_lock_bh(&iop_chan->lock);
1642         slot_cnt = iop_chan_memcpy_slot_count(0, &slots_per_op);
1643         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
1644         if (sw_desc) {
1645                 grp_start = sw_desc->group_head;
1646
1647                 list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
1648                 async_tx_ack(&sw_desc->async_tx);
1649                 iop_desc_init_memcpy(grp_start, 0);
1650                 iop_desc_set_byte_count(grp_start, iop_chan, 0);
1651                 iop_desc_set_dest_addr(grp_start, iop_chan, 0);
1652                 iop_desc_set_memcpy_src_addr(grp_start, 0);
1653
1654                 cookie = iop_chan->common.cookie;
1655                 cookie++;
1656                 if (cookie <= 1)
1657                         cookie = 2;
1658
1659                 /* initialize the completed cookie to be less than
1660                  * the most recently used cookie
1661                  */
1662                 iop_chan->completed_cookie = cookie - 1;
1663                 iop_chan->common.cookie = sw_desc->async_tx.cookie = cookie;
1664
1665                 /* channel should not be busy */
1666                 BUG_ON(iop_chan_is_busy(iop_chan));
1667
1668                 /* clear any prior error-status bits */
1669                 iop_adma_device_clear_err_status(iop_chan);
1670
1671                 /* disable operation */
1672                 iop_chan_disable(iop_chan);
1673
1674                 /* set the descriptor address */
1675                 iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys);
1676
1677                 /* 1/ don't add pre-chained descriptors
1678                  * 2/ dummy read to flush next_desc write
1679                  */
1680                 BUG_ON(iop_desc_get_next_desc(sw_desc));
1681
1682                 /* run the descriptor */
1683                 iop_chan_enable(iop_chan);
1684         } else
1685                 dev_printk(KERN_ERR, iop_chan->device->common.dev,
1686                          "failed to allocate null descriptor\n");
1687         spin_unlock_bh(&iop_chan->lock);
1688 }
1689
1690 static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan)
1691 {
1692         struct iop_adma_desc_slot *sw_desc, *grp_start;
1693         dma_cookie_t cookie;
1694         int slot_cnt, slots_per_op;
1695
1696         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
1697
1698         spin_lock_bh(&iop_chan->lock);
1699         slot_cnt = iop_chan_xor_slot_count(0, 2, &slots_per_op);
1700         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
1701         if (sw_desc) {
1702                 grp_start = sw_desc->group_head;
1703                 list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
1704                 async_tx_ack(&sw_desc->async_tx);
1705                 iop_desc_init_null_xor(grp_start, 2, 0);
1706                 iop_desc_set_byte_count(grp_start, iop_chan, 0);
1707                 iop_desc_set_dest_addr(grp_start, iop_chan, 0);
1708                 iop_desc_set_xor_src_addr(grp_start, 0, 0);
1709                 iop_desc_set_xor_src_addr(grp_start, 1, 0);
1710
1711                 cookie = iop_chan->common.cookie;
1712                 cookie++;
1713                 if (cookie <= 1)
1714                         cookie = 2;
1715
1716                 /* initialize the completed cookie to be less than
1717                  * the most recently used cookie
1718                  */
1719                 iop_chan->completed_cookie = cookie - 1;
1720                 iop_chan->common.cookie = sw_desc->async_tx.cookie = cookie;
1721
1722                 /* channel should not be busy */
1723                 BUG_ON(iop_chan_is_busy(iop_chan));
1724
1725                 /* clear any prior error-status bits */
1726                 iop_adma_device_clear_err_status(iop_chan);
1727
1728                 /* disable operation */
1729                 iop_chan_disable(iop_chan);
1730
1731                 /* set the descriptor address */
1732                 iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys);
1733
1734                 /* 1/ don't add pre-chained descriptors
1735                  * 2/ dummy read to flush next_desc write
1736                  */
1737                 BUG_ON(iop_desc_get_next_desc(sw_desc));
1738
1739                 /* run the descriptor */
1740                 iop_chan_enable(iop_chan);
1741         } else
1742                 dev_printk(KERN_ERR, iop_chan->device->common.dev,
1743                         "failed to allocate null descriptor\n");
1744         spin_unlock_bh(&iop_chan->lock);
1745 }
1746
1747 MODULE_ALIAS("platform:iop-adma");
1748
1749 static struct platform_driver iop_adma_driver = {
1750         .probe          = iop_adma_probe,
1751         .remove         = __devexit_p(iop_adma_remove),
1752         .driver         = {
1753                 .owner  = THIS_MODULE,
1754                 .name   = "iop-adma",
1755         },
1756 };
1757
1758 static int __init iop_adma_init (void)
1759 {
1760         return platform_driver_register(&iop_adma_driver);
1761 }
1762
1763 static void __exit iop_adma_exit (void)
1764 {
1765         platform_driver_unregister(&iop_adma_driver);
1766         return;
1767 }
1768 module_exit(iop_adma_exit);
1769 module_init(iop_adma_init);
1770
1771 MODULE_AUTHOR("Intel Corporation");
1772 MODULE_DESCRIPTION("IOP ADMA Engine Driver");
1773 MODULE_LICENSE("GPL");