drivers/dma/iop-adma.c

   1 /*
   2  * offload engine driver for the Intel Xscale series of i/o processors
   3  * Copyright © 2006, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  */
  19
  20 /*
  21  * This driver supports the asynchrounous DMA copy and RAID engines available
  22  * on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
  23  */
  24
  25 #include <linux/init.h>
  26 #include <linux/module.h>
  27 #include <linux/delay.h>
  28 #include <linux/dma-mapping.h>
  29 #include <linux/spinlock.h>
  30 #include <linux/interrupt.h>
  31 #include <linux/platform_device.h>
  32 #include <linux/memory.h>
  33 #include <linux/ioport.h>
  34 #include <linux/raid/pq.h>
  35 #include <linux/slab.h>
  36
  37 #include <mach/adma.h>
  38
  39 #define to_iop_adma_chan(chan) container_of(chan, struct iop_adma_chan, common)
  40 #define to_iop_adma_device(dev) \
  41         container_of(dev, struct iop_adma_device, common)
  42 #define tx_to_iop_adma_slot(tx) \
  43         container_of(tx, struct iop_adma_desc_slot, async_tx)
  44
  45 /**
  46  * iop_adma_free_slots - flags descriptor slots for reuse
  47  * @slot: Slot to free
  48  * Caller must hold &iop_chan->lock while calling this function
  49  */
  50 static void iop_adma_free_slots(struct iop_adma_desc_slot *slot)
  51 {
  52         int stride = slot->slots_per_op;
  53
  54         while (stride--) {
  55                 slot->slots_per_op = 0;
  56                 slot = list_entry(slot->slot_node.next,
  57                                 struct iop_adma_desc_slot,
  58                                 slot_node);
  59         }
  60 }
  61
  62 static void
  63 iop_desc_unmap(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc)
  64 {
  65         struct dma_async_tx_descriptor *tx = &desc->async_tx;
  66         struct iop_adma_desc_slot *unmap = desc->group_head;
  67         struct device *dev = &iop_chan->device->pdev->dev;
  68         u32 len = unmap->unmap_len;
  69         enum dma_ctrl_flags flags = tx->flags;
  70         u32 src_cnt;
  71         dma_addr_t addr;
  72         dma_addr_t dest;
  73
  74         src_cnt = unmap->unmap_src_cnt;
  75         dest = iop_desc_get_dest_addr(unmap, iop_chan);
  76         if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
  77                 enum dma_data_direction dir;
  78
  79                 if (src_cnt > 1) /* is xor? */
  80                         dir = DMA_BIDIRECTIONAL;
  81                 else
  82                         dir = DMA_FROM_DEVICE;
  83
  84                 dma_unmap_page(dev, dest, len, dir);
  85         }
  86
  87         if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
  88                 while (src_cnt--) {
  89                         addr = iop_desc_get_src_addr(unmap, iop_chan, src_cnt);
  90                         if (addr == dest)
  91                                 continue;
  92                         dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
  93                 }
  94         }
  95         desc->group_head = NULL;
  96 }
  97
  98 static void
  99 iop_desc_unmap_pq(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc)
 100 {
 101         struct dma_async_tx_descriptor *tx = &desc->async_tx;
 102         struct iop_adma_desc_slot *unmap = desc->group_head;
 103         struct device *dev = &iop_chan->device->pdev->dev;
 104         u32 len = unmap->unmap_len;
 105         enum dma_ctrl_flags flags = tx->flags;
 106         u32 src_cnt = unmap->unmap_src_cnt;
 107         dma_addr_t pdest = iop_desc_get_dest_addr(unmap, iop_chan);
 108         dma_addr_t qdest = iop_desc_get_qdest_addr(unmap, iop_chan);
 109         int i;
 110
 111         if (tx->flags & DMA_PREP_CONTINUE)
 112                 src_cnt -= 3;
 113
 114         if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP) && !desc->pq_check_result) {
 115                 dma_unmap_page(dev, pdest, len, DMA_BIDIRECTIONAL);
 116                 dma_unmap_page(dev, qdest, len, DMA_BIDIRECTIONAL);
 117         }
 118
 119         if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
 120                 dma_addr_t addr;
 121
 122                 for (i = 0; i < src_cnt; i++) {
 123                         addr = iop_desc_get_src_addr(unmap, iop_chan, i);
 124                         dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
 125                 }
 126                 if (desc->pq_check_result) {
 127                         dma_unmap_page(dev, pdest, len, DMA_TO_DEVICE);
 128                         dma_unmap_page(dev, qdest, len, DMA_TO_DEVICE);
 129                 }
 130         }
 131
 132         desc->group_head = NULL;
 133 }
 134
 135
 136 static dma_cookie_t
 137 iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
 138         struct iop_adma_chan *iop_chan, dma_cookie_t cookie)
 139 {
 140         struct dma_async_tx_descriptor *tx = &desc->async_tx;
 141
 142         BUG_ON(tx->cookie < 0);
 143         if (tx->cookie > 0) {
 144                 cookie = tx->cookie;
 145                 tx->cookie = 0;
 146
 147                 /* call the callback (must not sleep or submit new
 148                  * operations to this channel)
 149                  */
 150                 if (tx->callback)
 151                         tx->callback(tx->callback_param);
 152
 153                 /* unmap dma addresses
 154                  * (unmap_single vs unmap_page?)
 155                  */
 156                 if (desc->group_head && desc->unmap_len) {
 157                         if (iop_desc_is_pq(desc))
 158                                 iop_desc_unmap_pq(iop_chan, desc);
 159                         else
 160                                 iop_desc_unmap(iop_chan, desc);
 161                 }
 162         }
 163
 164         /* run dependent operations */
 165         dma_run_dependencies(tx);
 166
 167         return cookie;
 168 }
 169
 170 static int
 171 iop_adma_clean_slot(struct iop_adma_desc_slot *desc,
 172         struct iop_adma_chan *iop_chan)
 173 {
 174         /* the client is allowed to attach dependent operations
 175          * until 'ack' is set
 176          */
 177         if (!async_tx_test_ack(&desc->async_tx))
 178                 return 0;
 179
 180         /* leave the last descriptor in the chain
 181          * so we can append to it
 182          */
 183         if (desc->chain_node.next == &iop_chan->chain)
 184                 return 1;
 185
 186         dev_dbg(iop_chan->device->common.dev,
 187                 "\tfree slot: %d slots_per_op: %d\n",
 188                 desc->idx, desc->slots_per_op);
 189
 190         list_del(&desc->chain_node);
 191         iop_adma_free_slots(desc);
 192
 193         return 0;
 194 }
 195
 196 static void __iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
 197 {
 198         struct iop_adma_desc_slot *iter, *_iter, *grp_start = NULL;
 199         dma_cookie_t cookie = 0;
 200         u32 current_desc = iop_chan_get_current_descriptor(iop_chan);
 201         int busy = iop_chan_is_busy(iop_chan);
 202         int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
 203
 204         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
 205         /* free completed slots from the chain starting with
 206          * the oldest descriptor
 207          */
 208         list_for_each_entry_safe(iter, _iter, &iop_chan->chain,
 209                                         chain_node) {
 210                 pr_debug("\tcookie: %d slot: %d busy: %d "
 211                         "this_desc: %#x next_desc: %#x ack: %d\n",
 212                         iter->async_tx.cookie, iter->idx, busy,
 213                         iter->async_tx.phys, iop_desc_get_next_desc(iter),
 214                         async_tx_test_ack(&iter->async_tx));
 215                 prefetch(_iter);
 216                 prefetch(&_iter->async_tx);
 217
 218                 /* do not advance past the current descriptor loaded into the
 219                  * hardware channel, subsequent descriptors are either in
 220                  * process or have not been submitted
 221                  */
 222                 if (seen_current)
 223                         break;
 224
 225                 /* stop the search if we reach the current descriptor and the
 226                  * channel is busy, or if it appears that the current descriptor
 227                  * needs to be re-read (i.e. has been appended to)
 228                  */
 229                 if (iter->async_tx.phys == current_desc) {
 230                         BUG_ON(seen_current++);
 231                         if (busy || iop_desc_get_next_desc(iter))
 232                                 break;
 233                 }
 234
 235                 /* detect the start of a group transaction */
 236                 if (!slot_cnt && !slots_per_op) {
 237                         slot_cnt = iter->slot_cnt;
 238                         slots_per_op = iter->slots_per_op;
 239                         if (slot_cnt <= slots_per_op) {
 240                                 slot_cnt = 0;
 241                                 slots_per_op = 0;
 242                         }
 243                 }
 244
 245                 if (slot_cnt) {
 246                         pr_debug("\tgroup++\n");
 247                         if (!grp_start)
 248                                 grp_start = iter;
 249                         slot_cnt -= slots_per_op;
 250                 }
 251
 252                 /* all the members of a group are complete */
 253                 if (slots_per_op != 0 && slot_cnt == 0) {
 254                         struct iop_adma_desc_slot *grp_iter, *_grp_iter;
 255                         int end_of_chain = 0;
 256                         pr_debug("\tgroup end\n");
 257
 258                         /* collect the total results */
 259                         if (grp_start->xor_check_result) {
 260                                 u32 zero_sum_result = 0;
 261                                 slot_cnt = grp_start->slot_cnt;
 262                                 grp_iter = grp_start;
 263
 264                                 list_for_each_entry_from(grp_iter,
 265                                         &iop_chan->chain, chain_node) {
 266                                         zero_sum_result |=
 267                                             iop_desc_get_zero_result(grp_iter);
 268                                             pr_debug("\titer%d result: %d\n",
 269                                             grp_iter->idx, zero_sum_result);
 270                                         slot_cnt -= slots_per_op;
 271                                         if (slot_cnt == 0)
 272                                                 break;
 273                                 }
 274                                 pr_debug("\tgrp_start->xor_check_result: %p\n",
 275                                         grp_start->xor_check_result);
 276                                 *grp_start->xor_check_result = zero_sum_result;
 277                         }
 278
 279                         /* clean up the group */
 280                         slot_cnt = grp_start->slot_cnt;
 281                         grp_iter = grp_start;
 282                         list_for_each_entry_safe_from(grp_iter, _grp_iter,
 283                                 &iop_chan->chain, chain_node) {
 284                                 cookie = iop_adma_run_tx_complete_actions(
 285                                         grp_iter, iop_chan, cookie);
 286
 287                                 slot_cnt -= slots_per_op;
 288                                 end_of_chain = iop_adma_clean_slot(grp_iter,
 289                                         iop_chan);
 290
 291                                 if (slot_cnt == 0 || end_of_chain)
 292                                         break;
 293                         }
 294
 295                         /* the group should be complete at this point */
 296                         BUG_ON(slot_cnt);
 297
 298                         slots_per_op = 0;
 299                         grp_start = NULL;
 300                         if (end_of_chain)
 301                                 break;
 302                         else
 303                                 continue;
 304                 } else if (slots_per_op) /* wait for group completion */
 305                         continue;
 306
 307                 /* write back zero sum results (single descriptor case) */
 308                 if (iter->xor_check_result && iter->async_tx.cookie)
 309                         *iter->xor_check_result =
 310                                 iop_desc_get_zero_result(iter);
 311
 312                 cookie = iop_adma_run_tx_complete_actions(
 313                                         iter, iop_chan, cookie);
 314
 315                 if (iop_adma_clean_slot(iter, iop_chan))
 316                         break;
 317         }
 318
 319         if (cookie > 0) {
 320                 iop_chan->completed_cookie = cookie;
 321                 pr_debug("\tcompleted cookie %d\n", cookie);
 322         }
 323 }
 324
 325 static void
 326 iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
 327 {
 328         spin_lock_bh(&iop_chan->lock);
 329         __iop_adma_slot_cleanup(iop_chan);
 330         spin_unlock_bh(&iop_chan->lock);
 331 }
 332
 333 static void iop_adma_tasklet(unsigned long data)
 334 {
 335         struct iop_adma_chan *iop_chan = (struct iop_adma_chan *) data;
 336
 337         /* lockdep will flag depedency submissions as potentially
 338          * recursive locking, this is not the case as a dependency
 339          * submission will never recurse a channels submit routine.
 340          * There are checks in async_tx.c to prevent this.
 341          */
 342         spin_lock_nested(&iop_chan->lock, SINGLE_DEPTH_NESTING);
 343         __iop_adma_slot_cleanup(iop_chan);
 344         spin_unlock(&iop_chan->lock);
 345 }
 346
 347 static struct iop_adma_desc_slot *
 348 iop_adma_alloc_slots(struct iop_adma_chan *iop_chan, int num_slots,
 349                         int slots_per_op)
 350 {
 351         struct iop_adma_desc_slot *iter, *_iter, *alloc_start = NULL;
 352         LIST_HEAD(chain);
 353         int slots_found, retry = 0;
 354
 355         /* start search from the last allocated descrtiptor
 356          * if a contiguous allocation can not be found start searching
 357          * from the beginning of the list
 358          */
 359 retry:
 360         slots_found = 0;
 361         if (retry == 0)
 362                 iter = iop_chan->last_used;
 363         else
 364                 iter = list_entry(&iop_chan->all_slots,
 365                         struct iop_adma_desc_slot,
 366                         slot_node);
 367
 368         list_for_each_entry_safe_continue(
 369                 iter, _iter, &iop_chan->all_slots, slot_node) {
 370                 prefetch(_iter);
 371                 prefetch(&_iter->async_tx);
 372                 if (iter->slots_per_op) {
 373                         /* give up after finding the first busy slot
 374                          * on the second pass through the list
 375                          */
 376                         if (retry)
 377                                 break;
 378
 379                         slots_found = 0;
 380                         continue;
 381                 }
 382
 383                 /* start the allocation if the slot is correctly aligned */
 384                 if (!slots_found++) {
 385                         if (iop_desc_is_aligned(iter, slots_per_op))
 386                                 alloc_start = iter;
 387                         else {
 388                                 slots_found = 0;
 389                                 continue;
 390                         }
 391                 }
 392
 393                 if (slots_found == num_slots) {
 394                         struct iop_adma_desc_slot *alloc_tail = NULL;
 395                         struct iop_adma_desc_slot *last_used = NULL;
 396                         iter = alloc_start;
 397                         while (num_slots) {
 398                                 int i;
 399                                 dev_dbg(iop_chan->device->common.dev,
 400                                         "allocated slot: %d "
 401                                         "(desc %p phys: %#x) slots_per_op %d\n",
 402                                         iter->idx, iter->hw_desc,
 403                                         iter->async_tx.phys, slots_per_op);
 404
 405                                 /* pre-ack all but the last descriptor */
 406                                 if (num_slots != slots_per_op)
 407                                         async_tx_ack(&iter->async_tx);
 408
 409                                 list_add_tail(&iter->chain_node, &chain);
 410                                 alloc_tail = iter;
 411                                 iter->async_tx.cookie = 0;
 412                                 iter->slot_cnt = num_slots;
 413                                 iter->xor_check_result = NULL;
 414                                 for (i = 0; i < slots_per_op; i++) {
 415                                         iter->slots_per_op = slots_per_op - i;
 416                                         last_used = iter;
 417                                         iter = list_entry(iter->slot_node.next,
 418                                                 struct iop_adma_desc_slot,
 419                                                 slot_node);
 420                                 }
 421                                 num_slots -= slots_per_op;
 422                         }
 423                         alloc_tail->group_head = alloc_start;
 424                         alloc_tail->async_tx.cookie = -EBUSY;
 425                         list_splice(&chain, &alloc_tail->tx_list);
 426                         iop_chan->last_used = last_used;
 427                         iop_desc_clear_next_desc(alloc_start);
 428                         iop_desc_clear_next_desc(alloc_tail);
 429                         return alloc_tail;
 430                 }
 431         }
 432         if (!retry++)
 433                 goto retry;
 434
 435         /* perform direct reclaim if the allocation fails */
 436         __iop_adma_slot_cleanup(iop_chan);
 437
 438         return NULL;
 439 }
 440
 441 static dma_cookie_t
 442 iop_desc_assign_cookie(struct iop_adma_chan *iop_chan,
 443         struct iop_adma_desc_slot *desc)
 444 {
 445         dma_cookie_t cookie = iop_chan->common.cookie;
 446         cookie++;
 447         if (cookie < 0)
 448                 cookie = 1;
 449         iop_chan->common.cookie = desc->async_tx.cookie = cookie;
 450         return cookie;
 451 }
 452
 453 static void iop_adma_check_threshold(struct iop_adma_chan *iop_chan)
 454 {
 455         dev_dbg(iop_chan->device->common.dev, "pending: %d\n",
 456                 iop_chan->pending);
 457
 458         if (iop_chan->pending >= IOP_ADMA_THRESHOLD) {
 459                 iop_chan->pending = 0;
 460                 iop_chan_append(iop_chan);
 461         }
 462 }
 463
 464 static dma_cookie_t
 465 iop_adma_tx_submit(struct dma_async_tx_descriptor *tx)
 466 {
 467         struct iop_adma_desc_slot *sw_desc = tx_to_iop_adma_slot(tx);
 468         struct iop_adma_chan *iop_chan = to_iop_adma_chan(tx->chan);
 469         struct iop_adma_desc_slot *grp_start, *old_chain_tail;
 470         int slot_cnt;
 471         int slots_per_op;
 472         dma_cookie_t cookie;
 473         dma_addr_t next_dma;
 474
 475         grp_start = sw_desc->group_head;
 476         slot_cnt = grp_start->slot_cnt;
 477         slots_per_op = grp_start->slots_per_op;
 478
 479         spin_lock_bh(&iop_chan->lock);
 480         cookie = iop_desc_assign_cookie(iop_chan, sw_desc);
 481
 482         old_chain_tail = list_entry(iop_chan->chain.prev,
 483                 struct iop_adma_desc_slot, chain_node);
 484         list_splice_init(&sw_desc->tx_list,
 485                          &old_chain_tail->chain_node);
 486
 487         /* fix up the hardware chain */
 488         next_dma = grp_start->async_tx.phys;
 489         iop_desc_set_next_desc(old_chain_tail, next_dma);
 490         BUG_ON(iop_desc_get_next_desc(old_chain_tail) != next_dma); /* flush */
 491
 492         /* check for pre-chained descriptors */
 493         iop_paranoia(iop_desc_get_next_desc(sw_desc));
 494
 495         /* increment the pending count by the number of slots
 496          * memcpy operations have a 1:1 (slot:operation) relation
 497          * other operations are heavier and will pop the threshold
 498          * more often.
 499          */
 500         iop_chan->pending += slot_cnt;
 501         iop_adma_check_threshold(iop_chan);
 502         spin_unlock_bh(&iop_chan->lock);
 503
 504         dev_dbg(iop_chan->device->common.dev, "%s cookie: %d slot: %d\n",
 505                 __func__, sw_desc->async_tx.cookie, sw_desc->idx);
 506
 507         return cookie;
 508 }
 509
 510 static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan);
 511 static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan);
 512
 513 /**
 514  * iop_adma_alloc_chan_resources -  returns the number of allocated descriptors
 515  * @chan - allocate descriptor resources for this channel
 516  * @client - current client requesting the channel be ready for requests
 517  *
 518  * Note: We keep the slots for 1 operation on iop_chan->chain at all times.  To
 519  * avoid deadlock, via async_xor, num_descs_in_pool must at a minimum be
 520  * greater than 2x the number slots needed to satisfy a device->max_xor
 521  * request.
 522  * */
 523 static int iop_adma_alloc_chan_resources(struct dma_chan *chan)
 524 {
 525         char *hw_desc;
 526         int idx;
 527         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 528         struct iop_adma_desc_slot *slot = NULL;
 529         int init = iop_chan->slots_allocated ? 0 : 1;
 530         struct iop_adma_platform_data *plat_data =
 531                 iop_chan->device->pdev->dev.platform_data;
 532         int num_descs_in_pool = plat_data->pool_size/IOP_ADMA_SLOT_SIZE;
 533
 534         /* Allocate descriptor slots */
 535         do {
 536                 idx = iop_chan->slots_allocated;
 537                 if (idx == num_descs_in_pool)
 538                         break;
 539
 540                 slot = kzalloc(sizeof(*slot), GFP_KERNEL);
 541                 if (!slot) {
 542                         printk(KERN_INFO "IOP ADMA Channel only initialized"
 543                                 " %d descriptor slots", idx);
 544                         break;
 545                 }
 546                 hw_desc = (char *) iop_chan->device->dma_desc_pool_virt;
 547                 slot->hw_desc = (void *) &hw_desc[idx * IOP_ADMA_SLOT_SIZE];
 548
 549                 dma_async_tx_descriptor_init(&slot->async_tx, chan);
 550                 slot->async_tx.tx_submit = iop_adma_tx_submit;
 551                 INIT_LIST_HEAD(&slot->tx_list);
 552                 INIT_LIST_HEAD(&slot->chain_node);
 553                 INIT_LIST_HEAD(&slot->slot_node);
 554                 hw_desc = (char *) iop_chan->device->dma_desc_pool;
 555                 slot->async_tx.phys =
 556                         (dma_addr_t) &hw_desc[idx * IOP_ADMA_SLOT_SIZE];
 557                 slot->idx = idx;
 558
 559                 spin_lock_bh(&iop_chan->lock);
 560                 iop_chan->slots_allocated++;
 561                 list_add_tail(&slot->slot_node, &iop_chan->all_slots);
 562                 spin_unlock_bh(&iop_chan->lock);
 563         } while (iop_chan->slots_allocated < num_descs_in_pool);
 564
 565         if (idx && !iop_chan->last_used)
 566                 iop_chan->last_used = list_entry(iop_chan->all_slots.next,
 567                                         struct iop_adma_desc_slot,
 568                                         slot_node);
 569
 570         dev_dbg(iop_chan->device->common.dev,
 571                 "allocated %d descriptor slots last_used: %p\n",
 572                 iop_chan->slots_allocated, iop_chan->last_used);
 573
 574         /* initialize the channel and the chain with a null operation */
 575         if (init) {
 576                 if (dma_has_cap(DMA_MEMCPY,
 577                         iop_chan->device->common.cap_mask))
 578                         iop_chan_start_null_memcpy(iop_chan);
 579                 else if (dma_has_cap(DMA_XOR,
 580                         iop_chan->device->common.cap_mask))
 581                         iop_chan_start_null_xor(iop_chan);
 582                 else
 583                         BUG();
 584         }
 585
 586         return (idx > 0) ? idx : -ENOMEM;
 587 }
 588
 589 static struct dma_async_tx_descriptor *
 590 iop_adma_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
 591 {
 592         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 593         struct iop_adma_desc_slot *sw_desc, *grp_start;
 594         int slot_cnt, slots_per_op;
 595
 596         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
 597
 598         spin_lock_bh(&iop_chan->lock);
 599         slot_cnt = iop_chan_interrupt_slot_count(&slots_per_op, iop_chan);
 600         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 601         if (sw_desc) {
 602                 grp_start = sw_desc->group_head;
 603                 iop_desc_init_interrupt(grp_start, iop_chan);
 604                 grp_start->unmap_len = 0;
 605                 sw_desc->async_tx.flags = flags;
 606         }
 607         spin_unlock_bh(&iop_chan->lock);
 608
 609         return sw_desc ? &sw_desc->async_tx : NULL;
 610 }
 611
 612 static struct dma_async_tx_descriptor *
 613 iop_adma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
 614                          dma_addr_t dma_src, size_t len, unsigned long flags)
 615 {
 616         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 617         struct iop_adma_desc_slot *sw_desc, *grp_start;
 618         int slot_cnt, slots_per_op;
 619
 620         if (unlikely(!len))
 621                 return NULL;
 622         BUG_ON(len > IOP_ADMA_MAX_BYTE_COUNT);
 623
 624         dev_dbg(iop_chan->device->common.dev, "%s len: %u\n",
 625                 __func__, len);
 626
 627         spin_lock_bh(&iop_chan->lock);
 628         slot_cnt = iop_chan_memcpy_slot_count(len, &slots_per_op);
 629         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 630         if (sw_desc) {
 631                 grp_start = sw_desc->group_head;
 632                 iop_desc_init_memcpy(grp_start, flags);
 633                 iop_desc_set_byte_count(grp_start, iop_chan, len);
 634                 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
 635                 iop_desc_set_memcpy_src_addr(grp_start, dma_src);
 636                 sw_desc->unmap_src_cnt = 1;
 637                 sw_desc->unmap_len = len;
 638                 sw_desc->async_tx.flags = flags;
 639         }
 640         spin_unlock_bh(&iop_chan->lock);
 641
 642         return sw_desc ? &sw_desc->async_tx : NULL;
 643 }
 644
 645 static struct dma_async_tx_descriptor *
 646 iop_adma_prep_dma_memset(struct dma_chan *chan, dma_addr_t dma_dest,
 647                          int value, size_t len, unsigned long flags)
 648 {
 649         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 650         struct iop_adma_desc_slot *sw_desc, *grp_start;
 651         int slot_cnt, slots_per_op;
 652
 653         if (unlikely(!len))
 654                 return NULL;
 655         BUG_ON(len > IOP_ADMA_MAX_BYTE_COUNT);
 656
 657         dev_dbg(iop_chan->device->common.dev, "%s len: %u\n",
 658                 __func__, len);
 659
 660         spin_lock_bh(&iop_chan->lock);
 661         slot_cnt = iop_chan_memset_slot_count(len, &slots_per_op);
 662         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 663         if (sw_desc) {
 664                 grp_start = sw_desc->group_head;
 665                 iop_desc_init_memset(grp_start, flags);
 666                 iop_desc_set_byte_count(grp_start, iop_chan, len);
 667                 iop_desc_set_block_fill_val(grp_start, value);
 668                 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
 669                 sw_desc->unmap_src_cnt = 1;
 670                 sw_desc->unmap_len = len;
 671                 sw_desc->async_tx.flags = flags;
 672         }
 673         spin_unlock_bh(&iop_chan->lock);
 674
 675         return sw_desc ? &sw_desc->async_tx : NULL;
 676 }
 677
 678 static struct dma_async_tx_descriptor *
 679 iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest,
 680                       dma_addr_t *dma_src, unsigned int src_cnt, size_t len,
 681                       unsigned long flags)
 682 {
 683         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 684         struct iop_adma_desc_slot *sw_desc, *grp_start;
 685         int slot_cnt, slots_per_op;
 686
 687         if (unlikely(!len))
 688                 return NULL;
 689         BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
 690
 691         dev_dbg(iop_chan->device->common.dev,
 692                 "%s src_cnt: %d len: %u flags: %lx\n",
 693                 __func__, src_cnt, len, flags);
 694
 695         spin_lock_bh(&iop_chan->lock);
 696         slot_cnt = iop_chan_xor_slot_count(len, src_cnt, &slots_per_op);
 697         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 698         if (sw_desc) {
 699                 grp_start = sw_desc->group_head;
 700                 iop_desc_init_xor(grp_start, src_cnt, flags);
 701                 iop_desc_set_byte_count(grp_start, iop_chan, len);
 702                 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
 703                 sw_desc->unmap_src_cnt = src_cnt;
 704                 sw_desc->unmap_len = len;
 705                 sw_desc->async_tx.flags = flags;
 706                 while (src_cnt--)
 707                         iop_desc_set_xor_src_addr(grp_start, src_cnt,
 708                                                   dma_src[src_cnt]);
 709         }
 710         spin_unlock_bh(&iop_chan->lock);
 711
 712         return sw_desc ? &sw_desc->async_tx : NULL;
 713 }
 714
 715 static struct dma_async_tx_descriptor *
 716 iop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src,
 717                           unsigned int src_cnt, size_t len, u32 *result,
 718                           unsigned long flags)
 719 {
 720         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 721         struct iop_adma_desc_slot *sw_desc, *grp_start;
 722         int slot_cnt, slots_per_op;
 723
 724         if (unlikely(!len))
 725                 return NULL;
 726
 727         dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
 728                 __func__, src_cnt, len);
 729
 730         spin_lock_bh(&iop_chan->lock);
 731         slot_cnt = iop_chan_zero_sum_slot_count(len, src_cnt, &slots_per_op);
 732         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 733         if (sw_desc) {
 734                 grp_start = sw_desc->group_head;
 735                 iop_desc_init_zero_sum(grp_start, src_cnt, flags);
 736                 iop_desc_set_zero_sum_byte_count(grp_start, len);
 737                 grp_start->xor_check_result = result;
 738                 pr_debug("\t%s: grp_start->xor_check_result: %p\n",
 739                         __func__, grp_start->xor_check_result);
 740                 sw_desc->unmap_src_cnt = src_cnt;
 741                 sw_desc->unmap_len = len;
 742                 sw_desc->async_tx.flags = flags;
 743                 while (src_cnt--)
 744                         iop_desc_set_zero_sum_src_addr(grp_start, src_cnt,
 745                                                        dma_src[src_cnt]);
 746         }
 747         spin_unlock_bh(&iop_chan->lock);
 748
 749         return sw_desc ? &sw_desc->async_tx : NULL;
 750 }
 751
 752 static struct dma_async_tx_descriptor *
 753 iop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
 754                      unsigned int src_cnt, const unsigned char *scf, size_t len,
 755                      unsigned long flags)
 756 {
 757         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 758         struct iop_adma_desc_slot *sw_desc, *g;
 759         int slot_cnt, slots_per_op;
 760         int continue_srcs;
 761
 762         if (unlikely(!len))
 763                 return NULL;
 764         BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
 765
 766         dev_dbg(iop_chan->device->common.dev,
 767                 "%s src_cnt: %d len: %u flags: %lx\n",
 768                 __func__, src_cnt, len, flags);
 769
 770         if (dmaf_p_disabled_continue(flags))
 771                 continue_srcs = 1+src_cnt;
 772         else if (dmaf_continue(flags))
 773                 continue_srcs = 3+src_cnt;
 774         else
 775                 continue_srcs = 0+src_cnt;
 776
 777         spin_lock_bh(&iop_chan->lock);
 778         slot_cnt = iop_chan_pq_slot_count(len, continue_srcs, &slots_per_op);
 779         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 780         if (sw_desc) {
 781                 int i;
 782
 783                 g = sw_desc->group_head;
 784                 iop_desc_set_byte_count(g, iop_chan, len);
 785
 786                 /* even if P is disabled its destination address (bits
 787                  * [3:0]) must match Q.  It is ok if P points to an
 788                  * invalid address, it won't be written.
 789                  */
 790                 if (flags & DMA_PREP_PQ_DISABLE_P)
 791                         dst[0] = dst[1] & 0x7;
 792
 793                 iop_desc_set_pq_addr(g, dst);
 794                 sw_desc->unmap_src_cnt = src_cnt;
 795                 sw_desc->unmap_len = len;
 796                 sw_desc->async_tx.flags = flags;
 797                 for (i = 0; i < src_cnt; i++)
 798                         iop_desc_set_pq_src_addr(g, i, src[i], scf[i]);
 799
 800                 /* if we are continuing a previous operation factor in
 801                  * the old p and q values, see the comment for dma_maxpq
 802                  * in include/linux/dmaengine.h
 803                  */
 804                 if (dmaf_p_disabled_continue(flags))
 805                         iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
 806                 else if (dmaf_continue(flags)) {
 807                         iop_desc_set_pq_src_addr(g, i++, dst[0], 0);
 808                         iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
 809                         iop_desc_set_pq_src_addr(g, i++, dst[1], 0);
 810                 }
 811                 iop_desc_init_pq(g, i, flags);
 812         }
 813         spin_unlock_bh(&iop_chan->lock);
 814
 815         return sw_desc ? &sw_desc->async_tx : NULL;
 816 }
 817
 818 static struct dma_async_tx_descriptor *
 819 iop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
 820                          unsigned int src_cnt, const unsigned char *scf,
 821                          size_t len, enum sum_check_flags *pqres,
 822                          unsigned long flags)
 823 {
 824         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 825         struct iop_adma_desc_slot *sw_desc, *g;
 826         int slot_cnt, slots_per_op;
 827
 828         if (unlikely(!len))
 829                 return NULL;
 830         BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
 831
 832         dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
 833                 __func__, src_cnt, len);
 834
 835         spin_lock_bh(&iop_chan->lock);
 836         slot_cnt = iop_chan_pq_zero_sum_slot_count(len, src_cnt + 2, &slots_per_op);
 837         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 838         if (sw_desc) {
 839                 /* for validate operations p and q are tagged onto the
 840                  * end of the source list
 841                  */
 842                 int pq_idx = src_cnt;
 843
 844                 g = sw_desc->group_head;
 845                 iop_desc_init_pq_zero_sum(g, src_cnt+2, flags);
 846                 iop_desc_set_pq_zero_sum_byte_count(g, len);
 847                 g->pq_check_result = pqres;
 848                 pr_debug("\t%s: g->pq_check_result: %p\n",
 849                         __func__, g->pq_check_result);
 850                 sw_desc->unmap_src_cnt = src_cnt+2;
 851                 sw_desc->unmap_len = len;
 852                 sw_desc->async_tx.flags = flags;
 853                 while (src_cnt--)
 854                         iop_desc_set_pq_zero_sum_src_addr(g, src_cnt,
 855                                                           src[src_cnt],
 856                                                           scf[src_cnt]);
 857                 iop_desc_set_pq_zero_sum_addr(g, pq_idx, src);
 858         }
 859         spin_unlock_bh(&iop_chan->lock);
 860
 861         return sw_desc ? &sw_desc->async_tx : NULL;
 862 }
 863
 864 static void iop_adma_free_chan_resources(struct dma_chan *chan)
 865 {
 866         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 867         struct iop_adma_desc_slot *iter, *_iter;
 868         int in_use_descs = 0;
 869
 870         iop_adma_slot_cleanup(iop_chan);
 871
 872         spin_lock_bh(&iop_chan->lock);
 873         list_for_each_entry_safe(iter, _iter, &iop_chan->chain,
 874                                         chain_node) {
 875                 in_use_descs++;
 876                 list_del(&iter->chain_node);
 877         }
 878         list_for_each_entry_safe_reverse(
 879                 iter, _iter, &iop_chan->all_slots, slot_node) {
 880                 list_del(&iter->slot_node);
 881                 kfree(iter);
 882                 iop_chan->slots_allocated--;
 883         }
 884         iop_chan->last_used = NULL;
 885
 886         dev_dbg(iop_chan->device->common.dev, "%s slots_allocated %d\n",
 887                 __func__, iop_chan->slots_allocated);
 888         spin_unlock_bh(&iop_chan->lock);
 889
 890         /* one is ok since we left it on there on purpose */
 891         if (in_use_descs > 1)
 892                 printk(KERN_ERR "IOP: Freeing %d in use descriptors!\n",
 893                         in_use_descs - 1);
 894 }
 895
 896 /**
 897  * iop_adma_status - poll the status of an ADMA transaction
 898  * @chan: ADMA channel handle
 899  * @cookie: ADMA transaction identifier
 900  * @txstate: a holder for the current state of the channel or NULL
 901  */
 902 static enum dma_status iop_adma_status(struct dma_chan *chan,
 903                                         dma_cookie_t cookie,
 904                                         struct dma_tx_state *txstate)
 905 {
 906         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 907         dma_cookie_t last_used;
 908         dma_cookie_t last_complete;
 909         enum dma_status ret;
 910
 911         last_used = chan->cookie;
 912         last_complete = iop_chan->completed_cookie;
 913         dma_set_tx_state(txstate, last_complete, last_used, 0);
 914         ret = dma_async_is_complete(cookie, last_complete, last_used);
 915         if (ret == DMA_SUCCESS)
 916                 return ret;
 917
 918         iop_adma_slot_cleanup(iop_chan);
 919
 920         last_used = chan->cookie;
 921         last_complete = iop_chan->completed_cookie;
 922         dma_set_tx_state(txstate, last_complete, last_used, 0);
 923
 924         return dma_async_is_complete(cookie, last_complete, last_used);
 925 }
 926
 927 static irqreturn_t iop_adma_eot_handler(int irq, void *data)
 928 {
 929         struct iop_adma_chan *chan = data;
 930
 931         dev_dbg(chan->device->common.dev, "%s\n", __func__);
 932
 933         tasklet_schedule(&chan->irq_tasklet);
 934
 935         iop_adma_device_clear_eot_status(chan);
 936
 937         return IRQ_HANDLED;
 938 }
 939
 940 static irqreturn_t iop_adma_eoc_handler(int irq, void *data)
 941 {
 942         struct iop_adma_chan *chan = data;
 943
 944         dev_dbg(chan->device->common.dev, "%s\n", __func__);
 945
 946         tasklet_schedule(&chan->irq_tasklet);
 947
 948         iop_adma_device_clear_eoc_status(chan);
 949
 950         return IRQ_HANDLED;
 951 }
 952
 953 static irqreturn_t iop_adma_err_handler(int irq, void *data)
 954 {
 955         struct iop_adma_chan *chan = data;
 956         unsigned long status = iop_chan_get_status(chan);
 957
 958         dev_printk(KERN_ERR, chan->device->common.dev,
 959                 "error ( %s%s%s%s%s%s%s)\n",
 960                 iop_is_err_int_parity(status, chan) ? "int_parity " : "",
 961                 iop_is_err_mcu_abort(status, chan) ? "mcu_abort " : "",
 962                 iop_is_err_int_tabort(status, chan) ? "int_tabort " : "",
 963                 iop_is_err_int_mabort(status, chan) ? "int_mabort " : "",
 964                 iop_is_err_pci_tabort(status, chan) ? "pci_tabort " : "",
 965                 iop_is_err_pci_mabort(status, chan) ? "pci_mabort " : "",
 966                 iop_is_err_split_tx(status, chan) ? "split_tx " : "");
 967
 968         iop_adma_device_clear_err_status(chan);
 969
 970         BUG();
 971
 972         return IRQ_HANDLED;
 973 }
 974
 975 static void iop_adma_issue_pending(struct dma_chan *chan)
 976 {
 977         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 978
 979         if (iop_chan->pending) {
 980                 iop_chan->pending = 0;
 981                 iop_chan_append(iop_chan);
 982         }
 983 }
 984
 985 /*
 986  * Perform a transaction to verify the HW works.
 987  */
 988 #define IOP_ADMA_TEST_SIZE 2000
 989
 990 static int __devinit iop_adma_memcpy_self_test(struct iop_adma_device *device)
 991 {
 992         int i;
 993         void *src, *dest;
 994         dma_addr_t src_dma, dest_dma;
 995         struct dma_chan *dma_chan;
 996         dma_cookie_t cookie;
 997         struct dma_async_tx_descriptor *tx;
 998         int err = 0;
 999         struct iop_adma_chan *iop_chan;
1000
1001         dev_dbg(device->common.dev, "%s\n", __func__);
1002
1003         src = kmalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL);
1004         if (!src)
1005                 return -ENOMEM;
1006         dest = kzalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL);
1007         if (!dest) {
1008                 kfree(src);
1009                 return -ENOMEM;
1010         }
1011
1012         /* Fill in src buffer */
1013         for (i = 0; i < IOP_ADMA_TEST_SIZE; i++)
1014                 ((u8 *) src)[i] = (u8)i;
1015
1016         /* Start copy, using first DMA channel */
1017         dma_chan = container_of(device->common.channels.next,
1018                                 struct dma_chan,
1019                                 device_node);
1020         if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
1021                 err = -ENODEV;
1022                 goto out;
1023         }
1024
1025         dest_dma = dma_map_single(dma_chan->device->dev, dest,
1026                                 IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
1027         src_dma = dma_map_single(dma_chan->device->dev, src,
1028                                 IOP_ADMA_TEST_SIZE, DMA_TO_DEVICE);
1029         tx = iop_adma_prep_dma_memcpy(dma_chan, dest_dma, src_dma,
1030                                       IOP_ADMA_TEST_SIZE,
1031                                       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1032
1033         cookie = iop_adma_tx_submit(tx);
1034         iop_adma_issue_pending(dma_chan);
1035         msleep(1);
1036
1037         if (iop_adma_status(dma_chan, cookie, NULL) !=
1038                         DMA_SUCCESS) {
1039                 dev_printk(KERN_ERR, dma_chan->device->dev,
1040                         "Self-test copy timed out, disabling\n");
1041                 err = -ENODEV;
1042                 goto free_resources;
1043         }
1044
1045         iop_chan = to_iop_adma_chan(dma_chan);
1046         dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma,
1047                 IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
1048         if (memcmp(src, dest, IOP_ADMA_TEST_SIZE)) {
1049                 dev_printk(KERN_ERR, dma_chan->device->dev,
1050                         "Self-test copy failed compare, disabling\n");
1051                 err = -ENODEV;
1052                 goto free_resources;
1053         }
1054
1055 free_resources:
1056         iop_adma_free_chan_resources(dma_chan);
1057 out:
1058         kfree(src);
1059         kfree(dest);
1060         return err;
1061 }
1062
1063 #define IOP_ADMA_NUM_SRC_TEST 4 /* must be <= 15 */
1064 static int __devinit
1065 iop_adma_xor_val_self_test(struct iop_adma_device *device)
1066 {
1067         int i, src_idx;
1068         struct page *dest;
1069         struct page *xor_srcs[IOP_ADMA_NUM_SRC_TEST];
1070         struct page *zero_sum_srcs[IOP_ADMA_NUM_SRC_TEST + 1];
1071         dma_addr_t dma_srcs[IOP_ADMA_NUM_SRC_TEST + 1];
1072         dma_addr_t dma_addr, dest_dma;
1073         struct dma_async_tx_descriptor *tx;
1074         struct dma_chan *dma_chan;
1075         dma_cookie_t cookie;
1076         u8 cmp_byte = 0;
1077         u32 cmp_word;
1078         u32 zero_sum_result;
1079         int err = 0;
1080         struct iop_adma_chan *iop_chan;
1081
1082         dev_dbg(device->common.dev, "%s\n", __func__);
1083
1084         for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) {
1085                 xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
1086                 if (!xor_srcs[src_idx]) {
1087                         while (src_idx--)
1088                                 __free_page(xor_srcs[src_idx]);
1089                         return -ENOMEM;
1090                 }
1091         }
1092
1093         dest = alloc_page(GFP_KERNEL);
1094         if (!dest) {
1095                 while (src_idx--)
1096                         __free_page(xor_srcs[src_idx]);
1097                 return -ENOMEM;
1098         }
1099
1100         /* Fill in src buffers */
1101         for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) {
1102                 u8 *ptr = page_address(xor_srcs[src_idx]);
1103                 for (i = 0; i < PAGE_SIZE; i++)
1104                         ptr[i] = (1 << src_idx);
1105         }
1106
1107         for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++)
1108                 cmp_byte ^= (u8) (1 << src_idx);
1109
1110         cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
1111                         (cmp_byte << 8) | cmp_byte;
1112
1113         memset(page_address(dest), 0, PAGE_SIZE);
1114
1115         dma_chan = container_of(device->common.channels.next,
1116                                 struct dma_chan,
1117                                 device_node);
1118         if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
1119                 err = -ENODEV;
1120                 goto out;
1121         }
1122
1123         /* test xor */
1124         dest_dma = dma_map_page(dma_chan->device->dev, dest, 0,
1125                                 PAGE_SIZE, DMA_FROM_DEVICE);
1126         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1127                 dma_srcs[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i],
1128                                            0, PAGE_SIZE, DMA_TO_DEVICE);
1129         tx = iop_adma_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
1130                                    IOP_ADMA_NUM_SRC_TEST, PAGE_SIZE,
1131                                    DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1132
1133         cookie = iop_adma_tx_submit(tx);
1134         iop_adma_issue_pending(dma_chan);
1135         msleep(8);
1136
1137         if (iop_adma_status(dma_chan, cookie, NULL) !=
1138                 DMA_SUCCESS) {
1139                 dev_printk(KERN_ERR, dma_chan->device->dev,
1140                         "Self-test xor timed out, disabling\n");
1141                 err = -ENODEV;
1142                 goto free_resources;
1143         }
1144
1145         iop_chan = to_iop_adma_chan(dma_chan);
1146         dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma,
1147                 PAGE_SIZE, DMA_FROM_DEVICE);
1148         for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
1149                 u32 *ptr = page_address(dest);
1150                 if (ptr[i] != cmp_word) {
1151                         dev_printk(KERN_ERR, dma_chan->device->dev,
1152                                 "Self-test xor failed compare, disabling\n");
1153                         err = -ENODEV;
1154                         goto free_resources;
1155                 }
1156         }
1157         dma_sync_single_for_device(&iop_chan->device->pdev->dev, dest_dma,
1158                 PAGE_SIZE, DMA_TO_DEVICE);
1159
1160         /* skip zero sum if the capability is not present */
1161         if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask))
1162                 goto free_resources;
1163
1164         /* zero sum the sources with the destintation page */
1165         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1166                 zero_sum_srcs[i] = xor_srcs[i];
1167         zero_sum_srcs[i] = dest;
1168
1169         zero_sum_result = 1;
1170
1171         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++)
1172                 dma_srcs[i] = dma_map_page(dma_chan->device->dev,
1173                                            zero_sum_srcs[i], 0, PAGE_SIZE,
1174                                            DMA_TO_DEVICE);
1175         tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
1176                                        IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
1177                                        &zero_sum_result,
1178                                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1179
1180         cookie = iop_adma_tx_submit(tx);
1181         iop_adma_issue_pending(dma_chan);
1182         msleep(8);
1183
1184         if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) {
1185                 dev_printk(KERN_ERR, dma_chan->device->dev,
1186                         "Self-test zero sum timed out, disabling\n");
1187                 err = -ENODEV;
1188                 goto free_resources;
1189         }
1190
1191         if (zero_sum_result != 0) {
1192                 dev_printk(KERN_ERR, dma_chan->device->dev,
1193                         "Self-test zero sum failed compare, disabling\n");
1194                 err = -ENODEV;
1195                 goto free_resources;
1196         }
1197
1198         /* test memset */
1199         dma_addr = dma_map_page(dma_chan->device->dev, dest, 0,
1200                         PAGE_SIZE, DMA_FROM_DEVICE);
1201         tx = iop_adma_prep_dma_memset(dma_chan, dma_addr, 0, PAGE_SIZE,
1202                                       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1203
1204         cookie = iop_adma_tx_submit(tx);
1205         iop_adma_issue_pending(dma_chan);
1206         msleep(8);
1207
1208         if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) {
1209                 dev_printk(KERN_ERR, dma_chan->device->dev,
1210                         "Self-test memset timed out, disabling\n");
1211                 err = -ENODEV;
1212                 goto free_resources;
1213         }
1214
1215         for (i = 0; i < PAGE_SIZE/sizeof(u32); i++) {
1216                 u32 *ptr = page_address(dest);
1217                 if (ptr[i]) {
1218                         dev_printk(KERN_ERR, dma_chan->device->dev,
1219                                 "Self-test memset failed compare, disabling\n");
1220                         err = -ENODEV;
1221                         goto free_resources;
1222                 }
1223         }
1224
1225         /* test for non-zero parity sum */
1226         zero_sum_result = 0;
1227         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++)
1228                 dma_srcs[i] = dma_map_page(dma_chan->device->dev,
1229                                            zero_sum_srcs[i], 0, PAGE_SIZE,
1230                                            DMA_TO_DEVICE);
1231         tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
1232                                        IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
1233                                        &zero_sum_result,
1234                                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1235
1236         cookie = iop_adma_tx_submit(tx);
1237         iop_adma_issue_pending(dma_chan);
1238         msleep(8);
1239
1240         if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) {
1241                 dev_printk(KERN_ERR, dma_chan->device->dev,
1242                         "Self-test non-zero sum timed out, disabling\n");
1243                 err = -ENODEV;
1244                 goto free_resources;
1245         }
1246
1247         if (zero_sum_result != 1) {
1248                 dev_printk(KERN_ERR, dma_chan->device->dev,
1249                         "Self-test non-zero sum failed compare, disabling\n");
1250                 err = -ENODEV;
1251                 goto free_resources;
1252         }
1253
1254 free_resources:
1255         iop_adma_free_chan_resources(dma_chan);
1256 out:
1257         src_idx = IOP_ADMA_NUM_SRC_TEST;
1258         while (src_idx--)
1259                 __free_page(xor_srcs[src_idx]);
1260         __free_page(dest);
1261         return err;
1262 }
1263
1264 #ifdef CONFIG_RAID6_PQ
1265 static int __devinit
1266 iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device)
1267 {
1268         /* combined sources, software pq results, and extra hw pq results */
1269         struct page *pq[IOP_ADMA_NUM_SRC_TEST+2+2];
1270         /* ptr to the extra hw pq buffers defined above */
1271         struct page **pq_hw = &pq[IOP_ADMA_NUM_SRC_TEST+2];
1272         /* address conversion buffers (dma_map / page_address) */
1273         void *pq_sw[IOP_ADMA_NUM_SRC_TEST+2];
1274         dma_addr_t pq_src[IOP_ADMA_NUM_SRC_TEST];
1275         dma_addr_t pq_dest[2];
1276
1277         int i;
1278         struct dma_async_tx_descriptor *tx;
1279         struct dma_chan *dma_chan;
1280         dma_cookie_t cookie;
1281         u32 zero_sum_result;
1282         int err = 0;
1283         struct device *dev;
1284
1285         dev_dbg(device->common.dev, "%s\n", __func__);
1286
1287         for (i = 0; i < ARRAY_SIZE(pq); i++) {
1288                 pq[i] = alloc_page(GFP_KERNEL);
1289                 if (!pq[i]) {
1290                         while (i--)
1291                                 __free_page(pq[i]);
1292                         return -ENOMEM;
1293                 }
1294         }
1295
1296         /* Fill in src buffers */
1297         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) {
1298                 pq_sw[i] = page_address(pq[i]);
1299                 memset(pq_sw[i], 0x11111111 * (1<<i), PAGE_SIZE);
1300         }
1301         pq_sw[i] = page_address(pq[i]);
1302         pq_sw[i+1] = page_address(pq[i+1]);
1303
1304         dma_chan = container_of(device->common.channels.next,
1305                                 struct dma_chan,
1306                                 device_node);
1307         if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
1308                 err = -ENODEV;
1309                 goto out;
1310         }
1311
1312         dev = dma_chan->device->dev;
1313
1314         /* initialize the dests */
1315         memset(page_address(pq_hw[0]), 0 , PAGE_SIZE);
1316         memset(page_address(pq_hw[1]), 0 , PAGE_SIZE);
1317
1318         /* test pq */
1319         pq_dest[0] = dma_map_page(dev, pq_hw[0], 0, PAGE_SIZE, DMA_FROM_DEVICE);
1320         pq_dest[1] = dma_map_page(dev, pq_hw[1], 0, PAGE_SIZE, DMA_FROM_DEVICE);
1321         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1322                 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1323                                          DMA_TO_DEVICE);
1324
1325         tx = iop_adma_prep_dma_pq(dma_chan, pq_dest, pq_src,
1326                                   IOP_ADMA_NUM_SRC_TEST, (u8 *)raid6_gfexp,
1327                                   PAGE_SIZE,
1328                                   DMA_PREP_INTERRUPT |
1329                                   DMA_CTRL_ACK);
1330
1331         cookie = iop_adma_tx_submit(tx);
1332         iop_adma_issue_pending(dma_chan);
1333         msleep(8);
1334
1335         if (iop_adma_status(dma_chan, cookie, NULL) !=
1336                 DMA_SUCCESS) {
1337                 dev_err(dev, "Self-test pq timed out, disabling\n");
1338                 err = -ENODEV;
1339                 goto free_resources;
1340         }
1341
1342         raid6_call.gen_syndrome(IOP_ADMA_NUM_SRC_TEST+2, PAGE_SIZE, pq_sw);
1343
1344         if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST],
1345                    page_address(pq_hw[0]), PAGE_SIZE) != 0) {
1346                 dev_err(dev, "Self-test p failed compare, disabling\n");
1347                 err = -ENODEV;
1348                 goto free_resources;
1349         }
1350         if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST+1],
1351                    page_address(pq_hw[1]), PAGE_SIZE) != 0) {
1352                 dev_err(dev, "Self-test q failed compare, disabling\n");
1353                 err = -ENODEV;
1354                 goto free_resources;
1355         }
1356
1357         /* test correct zero sum using the software generated pq values */
1358         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
1359                 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1360                                          DMA_TO_DEVICE);
1361
1362         zero_sum_result = ~0;
1363         tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
1364                                       pq_src, IOP_ADMA_NUM_SRC_TEST,
1365                                       raid6_gfexp, PAGE_SIZE, &zero_sum_result,
1366                                       DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
1367
1368         cookie = iop_adma_tx_submit(tx);
1369         iop_adma_issue_pending(dma_chan);
1370         msleep(8);
1371
1372         if (iop_adma_status(dma_chan, cookie, NULL) !=
1373                 DMA_SUCCESS) {
1374                 dev_err(dev, "Self-test pq-zero-sum timed out, disabling\n");
1375                 err = -ENODEV;
1376                 goto free_resources;
1377         }
1378
1379         if (zero_sum_result != 0) {
1380                 dev_err(dev, "Self-test pq-zero-sum failed to validate: %x\n",
1381                         zero_sum_result);
1382                 err = -ENODEV;
1383                 goto free_resources;
1384         }
1385
1386         /* test incorrect zero sum */
1387         i = IOP_ADMA_NUM_SRC_TEST;
1388         memset(pq_sw[i] + 100, 0, 100);
1389         memset(pq_sw[i+1] + 200, 0, 200);
1390         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
1391                 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1392                                          DMA_TO_DEVICE);
1393
1394         zero_sum_result = 0;
1395         tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
1396                                       pq_src, IOP_ADMA_NUM_SRC_TEST,
1397                                       raid6_gfexp, PAGE_SIZE, &zero_sum_result,
1398                                       DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
1399
1400         cookie = iop_adma_tx_submit(tx);
1401         iop_adma_issue_pending(dma_chan);
1402         msleep(8);
1403
1404         if (iop_adma_status(dma_chan, cookie, NULL) !=
1405                 DMA_SUCCESS) {
1406                 dev_err(dev, "Self-test !pq-zero-sum timed out, disabling\n");
1407                 err = -ENODEV;
1408                 goto free_resources;
1409         }
1410
1411         if (zero_sum_result != (SUM_CHECK_P_RESULT | SUM_CHECK_Q_RESULT)) {
1412                 dev_err(dev, "Self-test !pq-zero-sum failed to validate: %x\n",
1413                         zero_sum_result);
1414                 err = -ENODEV;
1415                 goto free_resources;
1416         }
1417
1418 free_resources:
1419         iop_adma_free_chan_resources(dma_chan);
1420 out:
1421         i = ARRAY_SIZE(pq);
1422         while (i--)
1423                 __free_page(pq[i]);
1424         return err;
1425 }
1426 #endif
1427
1428 static int __devexit iop_adma_remove(struct platform_device *dev)
1429 {
1430         struct iop_adma_device *device = platform_get_drvdata(dev);
1431         struct dma_chan *chan, *_chan;
1432         struct iop_adma_chan *iop_chan;
1433         struct iop_adma_platform_data *plat_data = dev->dev.platform_data;
1434
1435         dma_async_device_unregister(&device->common);
1436
1437         dma_free_coherent(&dev->dev, plat_data->pool_size,
1438                         device->dma_desc_pool_virt, device->dma_desc_pool);
1439
1440         list_for_each_entry_safe(chan, _chan, &device->common.channels,
1441                                 device_node) {
1442                 iop_chan = to_iop_adma_chan(chan);
1443                 list_del(&chan->device_node);
1444                 kfree(iop_chan);
1445         }
1446         kfree(device);
1447
1448         return 0;
1449 }
1450
1451 static int __devinit iop_adma_probe(struct platform_device *pdev)
1452 {
1453         struct resource *res;
1454         int ret = 0, i;
1455         struct iop_adma_device *adev;
1456         struct iop_adma_chan *iop_chan;
1457         struct dma_device *dma_dev;
1458         struct iop_adma_platform_data *plat_data = pdev->dev.platform_data;
1459
1460         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1461         if (!res)
1462                 return -ENODEV;
1463
1464         if (!devm_request_mem_region(&pdev->dev, res->start,
1465                                 resource_size(res), pdev->name))
1466                 return -EBUSY;
1467
1468         adev = kzalloc(sizeof(*adev), GFP_KERNEL);
1469         if (!adev)
1470                 return -ENOMEM;
1471         dma_dev = &adev->common;
1472
1473         /* allocate coherent memory for hardware descriptors
1474          * note: writecombine gives slightly better performance, but
1475          * requires that we explicitly flush the writes
1476          */
1477         if ((adev->dma_desc_pool_virt = dma_alloc_writecombine(&pdev->dev,
1478                                         plat_data->pool_size,
1479                                         &adev->dma_desc_pool,
1480                                         GFP_KERNEL)) == NULL) {
1481                 ret = -ENOMEM;
1482                 goto err_free_adev;
1483         }
1484
1485         dev_dbg(&pdev->dev, "%s: allocted descriptor pool virt %p phys %p\n",
1486                 __func__, adev->dma_desc_pool_virt,
1487                 (void *) adev->dma_desc_pool);
1488
1489         adev->id = plat_data->hw_id;
1490
1491         /* discover transaction capabilites from the platform data */
1492         dma_dev->cap_mask = plat_data->cap_mask;
1493
1494         adev->pdev = pdev;
1495         platform_set_drvdata(pdev, adev);
1496
1497         INIT_LIST_HEAD(&dma_dev->channels);
1498
1499         /* set base routines */
1500         dma_dev->device_alloc_chan_resources = iop_adma_alloc_chan_resources;
1501         dma_dev->device_free_chan_resources = iop_adma_free_chan_resources;
1502         dma_dev->device_tx_status = iop_adma_status;
1503         dma_dev->device_issue_pending = iop_adma_issue_pending;
1504         dma_dev->dev = &pdev->dev;
1505
1506         /* set prep routines based on capability */
1507         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1508                 dma_dev->device_prep_dma_memcpy = iop_adma_prep_dma_memcpy;
1509         if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask))
1510                 dma_dev->device_prep_dma_memset = iop_adma_prep_dma_memset;
1511         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1512                 dma_dev->max_xor = iop_adma_get_max_xor();
1513                 dma_dev->device_prep_dma_xor = iop_adma_prep_dma_xor;
1514         }
1515         if (dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask))
1516                 dma_dev->device_prep_dma_xor_val =
1517                         iop_adma_prep_dma_xor_val;
1518         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1519                 dma_set_maxpq(dma_dev, iop_adma_get_max_pq(), 0);
1520                 dma_dev->device_prep_dma_pq = iop_adma_prep_dma_pq;
1521         }
1522         if (dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask))
1523                 dma_dev->device_prep_dma_pq_val =
1524                         iop_adma_prep_dma_pq_val;
1525         if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1526                 dma_dev->device_prep_dma_interrupt =
1527                         iop_adma_prep_dma_interrupt;
1528
1529         iop_chan = kzalloc(sizeof(*iop_chan), GFP_KERNEL);
1530         if (!iop_chan) {
1531                 ret = -ENOMEM;
1532                 goto err_free_dma;
1533         }
1534         iop_chan->device = adev;
1535
1536         iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start,
1537                                         resource_size(res));
1538         if (!iop_chan->mmr_base) {
1539                 ret = -ENOMEM;
1540                 goto err_free_iop_chan;
1541         }
1542         tasklet_init(&iop_chan->irq_tasklet, iop_adma_tasklet, (unsigned long)
1543                 iop_chan);
1544
1545         /* clear errors before enabling interrupts */
1546         iop_adma_device_clear_err_status(iop_chan);
1547
1548         for (i = 0; i < 3; i++) {
1549                 irq_handler_t handler[] = { iop_adma_eot_handler,
1550                                         iop_adma_eoc_handler,
1551                                         iop_adma_err_handler };
1552                 int irq = platform_get_irq(pdev, i);
1553                 if (irq < 0) {
1554                         ret = -ENXIO;
1555                         goto err_free_iop_chan;
1556                 } else {
1557                         ret = devm_request_irq(&pdev->dev, irq,
1558                                         handler[i], 0, pdev->name, iop_chan);
1559                         if (ret)
1560                                 goto err_free_iop_chan;
1561                 }
1562         }
1563
1564         spin_lock_init(&iop_chan->lock);
1565         INIT_LIST_HEAD(&iop_chan->chain);
1566         INIT_LIST_HEAD(&iop_chan->all_slots);
1567         iop_chan->common.device = dma_dev;
1568         list_add_tail(&iop_chan->common.device_node, &dma_dev->channels);
1569
1570         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1571                 ret = iop_adma_memcpy_self_test(adev);
1572                 dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret);
1573                 if (ret)
1574                         goto err_free_iop_chan;
1575         }
1576
1577         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask) ||
1578             dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) {
1579                 ret = iop_adma_xor_val_self_test(adev);
1580                 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
1581                 if (ret)
1582                         goto err_free_iop_chan;
1583         }
1584
1585         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask) &&
1586             dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) {
1587                 #ifdef CONFIG_RAID6_PQ
1588                 ret = iop_adma_pq_zero_sum_self_test(adev);
1589                 dev_dbg(&pdev->dev, "pq self test returned %d\n", ret);
1590                 #else
1591                 /* can not test raid6, so do not publish capability */
1592                 dma_cap_clear(DMA_PQ, dma_dev->cap_mask);
1593                 dma_cap_clear(DMA_PQ_VAL, dma_dev->cap_mask);
1594                 ret = 0;
1595                 #endif
1596                 if (ret)
1597                         goto err_free_iop_chan;
1598         }
1599
1600         dev_printk(KERN_INFO, &pdev->dev, "Intel(R) IOP: "
1601           "( %s%s%s%s%s%s%s)\n",
1602           dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "",
1603           dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask) ? "pq_val " : "",
1604           dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1605           dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask) ? "xor_val " : "",
1606           dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)  ? "fill " : "",
1607           dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
1608           dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
1609
1610         dma_async_device_register(dma_dev);
1611         goto out;
1612
1613  err_free_iop_chan:
1614         kfree(iop_chan);
1615  err_free_dma:
1616         dma_free_coherent(&adev->pdev->dev, plat_data->pool_size,
1617                         adev->dma_desc_pool_virt, adev->dma_desc_pool);
1618  err_free_adev:
1619         kfree(adev);
1620  out:
1621         return ret;
1622 }
1623
1624 static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan)
1625 {
1626         struct iop_adma_desc_slot *sw_desc, *grp_start;
1627         dma_cookie_t cookie;
1628         int slot_cnt, slots_per_op;
1629
1630         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
1631
1632         spin_lock_bh(&iop_chan->lock);
1633         slot_cnt = iop_chan_memcpy_slot_count(0, &slots_per_op);
1634         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
1635         if (sw_desc) {
1636                 grp_start = sw_desc->group_head;
1637
1638                 list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
1639                 async_tx_ack(&sw_desc->async_tx);
1640                 iop_desc_init_memcpy(grp_start, 0);
1641                 iop_desc_set_byte_count(grp_start, iop_chan, 0);
1642                 iop_desc_set_dest_addr(grp_start, iop_chan, 0);
1643                 iop_desc_set_memcpy_src_addr(grp_start, 0);
1644
1645                 cookie = iop_chan->common.cookie;
1646                 cookie++;
1647                 if (cookie <= 1)
1648                         cookie = 2;
1649
1650                 /* initialize the completed cookie to be less than
1651                  * the most recently used cookie
1652                  */
1653                 iop_chan->completed_cookie = cookie - 1;
1654                 iop_chan->common.cookie = sw_desc->async_tx.cookie = cookie;
1655
1656                 /* channel should not be busy */
1657                 BUG_ON(iop_chan_is_busy(iop_chan));
1658
1659                 /* clear any prior error-status bits */
1660                 iop_adma_device_clear_err_status(iop_chan);
1661
1662                 /* disable operation */
1663                 iop_chan_disable(iop_chan);
1664
1665                 /* set the descriptor address */
1666                 iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys);
1667
1668                 /* 1/ don't add pre-chained descriptors
1669                  * 2/ dummy read to flush next_desc write
1670                  */
1671                 BUG_ON(iop_desc_get_next_desc(sw_desc));
1672
1673                 /* run the descriptor */
1674                 iop_chan_enable(iop_chan);
1675         } else
1676                 dev_printk(KERN_ERR, iop_chan->device->common.dev,
1677                          "failed to allocate null descriptor\n");
1678         spin_unlock_bh(&iop_chan->lock);
1679 }
1680
1681 static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan)
1682 {
1683         struct iop_adma_desc_slot *sw_desc, *grp_start;
1684         dma_cookie_t cookie;
1685         int slot_cnt, slots_per_op;
1686
1687         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
1688
1689         spin_lock_bh(&iop_chan->lock);
1690         slot_cnt = iop_chan_xor_slot_count(0, 2, &slots_per_op);
1691         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
1692         if (sw_desc) {
1693                 grp_start = sw_desc->group_head;
1694                 list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
1695                 async_tx_ack(&sw_desc->async_tx);
1696                 iop_desc_init_null_xor(grp_start, 2, 0);
1697                 iop_desc_set_byte_count(grp_start, iop_chan, 0);
1698                 iop_desc_set_dest_addr(grp_start, iop_chan, 0);
1699                 iop_desc_set_xor_src_addr(grp_start, 0, 0);
1700                 iop_desc_set_xor_src_addr(grp_start, 1, 0);
1701
1702                 cookie = iop_chan->common.cookie;
1703                 cookie++;
1704                 if (cookie <= 1)
1705                         cookie = 2;
1706
1707                 /* initialize the completed cookie to be less than
1708                  * the most recently used cookie
1709                  */
1710                 iop_chan->completed_cookie = cookie - 1;
1711                 iop_chan->common.cookie = sw_desc->async_tx.cookie = cookie;
1712
1713                 /* channel should not be busy */
1714                 BUG_ON(iop_chan_is_busy(iop_chan));
1715
1716                 /* clear any prior error-status bits */
1717                 iop_adma_device_clear_err_status(iop_chan);
1718
1719                 /* disable operation */
1720                 iop_chan_disable(iop_chan);
1721
1722                 /* set the descriptor address */
1723                 iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys);
1724
1725                 /* 1/ don't add pre-chained descriptors
1726                  * 2/ dummy read to flush next_desc write
1727                  */
1728                 BUG_ON(iop_desc_get_next_desc(sw_desc));
1729
1730                 /* run the descriptor */
1731                 iop_chan_enable(iop_chan);
1732         } else
1733                 dev_printk(KERN_ERR, iop_chan->device->common.dev,
1734                         "failed to allocate null descriptor\n");
1735         spin_unlock_bh(&iop_chan->lock);
1736 }
1737
1738 MODULE_ALIAS("platform:iop-adma");
1739
1740 static struct platform_driver iop_adma_driver = {
1741         .probe          = iop_adma_probe,
1742         .remove         = __devexit_p(iop_adma_remove),
1743         .driver         = {
1744                 .owner  = THIS_MODULE,
1745                 .name   = "iop-adma",
1746         },
1747 };
1748
1749 static int __init iop_adma_init (void)
1750 {
1751         return platform_driver_register(&iop_adma_driver);
1752 }
1753
1754 static void __exit iop_adma_exit (void)
1755 {
1756         platform_driver_unregister(&iop_adma_driver);
1757         return;
1758 }
1759 module_exit(iop_adma_exit);
1760 module_init(iop_adma_init);
1761
1762 MODULE_AUTHOR("Intel Corporation");
1763 MODULE_DESCRIPTION("IOP ADMA Engine Driver");
1764 MODULE_LICENSE("GPL");