drivers/dma/ppc4xx/adma.c

   1 /*
   2  * Copyright (C) 2006-2009 DENX Software Engineering.
   3  *
   4  * Author: Yuri Tikhonov <yur@emcraft.com>
   5  *
   6  * Further porting to arch/powerpc by
   7  *      Anatolij Gustschin <agust@denx.de>
   8  *
   9  * This program is free software; you can redistribute it and/or modify it
  10  * under the terms of the GNU General Public License as published by the Free
  11  * Software Foundation; either version 2 of the License, or (at your option)
  12  * any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful, but WITHOUT
  15  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  16  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  17  * more details.
  18  *
  19  * You should have received a copy of the GNU General Public License along with
  20  * this program; if not, write to the Free Software Foundation, Inc., 59
  21  * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  22  *
  23  * The full GNU General Public License is included in this distribution in the
  24  * file called COPYING.
  25  */
  26
  27 /*
  28  * This driver supports the asynchrounous DMA copy and RAID engines available
  29  * on the AMCC PPC440SPe Processors.
  30  * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
  31  * ADMA driver written by D.Williams.
  32  */
  33
  34 #include <linux/init.h>
  35 #include <linux/module.h>
  36 #include <linux/async_tx.h>
  37 #include <linux/delay.h>
  38 #include <linux/dma-mapping.h>
  39 #include <linux/spinlock.h>
  40 #include <linux/interrupt.h>
  41 #include <linux/slab.h>
  42 #include <linux/uaccess.h>
  43 #include <linux/proc_fs.h>
  44 #include <linux/of.h>
  45 #include <linux/of_address.h>
  46 #include <linux/of_irq.h>
  47 #include <linux/of_platform.h>
  48 #include <asm/dcr.h>
  49 #include <asm/dcr-regs.h>
  50 #include "adma.h"
  51 #include "../dmaengine.h"
  52
  53 enum ppc_adma_init_code {
  54         PPC_ADMA_INIT_OK = 0,
  55         PPC_ADMA_INIT_MEMRES,
  56         PPC_ADMA_INIT_MEMREG,
  57         PPC_ADMA_INIT_ALLOC,
  58         PPC_ADMA_INIT_COHERENT,
  59         PPC_ADMA_INIT_CHANNEL,
  60         PPC_ADMA_INIT_IRQ1,
  61         PPC_ADMA_INIT_IRQ2,
  62         PPC_ADMA_INIT_REGISTER
  63 };
  64
  65 static char *ppc_adma_errors[] = {
  66         [PPC_ADMA_INIT_OK] = "ok",
  67         [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource",
  68         [PPC_ADMA_INIT_MEMREG] = "failed to request memory region",
  69         [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev "
  70                                 "structure",
  71         [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for "
  72                                    "hardware descriptors",
  73         [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel",
  74         [PPC_ADMA_INIT_IRQ1] = "failed to request first irq",
  75         [PPC_ADMA_INIT_IRQ2] = "failed to request second irq",
  76         [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device",
  77 };
  78
  79 static enum ppc_adma_init_code
  80 ppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM];
  81
  82 struct ppc_dma_chan_ref {
  83         struct dma_chan *chan;
  84         struct list_head node;
  85 };
  86
  87 /* The list of channels exported by ppc440spe ADMA */
  88 struct list_head
  89 ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list);
  90
  91 /* This flag is set when want to refetch the xor chain in the interrupt
  92  * handler
  93  */
  94 static u32 do_xor_refetch;
  95
  96 /* Pointer to DMA0, DMA1 CP/CS FIFO */
  97 static void *ppc440spe_dma_fifo_buf;
  98
  99 /* Pointers to last submitted to DMA0, DMA1 CDBs */
 100 static struct ppc440spe_adma_desc_slot *chan_last_sub[3];
 101 static struct ppc440spe_adma_desc_slot *chan_first_cdb[3];
 102
 103 /* Pointer to last linked and submitted xor CB */
 104 static struct ppc440spe_adma_desc_slot *xor_last_linked;
 105 static struct ppc440spe_adma_desc_slot *xor_last_submit;
 106
 107 /* This array is used in data-check operations for storing a pattern */
 108 static char ppc440spe_qword[16];
 109
 110 static atomic_t ppc440spe_adma_err_irq_ref;
 111 static dcr_host_t ppc440spe_mq_dcr_host;
 112 static unsigned int ppc440spe_mq_dcr_len;
 113
 114 /* Since RXOR operations use the common register (MQ0_CF2H) for setting-up
 115  * the block size in transactions, then we do not allow to activate more than
 116  * only one RXOR transactions simultaneously. So use this var to store
 117  * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is
 118  * set) or not (PPC440SPE_RXOR_RUN is clear).
 119  */
 120 static unsigned long ppc440spe_rxor_state;
 121
 122 /* These are used in enable & check routines
 123  */
 124 static u32 ppc440spe_r6_enabled;
 125 static struct ppc440spe_adma_chan *ppc440spe_r6_tchan;
 126 static struct completion ppc440spe_r6_test_comp;
 127
 128 static int ppc440spe_adma_dma2rxor_prep_src(
 129                 struct ppc440spe_adma_desc_slot *desc,
 130                 struct ppc440spe_rxor *cursor, int index,
 131                 int src_cnt, u32 addr);
 132 static void ppc440spe_adma_dma2rxor_set_src(
 133                 struct ppc440spe_adma_desc_slot *desc,
 134                 int index, dma_addr_t addr);
 135 static void ppc440spe_adma_dma2rxor_set_mult(
 136                 struct ppc440spe_adma_desc_slot *desc,
 137                 int index, u8 mult);
 138
 139 #ifdef ADMA_LL_DEBUG
 140 #define ADMA_LL_DBG(x) ({ if (1) x; 0; })
 141 #else
 142 #define ADMA_LL_DBG(x) ({ if (0) x; 0; })
 143 #endif
 144
 145 static void print_cb(struct ppc440spe_adma_chan *chan, void *block)
 146 {
 147         struct dma_cdb *cdb;
 148         struct xor_cb *cb;
 149         int i;
 150
 151         switch (chan->device->id) {
 152         case 0:
 153         case 1:
 154                 cdb = block;
 155
 156                 pr_debug("CDB at %p [%d]:\n"
 157                         "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n"
 158                         "\t sg1u 0x%08x sg1l 0x%08x\n"
 159                         "\t sg2u 0x%08x sg2l 0x%08x\n"
 160                         "\t sg3u 0x%08x sg3l 0x%08x\n",
 161                         cdb, chan->device->id,
 162                         cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt),
 163                         le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l),
 164                         le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l),
 165                         le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l)
 166                 );
 167                 break;
 168         case 2:
 169                 cb = block;
 170
 171                 pr_debug("CB at %p [%d]:\n"
 172                         "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n"
 173                         "\t cbtah 0x%08x cbtal 0x%08x\n"
 174                         "\t cblah 0x%08x cblal 0x%08x\n",
 175                         cb, chan->device->id,
 176                         cb->cbc, cb->cbbc, cb->cbs,
 177                         cb->cbtah, cb->cbtal,
 178                         cb->cblah, cb->cblal);
 179                 for (i = 0; i < 16; i++) {
 180                         if (i && !cb->ops[i].h && !cb->ops[i].l)
 181                                 continue;
 182                         pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n",
 183                                 i, cb->ops[i].h, cb->ops[i].l);
 184                 }
 185                 break;
 186         }
 187 }
 188
 189 static void print_cb_list(struct ppc440spe_adma_chan *chan,
 190                           struct ppc440spe_adma_desc_slot *iter)
 191 {
 192         for (; iter; iter = iter->hw_next)
 193                 print_cb(chan, iter->hw_desc);
 194 }
 195
 196 static void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src,
 197                              unsigned int src_cnt)
 198 {
 199         int i;
 200
 201         pr_debug("\n%s(%d):\nsrc: ", __func__, id);
 202         for (i = 0; i < src_cnt; i++)
 203                 pr_debug("\t0x%016llx ", src[i]);
 204         pr_debug("dst:\n\t0x%016llx\n", dst);
 205 }
 206
 207 static void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src,
 208                             unsigned int src_cnt)
 209 {
 210         int i;
 211
 212         pr_debug("\n%s(%d):\nsrc: ", __func__, id);
 213         for (i = 0; i < src_cnt; i++)
 214                 pr_debug("\t0x%016llx ", src[i]);
 215         pr_debug("dst: ");
 216         for (i = 0; i < 2; i++)
 217                 pr_debug("\t0x%016llx ", dst[i]);
 218 }
 219
 220 static void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src,
 221                                     unsigned int src_cnt,
 222                                     const unsigned char *scf)
 223 {
 224         int i;
 225
 226         pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id);
 227         if (scf) {
 228                 for (i = 0; i < src_cnt; i++)
 229                         pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]);
 230         } else {
 231                 for (i = 0; i < src_cnt; i++)
 232                         pr_debug("\t0x%016llx(no) ", src[i]);
 233         }
 234
 235         pr_debug("dst: ");
 236         for (i = 0; i < 2; i++)
 237                 pr_debug("\t0x%016llx ", src[src_cnt + i]);
 238 }
 239
 240 /******************************************************************************
 241  * Command (Descriptor) Blocks low-level routines
 242  ******************************************************************************/
 243 /**
 244  * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT
 245  * pseudo operation
 246  */
 247 static void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc,
 248                                           struct ppc440spe_adma_chan *chan)
 249 {
 250         struct xor_cb *p;
 251
 252         switch (chan->device->id) {
 253         case PPC440SPE_XOR_ID:
 254                 p = desc->hw_desc;
 255                 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 256                 /* NOP with Command Block Complete Enable */
 257                 p->cbc = XOR_CBCR_CBCE_BIT;
 258                 break;
 259         case PPC440SPE_DMA0_ID:
 260         case PPC440SPE_DMA1_ID:
 261                 memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
 262                 /* NOP with interrupt */
 263                 set_bit(PPC440SPE_DESC_INT, &desc->flags);
 264                 break;
 265         default:
 266                 printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id,
 267                                 __func__);
 268                 break;
 269         }
 270 }
 271
 272 /**
 273  * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR
 274  * pseudo operation
 275  */
 276 static void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc)
 277 {
 278         memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 279         desc->hw_next = NULL;
 280         desc->src_cnt = 0;
 281         desc->dst_cnt = 1;
 282 }
 283
 284 /**
 285  * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation
 286  */
 287 static void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc,
 288                                          int src_cnt, unsigned long flags)
 289 {
 290         struct xor_cb *hw_desc = desc->hw_desc;
 291
 292         memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 293         desc->hw_next = NULL;
 294         desc->src_cnt = src_cnt;
 295         desc->dst_cnt = 1;
 296
 297         hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt;
 298         if (flags & DMA_PREP_INTERRUPT)
 299                 /* Enable interrupt on completion */
 300                 hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
 301 }
 302
 303 /**
 304  * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ
 305  * operation in DMA2 controller
 306  */
 307 static void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc,
 308                 int dst_cnt, int src_cnt, unsigned long flags)
 309 {
 310         struct xor_cb *hw_desc = desc->hw_desc;
 311
 312         memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 313         desc->hw_next = NULL;
 314         desc->src_cnt = src_cnt;
 315         desc->dst_cnt = dst_cnt;
 316         memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags));
 317         desc->descs_per_op = 0;
 318
 319         hw_desc->cbc = XOR_CBCR_TGT_BIT;
 320         if (flags & DMA_PREP_INTERRUPT)
 321                 /* Enable interrupt on completion */
 322                 hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
 323 }
 324
 325 #define DMA_CTRL_FLAGS_LAST     DMA_PREP_FENCE
 326 #define DMA_PREP_ZERO_P         (DMA_CTRL_FLAGS_LAST << 1)
 327 #define DMA_PREP_ZERO_Q         (DMA_PREP_ZERO_P << 1)
 328
 329 /**
 330  * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation
 331  * with DMA0/1
 332  */
 333 static void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc,
 334                                 int dst_cnt, int src_cnt, unsigned long flags,
 335                                 unsigned long op)
 336 {
 337         struct dma_cdb *hw_desc;
 338         struct ppc440spe_adma_desc_slot *iter;
 339         u8 dopc;
 340
 341         /* Common initialization of a PQ descriptors chain */
 342         set_bits(op, &desc->flags);
 343         desc->src_cnt = src_cnt;
 344         desc->dst_cnt = dst_cnt;
 345
 346         /* WXOR MULTICAST if both P and Q are being computed
 347          * MV_SG1_SG2 if Q only
 348          */
 349         dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ?
 350                 DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2;
 351
 352         list_for_each_entry(iter, &desc->group_list, chain_node) {
 353                 hw_desc = iter->hw_desc;
 354                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
 355
 356                 if (likely(!list_is_last(&iter->chain_node,
 357                                 &desc->group_list))) {
 358                         /* set 'next' pointer */
 359                         iter->hw_next = list_entry(iter->chain_node.next,
 360                                 struct ppc440spe_adma_desc_slot, chain_node);
 361                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
 362                 } else {
 363                         /* this is the last descriptor.
 364                          * this slot will be pasted from ADMA level
 365                          * each time it wants to configure parameters
 366                          * of the transaction (src, dst, ...)
 367                          */
 368                         iter->hw_next = NULL;
 369                         if (flags & DMA_PREP_INTERRUPT)
 370                                 set_bit(PPC440SPE_DESC_INT, &iter->flags);
 371                         else
 372                                 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
 373                 }
 374         }
 375
 376         /* Set OPS depending on WXOR/RXOR type of operation */
 377         if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) {
 378                 /* This is a WXOR only chain:
 379                  * - first descriptors are for zeroing destinations
 380                  *   if PPC440SPE_ZERO_P/Q set;
 381                  * - descriptors remained are for GF-XOR operations.
 382                  */
 383                 iter = list_first_entry(&desc->group_list,
 384                                         struct ppc440spe_adma_desc_slot,
 385                                         chain_node);
 386
 387                 if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) {
 388                         hw_desc = iter->hw_desc;
 389                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 390                         iter = list_first_entry(&iter->chain_node,
 391                                         struct ppc440spe_adma_desc_slot,
 392                                         chain_node);
 393                 }
 394
 395                 if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) {
 396                         hw_desc = iter->hw_desc;
 397                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 398                         iter = list_first_entry(&iter->chain_node,
 399                                         struct ppc440spe_adma_desc_slot,
 400                                         chain_node);
 401                 }
 402
 403                 list_for_each_entry_from(iter, &desc->group_list, chain_node) {
 404                         hw_desc = iter->hw_desc;
 405                         hw_desc->opc = dopc;
 406                 }
 407         } else {
 408                 /* This is either RXOR-only or mixed RXOR/WXOR */
 409
 410                 /* The first 1 or 2 slots in chain are always RXOR,
 411                  * if need to calculate P & Q, then there are two
 412                  * RXOR slots; if only P or only Q, then there is one
 413                  */
 414                 iter = list_first_entry(&desc->group_list,
 415                                         struct ppc440spe_adma_desc_slot,
 416                                         chain_node);
 417                 hw_desc = iter->hw_desc;
 418                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 419
 420                 if (desc->dst_cnt == DMA_DEST_MAX_NUM) {
 421                         iter = list_first_entry(&iter->chain_node,
 422                                                 struct ppc440spe_adma_desc_slot,
 423                                                 chain_node);
 424                         hw_desc = iter->hw_desc;
 425                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 426                 }
 427
 428                 /* The remaining descs (if any) are WXORs */
 429                 if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) {
 430                         iter = list_first_entry(&iter->chain_node,
 431                                                 struct ppc440spe_adma_desc_slot,
 432                                                 chain_node);
 433                         list_for_each_entry_from(iter, &desc->group_list,
 434                                                 chain_node) {
 435                                 hw_desc = iter->hw_desc;
 436                                 hw_desc->opc = dopc;
 437                         }
 438                 }
 439         }
 440 }
 441
 442 /**
 443  * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor
 444  * for PQ_ZERO_SUM operation
 445  */
 446 static void ppc440spe_desc_init_dma01pqzero_sum(
 447                                 struct ppc440spe_adma_desc_slot *desc,
 448                                 int dst_cnt, int src_cnt)
 449 {
 450         struct dma_cdb *hw_desc;
 451         struct ppc440spe_adma_desc_slot *iter;
 452         int i = 0;
 453         u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST :
 454                                    DMA_CDB_OPC_MV_SG1_SG2;
 455         /*
 456          * Initialize starting from 2nd or 3rd descriptor dependent
 457          * on dst_cnt. First one or two slots are for cloning P
 458          * and/or Q to chan->pdest and/or chan->qdest as we have
 459          * to preserve original P/Q.
 460          */
 461         iter = list_first_entry(&desc->group_list,
 462                                 struct ppc440spe_adma_desc_slot, chain_node);
 463         iter = list_entry(iter->chain_node.next,
 464                           struct ppc440spe_adma_desc_slot, chain_node);
 465
 466         if (dst_cnt > 1) {
 467                 iter = list_entry(iter->chain_node.next,
 468                                   struct ppc440spe_adma_desc_slot, chain_node);
 469         }
 470         /* initialize each source descriptor in chain */
 471         list_for_each_entry_from(iter, &desc->group_list, chain_node) {
 472                 hw_desc = iter->hw_desc;
 473                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
 474                 iter->src_cnt = 0;
 475                 iter->dst_cnt = 0;
 476
 477                 /* This is a ZERO_SUM operation:
 478                  * - <src_cnt> descriptors starting from 2nd or 3rd
 479                  *   descriptor are for GF-XOR operations;
 480                  * - remaining <dst_cnt> descriptors are for checking the result
 481                  */
 482                 if (i++ < src_cnt)
 483                         /* MV_SG1_SG2 if only Q is being verified
 484                          * MULTICAST if both P and Q are being verified
 485                          */
 486                         hw_desc->opc = dopc;
 487                 else
 488                         /* DMA_CDB_OPC_DCHECK128 operation */
 489                         hw_desc->opc = DMA_CDB_OPC_DCHECK128;
 490
 491                 if (likely(!list_is_last(&iter->chain_node,
 492                                          &desc->group_list))) {
 493                         /* set 'next' pointer */
 494                         iter->hw_next = list_entry(iter->chain_node.next,
 495                                                 struct ppc440spe_adma_desc_slot,
 496                                                 chain_node);
 497                 } else {
 498                         /* this is the last descriptor.
 499                          * this slot will be pasted from ADMA level
 500                          * each time it wants to configure parameters
 501                          * of the transaction (src, dst, ...)
 502                          */
 503                         iter->hw_next = NULL;
 504                         /* always enable interrupt generation since we get
 505                          * the status of pqzero from the handler
 506                          */
 507                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
 508                 }
 509         }
 510         desc->src_cnt = src_cnt;
 511         desc->dst_cnt = dst_cnt;
 512 }
 513
 514 /**
 515  * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation
 516  */
 517 static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc,
 518                                         unsigned long flags)
 519 {
 520         struct dma_cdb *hw_desc = desc->hw_desc;
 521
 522         memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
 523         desc->hw_next = NULL;
 524         desc->src_cnt = 1;
 525         desc->dst_cnt = 1;
 526
 527         if (flags & DMA_PREP_INTERRUPT)
 528                 set_bit(PPC440SPE_DESC_INT, &desc->flags);
 529         else
 530                 clear_bit(PPC440SPE_DESC_INT, &desc->flags);
 531
 532         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 533 }
 534
 535 /**
 536  * ppc440spe_desc_init_memset - initialize the descriptor for MEMSET operation
 537  */
 538 static void ppc440spe_desc_init_memset(struct ppc440spe_adma_desc_slot *desc,
 539                                         int value, unsigned long flags)
 540 {
 541         struct dma_cdb *hw_desc = desc->hw_desc;
 542
 543         memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
 544         desc->hw_next = NULL;
 545         desc->src_cnt = 1;
 546         desc->dst_cnt = 1;
 547
 548         if (flags & DMA_PREP_INTERRUPT)
 549                 set_bit(PPC440SPE_DESC_INT, &desc->flags);
 550         else
 551                 clear_bit(PPC440SPE_DESC_INT, &desc->flags);
 552
 553         hw_desc->sg1u = hw_desc->sg1l = cpu_to_le32((u32)value);
 554         hw_desc->sg3u = hw_desc->sg3l = cpu_to_le32((u32)value);
 555         hw_desc->opc = DMA_CDB_OPC_DFILL128;
 556 }
 557
 558 /**
 559  * ppc440spe_desc_set_src_addr - set source address into the descriptor
 560  */
 561 static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc,
 562                                         struct ppc440spe_adma_chan *chan,
 563                                         int src_idx, dma_addr_t addrh,
 564                                         dma_addr_t addrl)
 565 {
 566         struct dma_cdb *dma_hw_desc;
 567         struct xor_cb *xor_hw_desc;
 568         phys_addr_t addr64, tmplow, tmphi;
 569
 570         switch (chan->device->id) {
 571         case PPC440SPE_DMA0_ID:
 572         case PPC440SPE_DMA1_ID:
 573                 if (!addrh) {
 574                         addr64 = addrl;
 575                         tmphi = (addr64 >> 32);
 576                         tmplow = (addr64 & 0xFFFFFFFF);
 577                 } else {
 578                         tmphi = addrh;
 579                         tmplow = addrl;
 580                 }
 581                 dma_hw_desc = desc->hw_desc;
 582                 dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow);
 583                 dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi);
 584                 break;
 585         case PPC440SPE_XOR_ID:
 586                 xor_hw_desc = desc->hw_desc;
 587                 xor_hw_desc->ops[src_idx].l = addrl;
 588                 xor_hw_desc->ops[src_idx].h |= addrh;
 589                 break;
 590         }
 591 }
 592
 593 /**
 594  * ppc440spe_desc_set_src_mult - set source address mult into the descriptor
 595  */
 596 static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc,
 597                         struct ppc440spe_adma_chan *chan, u32 mult_index,
 598                         int sg_index, unsigned char mult_value)
 599 {
 600         struct dma_cdb *dma_hw_desc;
 601         struct xor_cb *xor_hw_desc;
 602         u32 *psgu;
 603
 604         switch (chan->device->id) {
 605         case PPC440SPE_DMA0_ID:
 606         case PPC440SPE_DMA1_ID:
 607                 dma_hw_desc = desc->hw_desc;
 608
 609                 switch (sg_index) {
 610                 /* for RXOR operations set multiplier
 611                  * into source cued address
 612                  */
 613                 case DMA_CDB_SG_SRC:
 614                         psgu = &dma_hw_desc->sg1u;
 615                         break;
 616                 /* for WXOR operations set multiplier
 617                  * into destination cued address(es)
 618                  */
 619                 case DMA_CDB_SG_DST1:
 620                         psgu = &dma_hw_desc->sg2u;
 621                         break;
 622                 case DMA_CDB_SG_DST2:
 623                         psgu = &dma_hw_desc->sg3u;
 624                         break;
 625                 default:
 626                         BUG();
 627                 }
 628
 629                 *psgu |= cpu_to_le32(mult_value << mult_index);
 630                 break;
 631         case PPC440SPE_XOR_ID:
 632                 xor_hw_desc = desc->hw_desc;
 633                 break;
 634         default:
 635                 BUG();
 636         }
 637 }
 638
 639 /**
 640  * ppc440spe_desc_set_dest_addr - set destination address into the descriptor
 641  */
 642 static void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc,
 643                                 struct ppc440spe_adma_chan *chan,
 644                                 dma_addr_t addrh, dma_addr_t addrl,
 645                                 u32 dst_idx)
 646 {
 647         struct dma_cdb *dma_hw_desc;
 648         struct xor_cb *xor_hw_desc;
 649         phys_addr_t addr64, tmphi, tmplow;
 650         u32 *psgu, *psgl;
 651
 652         switch (chan->device->id) {
 653         case PPC440SPE_DMA0_ID:
 654         case PPC440SPE_DMA1_ID:
 655                 if (!addrh) {
 656                         addr64 = addrl;
 657                         tmphi = (addr64 >> 32);
 658                         tmplow = (addr64 & 0xFFFFFFFF);
 659                 } else {
 660                         tmphi = addrh;
 661                         tmplow = addrl;
 662                 }
 663                 dma_hw_desc = desc->hw_desc;
 664
 665                 psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u;
 666                 psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l;
 667
 668                 *psgl = cpu_to_le32((u32)tmplow);
 669                 *psgu |= cpu_to_le32((u32)tmphi);
 670                 break;
 671         case PPC440SPE_XOR_ID:
 672                 xor_hw_desc = desc->hw_desc;
 673                 xor_hw_desc->cbtal = addrl;
 674                 xor_hw_desc->cbtah |= addrh;
 675                 break;
 676         }
 677 }
 678
 679 /**
 680  * ppc440spe_desc_set_byte_count - set number of data bytes involved
 681  * into the operation
 682  */
 683 static void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc,
 684                                 struct ppc440spe_adma_chan *chan,
 685                                 u32 byte_count)
 686 {
 687         struct dma_cdb *dma_hw_desc;
 688         struct xor_cb *xor_hw_desc;
 689
 690         switch (chan->device->id) {
 691         case PPC440SPE_DMA0_ID:
 692         case PPC440SPE_DMA1_ID:
 693                 dma_hw_desc = desc->hw_desc;
 694                 dma_hw_desc->cnt = cpu_to_le32(byte_count);
 695                 break;
 696         case PPC440SPE_XOR_ID:
 697                 xor_hw_desc = desc->hw_desc;
 698                 xor_hw_desc->cbbc = byte_count;
 699                 break;
 700         }
 701 }
 702
 703 /**
 704  * ppc440spe_desc_set_rxor_block_size - set RXOR block size
 705  */
 706 static inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count)
 707 {
 708         /* assume that byte_count is aligned on the 512-boundary;
 709          * thus write it directly to the register (bits 23:31 are
 710          * reserved there).
 711          */
 712         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count);
 713 }
 714
 715 /**
 716  * ppc440spe_desc_set_dcheck - set CHECK pattern
 717  */
 718 static void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc,
 719                                 struct ppc440spe_adma_chan *chan, u8 *qword)
 720 {
 721         struct dma_cdb *dma_hw_desc;
 722
 723         switch (chan->device->id) {
 724         case PPC440SPE_DMA0_ID:
 725         case PPC440SPE_DMA1_ID:
 726                 dma_hw_desc = desc->hw_desc;
 727                 iowrite32(qword[0], &dma_hw_desc->sg3l);
 728                 iowrite32(qword[4], &dma_hw_desc->sg3u);
 729                 iowrite32(qword[8], &dma_hw_desc->sg2l);
 730                 iowrite32(qword[12], &dma_hw_desc->sg2u);
 731                 break;
 732         default:
 733                 BUG();
 734         }
 735 }
 736
 737 /**
 738  * ppc440spe_xor_set_link - set link address in xor CB
 739  */
 740 static void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc,
 741                                 struct ppc440spe_adma_desc_slot *next_desc)
 742 {
 743         struct xor_cb *xor_hw_desc = prev_desc->hw_desc;
 744
 745         if (unlikely(!next_desc || !(next_desc->phys))) {
 746                 printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n",
 747                         __func__, next_desc,
 748                         next_desc ? next_desc->phys : 0);
 749                 BUG();
 750         }
 751
 752         xor_hw_desc->cbs = 0;
 753         xor_hw_desc->cblal = next_desc->phys;
 754         xor_hw_desc->cblah = 0;
 755         xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT;
 756 }
 757
 758 /**
 759  * ppc440spe_desc_set_link - set the address of descriptor following this
 760  * descriptor in chain
 761  */
 762 static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan,
 763                                 struct ppc440spe_adma_desc_slot *prev_desc,
 764                                 struct ppc440spe_adma_desc_slot *next_desc)
 765 {
 766         unsigned long flags;
 767         struct ppc440spe_adma_desc_slot *tail = next_desc;
 768
 769         if (unlikely(!prev_desc || !next_desc ||
 770                 (prev_desc->hw_next && prev_desc->hw_next != next_desc))) {
 771                 /* If previous next is overwritten something is wrong.
 772                  * though we may refetch from append to initiate list
 773                  * processing; in this case - it's ok.
 774                  */
 775                 printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; "
 776                         "prev->hw_next=0x%p\n", __func__, prev_desc,
 777                         next_desc, prev_desc ? prev_desc->hw_next : 0);
 778                 BUG();
 779         }
 780
 781         local_irq_save(flags);
 782
 783         /* do s/w chaining both for DMA and XOR descriptors */
 784         prev_desc->hw_next = next_desc;
 785
 786         switch (chan->device->id) {
 787         case PPC440SPE_DMA0_ID:
 788         case PPC440SPE_DMA1_ID:
 789                 break;
 790         case PPC440SPE_XOR_ID:
 791                 /* bind descriptor to the chain */
 792                 while (tail->hw_next)
 793                         tail = tail->hw_next;
 794                 xor_last_linked = tail;
 795
 796                 if (prev_desc == xor_last_submit)
 797                         /* do not link to the last submitted CB */
 798                         break;
 799                 ppc440spe_xor_set_link(prev_desc, next_desc);
 800                 break;
 801         }
 802
 803         local_irq_restore(flags);
 804 }
 805
 806 /**
 807  * ppc440spe_desc_get_src_addr - extract the source address from the descriptor
 808  */
 809 static u32 ppc440spe_desc_get_src_addr(struct ppc440spe_adma_desc_slot *desc,
 810                                 struct ppc440spe_adma_chan *chan, int src_idx)
 811 {
 812         struct dma_cdb *dma_hw_desc;
 813         struct xor_cb *xor_hw_desc;
 814
 815         switch (chan->device->id) {
 816         case PPC440SPE_DMA0_ID:
 817         case PPC440SPE_DMA1_ID:
 818                 dma_hw_desc = desc->hw_desc;
 819                 /* May have 0, 1, 2, or 3 sources */
 820                 switch (dma_hw_desc->opc) {
 821                 case DMA_CDB_OPC_NO_OP:
 822                 case DMA_CDB_OPC_DFILL128:
 823                         return 0;
 824                 case DMA_CDB_OPC_DCHECK128:
 825                         if (unlikely(src_idx)) {
 826                                 printk(KERN_ERR "%s: try to get %d source for"
 827                                     " DCHECK128\n", __func__, src_idx);
 828                                 BUG();
 829                         }
 830                         return le32_to_cpu(dma_hw_desc->sg1l);
 831                 case DMA_CDB_OPC_MULTICAST:
 832                 case DMA_CDB_OPC_MV_SG1_SG2:
 833                         if (unlikely(src_idx > 2)) {
 834                                 printk(KERN_ERR "%s: try to get %d source from"
 835                                     " DMA descr\n", __func__, src_idx);
 836                                 BUG();
 837                         }
 838                         if (src_idx) {
 839                                 if (le32_to_cpu(dma_hw_desc->sg1u) &
 840                                     DMA_CUED_XOR_WIN_MSK) {
 841                                         u8 region;
 842
 843                                         if (src_idx == 1)
 844                                                 return le32_to_cpu(
 845                                                     dma_hw_desc->sg1l) +
 846                                                         desc->unmap_len;
 847
 848                                         region = (le32_to_cpu(
 849                                             dma_hw_desc->sg1u)) >>
 850                                                 DMA_CUED_REGION_OFF;
 851
 852                                         region &= DMA_CUED_REGION_MSK;
 853                                         switch (region) {
 854                                         case DMA_RXOR123:
 855                                                 return le32_to_cpu(
 856                                                     dma_hw_desc->sg1l) +
 857                                                         (desc->unmap_len << 1);
 858                                         case DMA_RXOR124:
 859                                                 return le32_to_cpu(
 860                                                     dma_hw_desc->sg1l) +
 861                                                         (desc->unmap_len * 3);
 862                                         case DMA_RXOR125:
 863                                                 return le32_to_cpu(
 864                                                     dma_hw_desc->sg1l) +
 865                                                         (desc->unmap_len << 2);
 866                                         default:
 867                                                 printk(KERN_ERR
 868                                                     "%s: try to"
 869                                                     " get src3 for region %02x"
 870                                                     "PPC440SPE_DESC_RXOR12?\n",
 871                                                     __func__, region);
 872                                                 BUG();
 873                                         }
 874                                 } else {
 875                                         printk(KERN_ERR
 876                                                 "%s: try to get %d"
 877                                                 " source for non-cued descr\n",
 878                                                 __func__, src_idx);
 879                                         BUG();
 880                                 }
 881                         }
 882                         return le32_to_cpu(dma_hw_desc->sg1l);
 883                 default:
 884                         printk(KERN_ERR "%s: unknown OPC 0x%02x\n",
 885                                 __func__, dma_hw_desc->opc);
 886                         BUG();
 887                 }
 888                 return le32_to_cpu(dma_hw_desc->sg1l);
 889         case PPC440SPE_XOR_ID:
 890                 /* May have up to 16 sources */
 891                 xor_hw_desc = desc->hw_desc;
 892                 return xor_hw_desc->ops[src_idx].l;
 893         }
 894         return 0;
 895 }
 896
 897 /**
 898  * ppc440spe_desc_get_dest_addr - extract the destination address from the
 899  * descriptor
 900  */
 901 static u32 ppc440spe_desc_get_dest_addr(struct ppc440spe_adma_desc_slot *desc,
 902                                 struct ppc440spe_adma_chan *chan, int idx)
 903 {
 904         struct dma_cdb *dma_hw_desc;
 905         struct xor_cb *xor_hw_desc;
 906
 907         switch (chan->device->id) {
 908         case PPC440SPE_DMA0_ID:
 909         case PPC440SPE_DMA1_ID:
 910                 dma_hw_desc = desc->hw_desc;
 911
 912                 if (likely(!idx))
 913                         return le32_to_cpu(dma_hw_desc->sg2l);
 914                 return le32_to_cpu(dma_hw_desc->sg3l);
 915         case PPC440SPE_XOR_ID:
 916                 xor_hw_desc = desc->hw_desc;
 917                 return xor_hw_desc->cbtal;
 918         }
 919         return 0;
 920 }
 921
 922 /**
 923  * ppc440spe_desc_get_src_num - extract the number of source addresses from
 924  * the descriptor
 925  */
 926 static u32 ppc440spe_desc_get_src_num(struct ppc440spe_adma_desc_slot *desc,
 927                                 struct ppc440spe_adma_chan *chan)
 928 {
 929         struct dma_cdb *dma_hw_desc;
 930         struct xor_cb *xor_hw_desc;
 931
 932         switch (chan->device->id) {
 933         case PPC440SPE_DMA0_ID:
 934         case PPC440SPE_DMA1_ID:
 935                 dma_hw_desc = desc->hw_desc;
 936
 937                 switch (dma_hw_desc->opc) {
 938                 case DMA_CDB_OPC_NO_OP:
 939                 case DMA_CDB_OPC_DFILL128:
 940                         return 0;
 941                 case DMA_CDB_OPC_DCHECK128:
 942                         return 1;
 943                 case DMA_CDB_OPC_MV_SG1_SG2:
 944                 case DMA_CDB_OPC_MULTICAST:
 945                         /*
 946                          * Only for RXOR operations we have more than
 947                          * one source
 948                          */
 949                         if (le32_to_cpu(dma_hw_desc->sg1u) &
 950                             DMA_CUED_XOR_WIN_MSK) {
 951                                 /* RXOR op, there are 2 or 3 sources */
 952                                 if (((le32_to_cpu(dma_hw_desc->sg1u) >>
 953                                     DMA_CUED_REGION_OFF) &
 954                                       DMA_CUED_REGION_MSK) == DMA_RXOR12) {
 955                                         /* RXOR 1-2 */
 956                                         return 2;
 957                                 } else {
 958                                         /* RXOR 1-2-3/1-2-4/1-2-5 */
 959                                         return 3;
 960                                 }
 961                         }
 962                         return 1;
 963                 default:
 964                         printk(KERN_ERR "%s: unknown OPC 0x%02x\n",
 965                                 __func__, dma_hw_desc->opc);
 966                         BUG();
 967                 }
 968         case PPC440SPE_XOR_ID:
 969                 /* up to 16 sources */
 970                 xor_hw_desc = desc->hw_desc;
 971                 return xor_hw_desc->cbc & XOR_CDCR_OAC_MSK;
 972         default:
 973                 BUG();
 974         }
 975         return 0;
 976 }
 977
 978 /**
 979  * ppc440spe_desc_get_dst_num - get the number of destination addresses in
 980  * this descriptor
 981  */
 982 static u32 ppc440spe_desc_get_dst_num(struct ppc440spe_adma_desc_slot *desc,
 983                                 struct ppc440spe_adma_chan *chan)
 984 {
 985         struct dma_cdb *dma_hw_desc;
 986
 987         switch (chan->device->id) {
 988         case PPC440SPE_DMA0_ID:
 989         case PPC440SPE_DMA1_ID:
 990                 /* May be 1 or 2 destinations */
 991                 dma_hw_desc = desc->hw_desc;
 992                 switch (dma_hw_desc->opc) {
 993                 case DMA_CDB_OPC_NO_OP:
 994                 case DMA_CDB_OPC_DCHECK128:
 995                         return 0;
 996                 case DMA_CDB_OPC_MV_SG1_SG2:
 997                 case DMA_CDB_OPC_DFILL128:
 998                         return 1;
 999                 case DMA_CDB_OPC_MULTICAST:
1000                         if (desc->dst_cnt == 2)
1001                                 return 2;
1002                         else
1003                                 return 1;
1004                 default:
1005                         printk(KERN_ERR "%s: unknown OPC 0x%02x\n",
1006                                 __func__, dma_hw_desc->opc);
1007                         BUG();
1008                 }
1009         case PPC440SPE_XOR_ID:
1010                 /* Always only 1 destination */
1011                 return 1;
1012         default:
1013                 BUG();
1014         }
1015         return 0;
1016 }
1017
1018 /**
1019  * ppc440spe_desc_get_link - get the address of the descriptor that
1020  * follows this one
1021  */
1022 static inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc,
1023                                         struct ppc440spe_adma_chan *chan)
1024 {
1025         if (!desc->hw_next)
1026                 return 0;
1027
1028         return desc->hw_next->phys;
1029 }
1030
1031 /**
1032  * ppc440spe_desc_is_aligned - check alignment
1033  */
1034 static inline int ppc440spe_desc_is_aligned(
1035         struct ppc440spe_adma_desc_slot *desc, int num_slots)
1036 {
1037         return (desc->idx & (num_slots - 1)) ? 0 : 1;
1038 }
1039
1040 /**
1041  * ppc440spe_chan_xor_slot_count - get the number of slots necessary for
1042  * XOR operation
1043  */
1044 static int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt,
1045                         int *slots_per_op)
1046 {
1047         int slot_cnt;
1048
1049         /* each XOR descriptor provides up to 16 source operands */
1050         slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS;
1051
1052         if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT))
1053                 return slot_cnt;
1054
1055         printk(KERN_ERR "%s: len %d > max %d !!\n",
1056                 __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
1057         BUG();
1058         return slot_cnt;
1059 }
1060
1061 /**
1062  * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for
1063  * DMA2 PQ operation
1064  */
1065 static int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs,
1066                 int src_cnt, size_t len)
1067 {
1068         signed long long order = 0;
1069         int state = 0;
1070         int addr_count = 0;
1071         int i;
1072         for (i = 1; i < src_cnt; i++) {
1073                 dma_addr_t cur_addr = srcs[i];
1074                 dma_addr_t old_addr = srcs[i-1];
1075                 switch (state) {
1076                 case 0:
1077                         if (cur_addr == old_addr + len) {
1078                                 /* direct RXOR */
1079                                 order = 1;
1080                                 state = 1;
1081                                 if (i == src_cnt-1)
1082                                         addr_count++;
1083                         } else if (old_addr == cur_addr + len) {
1084                                 /* reverse RXOR */
1085                                 order = -1;
1086                                 state = 1;
1087                                 if (i == src_cnt-1)
1088                                         addr_count++;
1089                         } else {
1090                                 state = 3;
1091                         }
1092                         break;
1093                 case 1:
1094                         if (i == src_cnt-2 || (order == -1
1095                                 && cur_addr != old_addr - len)) {
1096                                 order = 0;
1097                                 state = 0;
1098                                 addr_count++;
1099                         } else if (cur_addr == old_addr + len*order) {
1100                                 state = 2;
1101                                 if (i == src_cnt-1)
1102                                         addr_count++;
1103                         } else if (cur_addr == old_addr + 2*len) {
1104                                 state = 2;
1105                                 if (i == src_cnt-1)
1106                                         addr_count++;
1107                         } else if (cur_addr == old_addr + 3*len) {
1108                                 state = 2;
1109                                 if (i == src_cnt-1)
1110                                         addr_count++;
1111                         } else {
1112                                 order = 0;
1113                                 state = 0;
1114                                 addr_count++;
1115                         }
1116                         break;
1117                 case 2:
1118                         order = 0;
1119                         state = 0;
1120                         addr_count++;
1121                                 break;
1122                 }
1123                 if (state == 3)
1124                         break;
1125         }
1126         if (src_cnt <= 1 || (state != 1 && state != 2)) {
1127                 pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n",
1128                         __func__, src_cnt, state, addr_count, order);
1129                 for (i = 0; i < src_cnt; i++)
1130                         pr_err("\t[%d] 0x%llx \n", i, srcs[i]);
1131                 BUG();
1132         }
1133
1134         return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS;
1135 }
1136
1137
1138 /******************************************************************************
1139  * ADMA channel low-level routines
1140  ******************************************************************************/
1141
1142 static u32
1143 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan);
1144 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan);
1145
1146 /**
1147  * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine
1148  */
1149 static void ppc440spe_adma_device_clear_eot_status(
1150                                         struct ppc440spe_adma_chan *chan)
1151 {
1152         struct dma_regs *dma_reg;
1153         struct xor_regs *xor_reg;
1154         u8 *p = chan->device->dma_desc_pool_virt;
1155         struct dma_cdb *cdb;
1156         u32 rv, i;
1157
1158         switch (chan->device->id) {
1159         case PPC440SPE_DMA0_ID:
1160         case PPC440SPE_DMA1_ID:
1161                 /* read FIFO to ack */
1162                 dma_reg = chan->device->dma_reg;
1163                 while ((rv = ioread32(&dma_reg->csfpl))) {
1164                         i = rv & DMA_CDB_ADDR_MSK;
1165                         cdb = (struct dma_cdb *)&p[i -
1166                             (u32)chan->device->dma_desc_pool];
1167
1168                         /* Clear opcode to ack. This is necessary for
1169                          * ZeroSum operations only
1170                          */
1171                         cdb->opc = 0;
1172
1173                         if (test_bit(PPC440SPE_RXOR_RUN,
1174                             &ppc440spe_rxor_state)) {
1175                                 /* probably this is a completed RXOR op,
1176                                  * get pointer to CDB using the fact that
1177                                  * physical and virtual addresses of CDB
1178                                  * in pools have the same offsets
1179                                  */
1180                                 if (le32_to_cpu(cdb->sg1u) &
1181                                     DMA_CUED_XOR_BASE) {
1182                                         /* this is a RXOR */
1183                                         clear_bit(PPC440SPE_RXOR_RUN,
1184                                                   &ppc440spe_rxor_state);
1185                                 }
1186                         }
1187
1188                         if (rv & DMA_CDB_STATUS_MSK) {
1189                                 /* ZeroSum check failed
1190                                  */
1191                                 struct ppc440spe_adma_desc_slot *iter;
1192                                 dma_addr_t phys = rv & ~DMA_CDB_MSK;
1193
1194                                 /*
1195                                  * Update the status of corresponding
1196                                  * descriptor.
1197                                  */
1198                                 list_for_each_entry(iter, &chan->chain,
1199                                     chain_node) {
1200                                         if (iter->phys == phys)
1201                                                 break;
1202                                 }
1203                                 /*
1204                                  * if cannot find the corresponding
1205                                  * slot it's a bug
1206                                  */
1207                                 BUG_ON(&iter->chain_node == &chan->chain);
1208
1209                                 if (iter->xor_check_result) {
1210                                         if (test_bit(PPC440SPE_DESC_PCHECK,
1211                                                      &iter->flags)) {
1212                                                 *iter->xor_check_result |=
1213                                                         SUM_CHECK_P_RESULT;
1214                                         } else
1215                                         if (test_bit(PPC440SPE_DESC_QCHECK,
1216                                                      &iter->flags)) {
1217                                                 *iter->xor_check_result |=
1218                                                         SUM_CHECK_Q_RESULT;
1219                                         } else
1220                                                 BUG();
1221                                 }
1222                         }
1223                 }
1224
1225                 rv = ioread32(&dma_reg->dsts);
1226                 if (rv) {
1227                         pr_err("DMA%d err status: 0x%x\n",
1228                                chan->device->id, rv);
1229                         /* write back to clear */
1230                         iowrite32(rv, &dma_reg->dsts);
1231                 }
1232                 break;
1233         case PPC440SPE_XOR_ID:
1234                 /* reset status bits to ack */
1235                 xor_reg = chan->device->xor_reg;
1236                 rv = ioread32be(&xor_reg->sr);
1237                 iowrite32be(rv, &xor_reg->sr);
1238
1239                 if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) {
1240                         if (rv & XOR_IE_RPTIE_BIT) {
1241                                 /* Read PLB Timeout Error.
1242                                  * Try to resubmit the CB
1243                                  */
1244                                 u32 val = ioread32be(&xor_reg->ccbalr);
1245
1246                                 iowrite32be(val, &xor_reg->cblalr);
1247
1248                                 val = ioread32be(&xor_reg->crsr);
1249                                 iowrite32be(val | XOR_CRSR_XAE_BIT,
1250                                             &xor_reg->crsr);
1251                         } else
1252                                 pr_err("XOR ERR 0x%x status\n", rv);
1253                         break;
1254                 }
1255
1256                 /*  if the XORcore is idle, but there are unprocessed CBs
1257                  * then refetch the s/w chain here
1258                  */
1259                 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) &&
1260                     do_xor_refetch)
1261                         ppc440spe_chan_append(chan);
1262                 break;
1263         }
1264 }
1265
1266 /**
1267  * ppc440spe_chan_is_busy - get the channel status
1268  */
1269 static int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan)
1270 {
1271         struct dma_regs *dma_reg;
1272         struct xor_regs *xor_reg;
1273         int busy = 0;
1274
1275         switch (chan->device->id) {
1276         case PPC440SPE_DMA0_ID:
1277         case PPC440SPE_DMA1_ID:
1278                 dma_reg = chan->device->dma_reg;
1279                 /*  if command FIFO's head and tail pointers are equal and
1280                  * status tail is the same as command, then channel is free
1281                  */
1282                 if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) ||
1283                     ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp))
1284                         busy = 1;
1285                 break;
1286         case PPC440SPE_XOR_ID:
1287                 /* use the special status bit for the XORcore
1288                  */
1289                 xor_reg = chan->device->xor_reg;
1290                 busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0;
1291                 break;
1292         }
1293
1294         return busy;
1295 }
1296
1297 /**
1298  * ppc440spe_chan_set_first_xor_descriptor -  init XORcore chain
1299  */
1300 static void ppc440spe_chan_set_first_xor_descriptor(
1301                                 struct ppc440spe_adma_chan *chan,
1302                                 struct ppc440spe_adma_desc_slot *next_desc)
1303 {
1304         struct xor_regs *xor_reg = chan->device->xor_reg;
1305
1306         if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)
1307                 printk(KERN_INFO "%s: Warn: XORcore is running "
1308                         "when try to set the first CDB!\n",
1309                         __func__);
1310
1311         xor_last_submit = xor_last_linked = next_desc;
1312
1313         iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr);
1314
1315         iowrite32be(next_desc->phys, &xor_reg->cblalr);
1316         iowrite32be(0, &xor_reg->cblahr);
1317         iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT,
1318                     &xor_reg->cbcr);
1319
1320         chan->hw_chain_inited = 1;
1321 }
1322
1323 /**
1324  * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO.
1325  * called with irqs disabled
1326  */
1327 static void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan,
1328                 struct ppc440spe_adma_desc_slot *desc)
1329 {
1330         u32 pcdb;
1331         struct dma_regs *dma_reg = chan->device->dma_reg;
1332
1333         pcdb = desc->phys;
1334         if (!test_bit(PPC440SPE_DESC_INT, &desc->flags))
1335                 pcdb |= DMA_CDB_NO_INT;
1336
1337         chan_last_sub[chan->device->id] = desc;
1338
1339         ADMA_LL_DBG(print_cb(chan, desc->hw_desc));
1340
1341         iowrite32(pcdb, &dma_reg->cpfpl);
1342 }
1343
1344 /**
1345  * ppc440spe_chan_append - update the h/w chain in the channel
1346  */
1347 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan)
1348 {
1349         struct xor_regs *xor_reg;
1350         struct ppc440spe_adma_desc_slot *iter;
1351         struct xor_cb *xcb;
1352         u32 cur_desc;
1353         unsigned long flags;
1354
1355         local_irq_save(flags);
1356
1357         switch (chan->device->id) {
1358         case PPC440SPE_DMA0_ID:
1359         case PPC440SPE_DMA1_ID:
1360                 cur_desc = ppc440spe_chan_get_current_descriptor(chan);
1361
1362                 if (likely(cur_desc)) {
1363                         iter = chan_last_sub[chan->device->id];
1364                         BUG_ON(!iter);
1365                 } else {
1366                         /* first peer */
1367                         iter = chan_first_cdb[chan->device->id];
1368                         BUG_ON(!iter);
1369                         ppc440spe_dma_put_desc(chan, iter);
1370                         chan->hw_chain_inited = 1;
1371                 }
1372
1373                 /* is there something new to append */
1374                 if (!iter->hw_next)
1375                         break;
1376
1377                 /* flush descriptors from the s/w queue to fifo */
1378                 list_for_each_entry_continue(iter, &chan->chain, chain_node) {
1379                         ppc440spe_dma_put_desc(chan, iter);
1380                         if (!iter->hw_next)
1381                                 break;
1382                 }
1383                 break;
1384         case PPC440SPE_XOR_ID:
1385                 /* update h/w links and refetch */
1386                 if (!xor_last_submit->hw_next)
1387                         break;
1388
1389                 xor_reg = chan->device->xor_reg;
1390                 /* the last linked CDB has to generate an interrupt
1391                  * that we'd be able to append the next lists to h/w
1392                  * regardless of the XOR engine state at the moment of
1393                  * appending of these next lists
1394                  */
1395                 xcb = xor_last_linked->hw_desc;
1396                 xcb->cbc |= XOR_CBCR_CBCE_BIT;
1397
1398                 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) {
1399                         /* XORcore is idle. Refetch now */
1400                         do_xor_refetch = 0;
1401                         ppc440spe_xor_set_link(xor_last_submit,
1402                                 xor_last_submit->hw_next);
1403
1404                         ADMA_LL_DBG(print_cb_list(chan,
1405                                 xor_last_submit->hw_next));
1406
1407                         xor_last_submit = xor_last_linked;
1408                         iowrite32be(ioread32be(&xor_reg->crsr) |
1409                                     XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT,
1410                                     &xor_reg->crsr);
1411                 } else {
1412                         /* XORcore is running. Refetch later in the handler */
1413                         do_xor_refetch = 1;
1414                 }
1415
1416                 break;
1417         }
1418
1419         local_irq_restore(flags);
1420 }
1421
1422 /**
1423  * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor
1424  */
1425 static u32
1426 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan)
1427 {
1428         struct dma_regs *dma_reg;
1429         struct xor_regs *xor_reg;
1430
1431         if (unlikely(!chan->hw_chain_inited))
1432                 /* h/w descriptor chain is not initialized yet */
1433                 return 0;
1434
1435         switch (chan->device->id) {
1436         case PPC440SPE_DMA0_ID:
1437         case PPC440SPE_DMA1_ID:
1438                 dma_reg = chan->device->dma_reg;
1439                 return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK);
1440         case PPC440SPE_XOR_ID:
1441                 xor_reg = chan->device->xor_reg;
1442                 return ioread32be(&xor_reg->ccbalr);
1443         }
1444         return 0;
1445 }
1446
1447 /**
1448  * ppc440spe_chan_run - enable the channel
1449  */
1450 static void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan)
1451 {
1452         struct xor_regs *xor_reg;
1453
1454         switch (chan->device->id) {
1455         case PPC440SPE_DMA0_ID:
1456         case PPC440SPE_DMA1_ID:
1457                 /* DMAs are always enabled, do nothing */
1458                 break;
1459         case PPC440SPE_XOR_ID:
1460                 /* drain write buffer */
1461                 xor_reg = chan->device->xor_reg;
1462
1463                 /* fetch descriptor pointed to in <link> */
1464                 iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT,
1465                             &xor_reg->crsr);
1466                 break;
1467         }
1468 }
1469
1470 /******************************************************************************
1471  * ADMA device level
1472  ******************************************************************************/
1473
1474 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan);
1475 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan);
1476
1477 static dma_cookie_t
1478 ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx);
1479
1480 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx,
1481                                     dma_addr_t addr, int index);
1482 static void
1483 ppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx,
1484                                   dma_addr_t addr, int index);
1485
1486 static void
1487 ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx,
1488                            dma_addr_t *paddr, unsigned long flags);
1489 static void
1490 ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx,
1491                           dma_addr_t addr, int index);
1492 static void
1493 ppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx,
1494                                unsigned char mult, int index, int dst_pos);
1495 static void
1496 ppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx,
1497                                    dma_addr_t paddr, dma_addr_t qaddr);
1498
1499 static struct page *ppc440spe_rxor_srcs[32];
1500
1501 /**
1502  * ppc440spe_can_rxor - check if the operands may be processed with RXOR
1503  */
1504 static int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len)
1505 {
1506         int i, order = 0, state = 0;
1507         int idx = 0;
1508
1509         if (unlikely(!(src_cnt > 1)))
1510                 return 0;
1511
1512         BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs));
1513
1514         /* Skip holes in the source list before checking */
1515         for (i = 0; i < src_cnt; i++) {
1516                 if (!srcs[i])
1517                         continue;
1518                 ppc440spe_rxor_srcs[idx++] = srcs[i];
1519         }
1520         src_cnt = idx;
1521
1522         for (i = 1; i < src_cnt; i++) {
1523                 char *cur_addr = page_address(ppc440spe_rxor_srcs[i]);
1524                 char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]);
1525
1526                 switch (state) {
1527                 case 0:
1528                         if (cur_addr == old_addr + len) {
1529                                 /* direct RXOR */
1530                                 order = 1;
1531                                 state = 1;
1532                         } else if (old_addr == cur_addr + len) {
1533                                 /* reverse RXOR */
1534                                 order = -1;
1535                                 state = 1;
1536                         } else
1537                                 goto out;
1538                         break;
1539                 case 1:
1540                         if ((i == src_cnt - 2) ||
1541                             (order == -1 && cur_addr != old_addr - len)) {
1542                                 order = 0;
1543                                 state = 0;
1544                         } else if ((cur_addr == old_addr + len * order) ||
1545                                    (cur_addr == old_addr + 2 * len) ||
1546                                    (cur_addr == old_addr + 3 * len)) {
1547                                 state = 2;
1548                         } else {
1549                                 order = 0;
1550                                 state = 0;
1551                         }
1552                         break;
1553                 case 2:
1554                         order = 0;
1555                         state = 0;
1556                         break;
1557                 }
1558         }
1559
1560 out:
1561         if (state == 1 || state == 2)
1562                 return 1;
1563
1564         return 0;
1565 }
1566
1567 /**
1568  * ppc440spe_adma_device_estimate - estimate the efficiency of processing
1569  *      the operation given on this channel. It's assumed that 'chan' is
1570  *      capable to process 'cap' type of operation.
1571  * @chan: channel to use
1572  * @cap: type of transaction
1573  * @dst_lst: array of destination pointers
1574  * @dst_cnt: number of destination operands
1575  * @src_lst: array of source pointers
1576  * @src_cnt: number of source operands
1577  * @src_sz: size of each source operand
1578  */
1579 static int ppc440spe_adma_estimate(struct dma_chan *chan,
1580         enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt,
1581         struct page **src_lst, int src_cnt, size_t src_sz)
1582 {
1583         int ef = 1;
1584
1585         if (cap == DMA_PQ || cap == DMA_PQ_VAL) {
1586                 /* If RAID-6 capabilities were not activated don't try
1587                  * to use them
1588                  */
1589                 if (unlikely(!ppc440spe_r6_enabled))
1590                         return -1;
1591         }
1592         /*  In the current implementation of ppc440spe ADMA driver it
1593          * makes sense to pick out only pq case, because it may be
1594          * processed:
1595          * (1) either using Biskup method on DMA2;
1596          * (2) or on DMA0/1.
1597          *  Thus we give a favour to (1) if the sources are suitable;
1598          * else let it be processed on one of the DMA0/1 engines.
1599          *  In the sum_product case where destination is also the
1600          * source process it on DMA0/1 only.
1601          */
1602         if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) {
1603
1604                 if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1])
1605                         ef = 0; /* sum_product case, process on DMA0/1 */
1606                 else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz))
1607                         ef = 3; /* override (DMA0/1 + idle) */
1608                 else
1609                         ef = 0; /* can't process on DMA2 if !rxor */
1610         }
1611
1612         /* channel idleness increases the priority */
1613         if (likely(ef) &&
1614             !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan)))
1615                 ef++;
1616
1617         return ef;
1618 }
1619
1620 struct dma_chan *
1621 ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap,
1622         struct page **dst_lst, int dst_cnt, struct page **src_lst,
1623         int src_cnt, size_t src_sz)
1624 {
1625         struct dma_chan *best_chan = NULL;
1626         struct ppc_dma_chan_ref *ref;
1627         int best_rank = -1;
1628
1629         if (unlikely(!src_sz))
1630                 return NULL;
1631         if (src_sz > PAGE_SIZE) {
1632                 /*
1633                  * should a user of the api ever pass > PAGE_SIZE requests
1634                  * we sort out cases where temporary page-sized buffers
1635                  * are used.
1636                  */
1637                 switch (cap) {
1638                 case DMA_PQ:
1639                         if (src_cnt == 1 && dst_lst[1] == src_lst[0])
1640                                 return NULL;
1641                         if (src_cnt == 2 && dst_lst[1] == src_lst[1])
1642                                 return NULL;
1643                         break;
1644                 case DMA_PQ_VAL:
1645                 case DMA_XOR_VAL:
1646                         return NULL;
1647                 default:
1648                         break;
1649                 }
1650         }
1651
1652         list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) {
1653                 if (dma_has_cap(cap, ref->chan->device->cap_mask)) {
1654                         int rank;
1655
1656                         rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst,
1657                                         dst_cnt, src_lst, src_cnt, src_sz);
1658                         if (rank > best_rank) {
1659                                 best_rank = rank;
1660                                 best_chan = ref->chan;
1661                         }
1662                 }
1663         }
1664
1665         return best_chan;
1666 }
1667 EXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel);
1668
1669 /**
1670  * ppc440spe_get_group_entry - get group entry with index idx
1671  * @tdesc: is the last allocated slot in the group.
1672  */
1673 static struct ppc440spe_adma_desc_slot *
1674 ppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx)
1675 {
1676         struct ppc440spe_adma_desc_slot *iter = tdesc->group_head;
1677         int i = 0;
1678
1679         if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) {
1680                 printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n",
1681                         __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt);
1682                 BUG();
1683         }
1684
1685         list_for_each_entry(iter, &tdesc->group_list, chain_node) {
1686                 if (i++ == entry_idx)
1687                         break;
1688         }
1689         return iter;
1690 }
1691
1692 /**
1693  * ppc440spe_adma_free_slots - flags descriptor slots for reuse
1694  * @slot: Slot to free
1695  * Caller must hold &ppc440spe_chan->lock while calling this function
1696  */
1697 static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot,
1698                                       struct ppc440spe_adma_chan *chan)
1699 {
1700         int stride = slot->slots_per_op;
1701
1702         while (stride--) {
1703                 slot->slots_per_op = 0;
1704                 slot = list_entry(slot->slot_node.next,
1705                                 struct ppc440spe_adma_desc_slot,
1706                                 slot_node);
1707         }
1708 }
1709
1710 static void ppc440spe_adma_unmap(struct ppc440spe_adma_chan *chan,
1711                                  struct ppc440spe_adma_desc_slot *desc)
1712 {
1713         u32 src_cnt, dst_cnt;
1714         dma_addr_t addr;
1715
1716         /*
1717          * get the number of sources & destination
1718          * included in this descriptor and unmap
1719          * them all
1720          */
1721         src_cnt = ppc440spe_desc_get_src_num(desc, chan);
1722         dst_cnt = ppc440spe_desc_get_dst_num(desc, chan);
1723
1724         /* unmap destinations */
1725         if (!(desc->async_tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
1726                 while (dst_cnt--) {
1727                         addr = ppc440spe_desc_get_dest_addr(
1728                                 desc, chan, dst_cnt);
1729                         dma_unmap_page(chan->device->dev,
1730                                         addr, desc->unmap_len,
1731                                         DMA_FROM_DEVICE);
1732                 }
1733         }
1734
1735         /* unmap sources */
1736         if (!(desc->async_tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
1737                 while (src_cnt--) {
1738                         addr = ppc440spe_desc_get_src_addr(
1739                                 desc, chan, src_cnt);
1740                         dma_unmap_page(chan->device->dev,
1741                                         addr, desc->unmap_len,
1742                                         DMA_TO_DEVICE);
1743                 }
1744         }
1745 }
1746
1747 /**
1748  * ppc440spe_adma_run_tx_complete_actions - call functions to be called
1749  * upon completion
1750  */
1751 static dma_cookie_t ppc440spe_adma_run_tx_complete_actions(
1752                 struct ppc440spe_adma_desc_slot *desc,
1753                 struct ppc440spe_adma_chan *chan,
1754                 dma_cookie_t cookie)
1755 {
1756         int i;
1757
1758         BUG_ON(desc->async_tx.cookie < 0);
1759         if (desc->async_tx.cookie > 0) {
1760                 cookie = desc->async_tx.cookie;
1761                 desc->async_tx.cookie = 0;
1762
1763                 /* call the callback (must not sleep or submit new
1764                  * operations to this channel)
1765                  */
1766                 if (desc->async_tx.callback)
1767                         desc->async_tx.callback(
1768                                 desc->async_tx.callback_param);
1769
1770                 /* unmap dma addresses
1771                  * (unmap_single vs unmap_page?)
1772                  *
1773                  * actually, ppc's dma_unmap_page() functions are empty, so
1774                  * the following code is just for the sake of completeness
1775                  */
1776                 if (chan && chan->needs_unmap && desc->group_head &&
1777                      desc->unmap_len) {
1778                         struct ppc440spe_adma_desc_slot *unmap =
1779                                                         desc->group_head;
1780                         /* assume 1 slot per op always */
1781                         u32 slot_count = unmap->slot_cnt;
1782
1783                         /* Run through the group list and unmap addresses */
1784                         for (i = 0; i < slot_count; i++) {
1785                                 BUG_ON(!unmap);
1786                                 ppc440spe_adma_unmap(chan, unmap);
1787                                 unmap = unmap->hw_next;
1788                         }
1789                 }
1790         }
1791
1792         /* run dependent operations */
1793         dma_run_dependencies(&desc->async_tx);
1794
1795         return cookie;
1796 }
1797
1798 /**
1799  * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set)
1800  */
1801 static int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc,
1802                 struct ppc440spe_adma_chan *chan)
1803 {
1804         /* the client is allowed to attach dependent operations
1805          * until 'ack' is set
1806          */
1807         if (!async_tx_test_ack(&desc->async_tx))
1808                 return 0;
1809
1810         /* leave the last descriptor in the chain
1811          * so we can append to it
1812          */
1813         if (list_is_last(&desc->chain_node, &chan->chain) ||
1814             desc->phys == ppc440spe_chan_get_current_descriptor(chan))
1815                 return 1;
1816
1817         if (chan->device->id != PPC440SPE_XOR_ID) {
1818                 /* our DMA interrupt handler clears opc field of
1819                  * each processed descriptor. For all types of
1820                  * operations except for ZeroSum we do not actually
1821                  * need ack from the interrupt handler. ZeroSum is a
1822                  * special case since the result of this operation
1823                  * is available from the handler only, so if we see
1824                  * such type of descriptor (which is unprocessed yet)
1825                  * then leave it in chain.
1826                  */
1827                 struct dma_cdb *cdb = desc->hw_desc;
1828                 if (cdb->opc == DMA_CDB_OPC_DCHECK128)
1829                         return 1;
1830         }
1831
1832         dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n",
1833                 desc->phys, desc->idx, desc->slots_per_op);
1834
1835         list_del(&desc->chain_node);
1836         ppc440spe_adma_free_slots(desc, chan);
1837         return 0;
1838 }
1839
1840 /**
1841  * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine
1842  *      which runs through the channel CDBs list until reach the descriptor
1843  *      currently processed. When routine determines that all CDBs of group
1844  *      are completed then corresponding callbacks (if any) are called and slots
1845  *      are freed.
1846  */
1847 static void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1848 {
1849         struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL;
1850         dma_cookie_t cookie = 0;
1851         u32 current_desc = ppc440spe_chan_get_current_descriptor(chan);
1852         int busy = ppc440spe_chan_is_busy(chan);
1853         int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
1854
1855         dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n",
1856                 chan->device->id, __func__);
1857
1858         if (!current_desc) {
1859                 /*  There were no transactions yet, so
1860                  * nothing to clean
1861                  */
1862                 return;
1863         }
1864
1865         /* free completed slots from the chain starting with
1866          * the oldest descriptor
1867          */
1868         list_for_each_entry_safe(iter, _iter, &chan->chain,
1869                                         chain_node) {
1870                 dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d "
1871                     "busy: %d this_desc: %#llx next_desc: %#x "
1872                     "cur: %#x ack: %d\n",
1873                     iter->async_tx.cookie, iter->idx, busy, iter->phys,
1874                     ppc440spe_desc_get_link(iter, chan), current_desc,
1875                     async_tx_test_ack(&iter->async_tx));
1876                 prefetch(_iter);
1877                 prefetch(&_iter->async_tx);
1878
1879                 /* do not advance past the current descriptor loaded into the
1880                  * hardware channel,subsequent descriptors are either in process
1881                  * or have not been submitted
1882                  */
1883                 if (seen_current)
1884                         break;
1885
1886                 /* stop the search if we reach the current descriptor and the
1887                  * channel is busy, or if it appears that the current descriptor
1888                  * needs to be re-read (i.e. has been appended to)
1889                  */
1890                 if (iter->phys == current_desc) {
1891                         BUG_ON(seen_current++);
1892                         if (busy || ppc440spe_desc_get_link(iter, chan)) {
1893                                 /* not all descriptors of the group have
1894                                  * been completed; exit.
1895                                  */
1896                                 break;
1897                         }
1898                 }
1899
1900                 /* detect the start of a group transaction */
1901                 if (!slot_cnt && !slots_per_op) {
1902                         slot_cnt = iter->slot_cnt;
1903                         slots_per_op = iter->slots_per_op;
1904                         if (slot_cnt <= slots_per_op) {
1905                                 slot_cnt = 0;
1906                                 slots_per_op = 0;
1907                         }
1908                 }
1909
1910                 if (slot_cnt) {
1911                         if (!group_start)
1912                                 group_start = iter;
1913                         slot_cnt -= slots_per_op;
1914                 }
1915
1916                 /* all the members of a group are complete */
1917                 if (slots_per_op != 0 && slot_cnt == 0) {
1918                         struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter;
1919                         int end_of_chain = 0;
1920
1921                         /* clean up the group */
1922                         slot_cnt = group_start->slot_cnt;
1923                         grp_iter = group_start;
1924                         list_for_each_entry_safe_from(grp_iter, _grp_iter,
1925                                 &chan->chain, chain_node) {
1926
1927                                 cookie = ppc440spe_adma_run_tx_complete_actions(
1928                                         grp_iter, chan, cookie);
1929
1930                                 slot_cnt -= slots_per_op;
1931                                 end_of_chain = ppc440spe_adma_clean_slot(
1932                                     grp_iter, chan);
1933                                 if (end_of_chain && slot_cnt) {
1934                                         /* Should wait for ZeroSum completion */
1935                                         if (cookie > 0)
1936                                                 chan->common.completed_cookie = cookie;
1937                                         return;
1938                                 }
1939
1940                                 if (slot_cnt == 0 || end_of_chain)
1941                                         break;
1942                         }
1943
1944                         /* the group should be complete at this point */
1945                         BUG_ON(slot_cnt);
1946
1947                         slots_per_op = 0;
1948                         group_start = NULL;
1949                         if (end_of_chain)
1950                                 break;
1951                         else
1952                                 continue;
1953                 } else if (slots_per_op) /* wait for group completion */
1954                         continue;
1955
1956                 cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan,
1957                     cookie);
1958
1959                 if (ppc440spe_adma_clean_slot(iter, chan))
1960                         break;
1961         }
1962
1963         BUG_ON(!seen_current);
1964
1965         if (cookie > 0) {
1966                 chan->common.completed_cookie = cookie;
1967                 pr_debug("\tcompleted cookie %d\n", cookie);
1968         }
1969
1970 }
1971
1972 /**
1973  * ppc440spe_adma_tasklet - clean up watch-dog initiator
1974  */
1975 static void ppc440spe_adma_tasklet(unsigned long data)
1976 {
1977         struct ppc440spe_adma_chan *chan = (struct ppc440spe_adma_chan *) data;
1978
1979         spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING);
1980         __ppc440spe_adma_slot_cleanup(chan);
1981         spin_unlock(&chan->lock);
1982 }
1983
1984 /**
1985  * ppc440spe_adma_slot_cleanup - clean up scheduled initiator
1986  */
1987 static void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1988 {
1989         spin_lock_bh(&chan->lock);
1990         __ppc440spe_adma_slot_cleanup(chan);
1991         spin_unlock_bh(&chan->lock);
1992 }
1993
1994 /**
1995  * ppc440spe_adma_alloc_slots - allocate free slots (if any)
1996  */
1997 static struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots(
1998                 struct ppc440spe_adma_chan *chan, int num_slots,
1999                 int slots_per_op)
2000 {
2001         struct ppc440spe_adma_desc_slot *iter = NULL, *_iter;
2002         struct ppc440spe_adma_desc_slot *alloc_start = NULL;
2003         struct list_head chain = LIST_HEAD_INIT(chain);
2004         int slots_found, retry = 0;
2005
2006
2007         BUG_ON(!num_slots || !slots_per_op);
2008         /* start search from the last allocated descrtiptor
2009          * if a contiguous allocation can not be found start searching
2010          * from the beginning of the list
2011          */
2012 retry:
2013         slots_found = 0;
2014         if (retry == 0)
2015                 iter = chan->last_used;
2016         else
2017                 iter = list_entry(&chan->all_slots,
2018                                   struct ppc440spe_adma_desc_slot,
2019                                   slot_node);
2020         list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots,
2021             slot_node) {
2022                 prefetch(_iter);
2023                 prefetch(&_iter->async_tx);
2024                 if (iter->slots_per_op) {
2025                         slots_found = 0;
2026                         continue;
2027                 }
2028
2029                 /* start the allocation if the slot is correctly aligned */
2030                 if (!slots_found++)
2031                         alloc_start = iter;
2032
2033                 if (slots_found == num_slots) {
2034                         struct ppc440spe_adma_desc_slot *alloc_tail = NULL;
2035                         struct ppc440spe_adma_desc_slot *last_used = NULL;
2036
2037                         iter = alloc_start;
2038                         while (num_slots) {
2039                                 int i;
2040                                 /* pre-ack all but the last descriptor */
2041                                 if (num_slots != slots_per_op)
2042                                         async_tx_ack(&iter->async_tx);
2043
2044                                 list_add_tail(&iter->chain_node, &chain);
2045                                 alloc_tail = iter;
2046                                 iter->async_tx.cookie = 0;
2047                                 iter->hw_next = NULL;
2048                                 iter->flags = 0;
2049                                 iter->slot_cnt = num_slots;
2050                                 iter->xor_check_result = NULL;
2051                                 for (i = 0; i < slots_per_op; i++) {
2052                                         iter->slots_per_op = slots_per_op - i;
2053                                         last_used = iter;
2054                                         iter = list_entry(iter->slot_node.next,
2055                                                 struct ppc440spe_adma_desc_slot,
2056                                                 slot_node);
2057                                 }
2058                                 num_slots -= slots_per_op;
2059                         }
2060                         alloc_tail->group_head = alloc_start;
2061                         alloc_tail->async_tx.cookie = -EBUSY;
2062                         list_splice(&chain, &alloc_tail->group_list);
2063                         chan->last_used = last_used;
2064                         return alloc_tail;
2065                 }
2066         }
2067         if (!retry++)
2068                 goto retry;
2069
2070         /* try to free some slots if the allocation fails */
2071         tasklet_schedule(&chan->irq_tasklet);
2072         return NULL;
2073 }
2074
2075 /**
2076  * ppc440spe_adma_alloc_chan_resources -  allocate pools for CDB slots
2077  */
2078 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan)
2079 {
2080         struct ppc440spe_adma_chan *ppc440spe_chan;
2081         struct ppc440spe_adma_desc_slot *slot = NULL;
2082         char *hw_desc;
2083         int i, db_sz;
2084         int init;
2085
2086         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2087         init = ppc440spe_chan->slots_allocated ? 0 : 1;
2088         chan->chan_id = ppc440spe_chan->device->id;
2089
2090         /* Allocate descriptor slots */
2091         i = ppc440spe_chan->slots_allocated;
2092         if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID)
2093                 db_sz = sizeof(struct dma_cdb);
2094         else
2095                 db_sz = sizeof(struct xor_cb);
2096
2097         for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) {
2098                 slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot),
2099                                GFP_KERNEL);
2100                 if (!slot) {
2101                         printk(KERN_INFO "SPE ADMA Channel only initialized"
2102                                 " %d descriptor slots", i--);
2103                         break;
2104                 }
2105
2106                 hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt;
2107                 slot->hw_desc = (void *) &hw_desc[i * db_sz];
2108                 dma_async_tx_descriptor_init(&slot->async_tx, chan);
2109                 slot->async_tx.tx_submit = ppc440spe_adma_tx_submit;
2110                 INIT_LIST_HEAD(&slot->chain_node);
2111                 INIT_LIST_HEAD(&slot->slot_node);
2112                 INIT_LIST_HEAD(&slot->group_list);
2113                 slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz;
2114                 slot->idx = i;
2115
2116                 spin_lock_bh(&ppc440spe_chan->lock);
2117                 ppc440spe_chan->slots_allocated++;
2118                 list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots);
2119                 spin_unlock_bh(&ppc440spe_chan->lock);
2120         }
2121
2122         if (i && !ppc440spe_chan->last_used) {
2123                 ppc440spe_chan->last_used =
2124                         list_entry(ppc440spe_chan->all_slots.next,
2125                                 struct ppc440spe_adma_desc_slot,
2126                                 slot_node);
2127         }
2128
2129         dev_dbg(ppc440spe_chan->device->common.dev,
2130                 "ppc440spe adma%d: allocated %d descriptor slots\n",
2131                 ppc440spe_chan->device->id, i);
2132
2133         /* initialize the channel and the chain with a null operation */
2134         if (init) {
2135                 switch (ppc440spe_chan->device->id) {
2136                 case PPC440SPE_DMA0_ID:
2137                 case PPC440SPE_DMA1_ID:
2138                         ppc440spe_chan->hw_chain_inited = 0;
2139                         /* Use WXOR for self-testing */
2140                         if (!ppc440spe_r6_tchan)
2141                                 ppc440spe_r6_tchan = ppc440spe_chan;
2142                         break;
2143                 case PPC440SPE_XOR_ID:
2144                         ppc440spe_chan_start_null_xor(ppc440spe_chan);
2145                         break;
2146                 default:
2147                         BUG();
2148                 }
2149                 ppc440spe_chan->needs_unmap = 1;
2150         }
2151
2152         return (i > 0) ? i : -ENOMEM;
2153 }
2154
2155 /**
2156  * ppc440spe_rxor_set_region_data -
2157  */
2158 static void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc,
2159         u8 xor_arg_no, u32 mask)
2160 {
2161         struct xor_cb *xcb = desc->hw_desc;
2162
2163         xcb->ops[xor_arg_no].h |= mask;
2164 }
2165
2166 /**
2167  * ppc440spe_rxor_set_src -
2168  */
2169 static void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc,
2170         u8 xor_arg_no, dma_addr_t addr)
2171 {
2172         struct xor_cb *xcb = desc->hw_desc;
2173
2174         xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE;
2175         xcb->ops[xor_arg_no].l = addr;
2176 }
2177
2178 /**
2179  * ppc440spe_rxor_set_mult -
2180  */
2181 static void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc,
2182         u8 xor_arg_no, u8 idx, u8 mult)
2183 {
2184         struct xor_cb *xcb = desc->hw_desc;
2185
2186         xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8);
2187 }
2188
2189 /**
2190  * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold
2191  *      has been achieved
2192  */
2193 static void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan)
2194 {
2195         dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n",
2196                 chan->device->id, chan->pending);
2197
2198         if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) {
2199                 chan->pending = 0;
2200                 ppc440spe_chan_append(chan);
2201         }
2202 }
2203
2204 /**
2205  * ppc440spe_adma_tx_submit - submit new descriptor group to the channel
2206  *      (it's not necessary that descriptors will be submitted to the h/w
2207  *      chains too right now)
2208  */
2209 static dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx)
2210 {
2211         struct ppc440spe_adma_desc_slot *sw_desc;
2212         struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan);
2213         struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail;
2214         int slot_cnt;
2215         int slots_per_op;
2216         dma_cookie_t cookie;
2217
2218         sw_desc = tx_to_ppc440spe_adma_slot(tx);
2219
2220         group_start = sw_desc->group_head;
2221         slot_cnt = group_start->slot_cnt;
2222         slots_per_op = group_start->slots_per_op;
2223
2224         spin_lock_bh(&chan->lock);
2225         cookie = dma_cookie_assign(tx);
2226
2227         if (unlikely(list_empty(&chan->chain))) {
2228                 /* first peer */
2229                 list_splice_init(&sw_desc->group_list, &chan->chain);
2230                 chan_first_cdb[chan->device->id] = group_start;
2231         } else {
2232                 /* isn't first peer, bind CDBs to chain */
2233                 old_chain_tail = list_entry(chan->chain.prev,
2234                                         struct ppc440spe_adma_desc_slot,
2235                                         chain_node);
2236                 list_splice_init(&sw_desc->group_list,
2237                     &old_chain_tail->chain_node);
2238                 /* fix up the hardware chain */
2239                 ppc440spe_desc_set_link(chan, old_chain_tail, group_start);
2240         }
2241
2242         /* increment the pending count by the number of operations */
2243         chan->pending += slot_cnt / slots_per_op;
2244         ppc440spe_adma_check_threshold(chan);
2245         spin_unlock_bh(&chan->lock);
2246
2247         dev_dbg(chan->device->common.dev,
2248                 "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n",
2249                 chan->device->id, __func__,
2250                 sw_desc->async_tx.cookie, sw_desc->idx, sw_desc);
2251
2252         return cookie;
2253 }
2254
2255 /**
2256  * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation
2257  */
2258 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt(
2259                 struct dma_chan *chan, unsigned long flags)
2260 {
2261         struct ppc440spe_adma_chan *ppc440spe_chan;
2262         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2263         int slot_cnt, slots_per_op;
2264
2265         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2266
2267         dev_dbg(ppc440spe_chan->device->common.dev,
2268                 "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id,
2269                 __func__);
2270
2271         spin_lock_bh(&ppc440spe_chan->lock);
2272         slot_cnt = slots_per_op = 1;
2273         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2274                         slots_per_op);
2275         if (sw_desc) {
2276                 group_start = sw_desc->group_head;
2277                 ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan);
2278                 group_start->unmap_len = 0;
2279                 sw_desc->async_tx.flags = flags;
2280         }
2281         spin_unlock_bh(&ppc440spe_chan->lock);
2282
2283         return sw_desc ? &sw_desc->async_tx : NULL;
2284 }
2285
2286 /**
2287  * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation
2288  */
2289 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy(
2290                 struct dma_chan *chan, dma_addr_t dma_dest,
2291                 dma_addr_t dma_src, size_t len, unsigned long flags)
2292 {
2293         struct ppc440spe_adma_chan *ppc440spe_chan;
2294         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2295         int slot_cnt, slots_per_op;
2296
2297         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2298
2299         if (unlikely(!len))
2300                 return NULL;
2301
2302         BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT);
2303
2304         spin_lock_bh(&ppc440spe_chan->lock);
2305
2306         dev_dbg(ppc440spe_chan->device->common.dev,
2307                 "ppc440spe adma%d: %s len: %u int_en %d\n",
2308                 ppc440spe_chan->device->id, __func__, len,
2309                 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2310         slot_cnt = slots_per_op = 1;
2311         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2312                 slots_per_op);
2313         if (sw_desc) {
2314                 group_start = sw_desc->group_head;
2315                 ppc440spe_desc_init_memcpy(group_start, flags);
2316                 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2317                 ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0);
2318                 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2319                 sw_desc->unmap_len = len;
2320                 sw_desc->async_tx.flags = flags;
2321         }
2322         spin_unlock_bh(&ppc440spe_chan->lock);
2323
2324         return sw_desc ? &sw_desc->async_tx : NULL;
2325 }
2326
2327 /**
2328  * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation
2329  */
2330 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor(
2331                 struct dma_chan *chan, dma_addr_t dma_dest,
2332                 dma_addr_t *dma_src, u32 src_cnt, size_t len,
2333                 unsigned long flags)
2334 {
2335         struct ppc440spe_adma_chan *ppc440spe_chan;
2336         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2337         int slot_cnt, slots_per_op;
2338
2339         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2340
2341         ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id,
2342                                      dma_dest, dma_src, src_cnt));
2343         if (unlikely(!len))
2344                 return NULL;
2345         BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
2346
2347         dev_dbg(ppc440spe_chan->device->common.dev,
2348                 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2349                 ppc440spe_chan->device->id, __func__, src_cnt, len,
2350                 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2351
2352         spin_lock_bh(&ppc440spe_chan->lock);
2353         slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op);
2354         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2355                         slots_per_op);
2356         if (sw_desc) {
2357                 group_start = sw_desc->group_head;
2358                 ppc440spe_desc_init_xor(group_start, src_cnt, flags);
2359                 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2360                 while (src_cnt--)
2361                         ppc440spe_adma_memcpy_xor_set_src(group_start,
2362                                 dma_src[src_cnt], src_cnt);
2363                 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2364                 sw_desc->unmap_len = len;
2365                 sw_desc->async_tx.flags = flags;
2366         }
2367         spin_unlock_bh(&ppc440spe_chan->lock);
2368
2369         return sw_desc ? &sw_desc->async_tx : NULL;
2370 }
2371
2372 static inline void
2373 ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc,
2374                                 int src_cnt);
2375 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor);
2376
2377 /**
2378  * ppc440spe_adma_init_dma2rxor_slot -
2379  */
2380 static void ppc440spe_adma_init_dma2rxor_slot(
2381                 struct ppc440spe_adma_desc_slot *desc,
2382                 dma_addr_t *src, int src_cnt)
2383 {
2384         int i;
2385
2386         /* initialize CDB */
2387         for (i = 0; i < src_cnt; i++) {
2388                 ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i,
2389                                                  desc->src_cnt, (u32)src[i]);
2390         }
2391 }
2392
2393 /**
2394  * ppc440spe_dma01_prep_mult -
2395  * for Q operation where destination is also the source
2396  */
2397 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult(
2398                 struct ppc440spe_adma_chan *ppc440spe_chan,
2399                 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2400                 const unsigned char *scf, size_t len, unsigned long flags)
2401 {
2402         struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2403         unsigned long op = 0;
2404         int slot_cnt;
2405
2406         set_bit(PPC440SPE_DESC_WXOR, &op);
2407         slot_cnt = 2;
2408
2409         spin_lock_bh(&ppc440spe_chan->lock);
2410
2411         /* use WXOR, each descriptor occupies one slot */
2412         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2413         if (sw_desc) {
2414                 struct ppc440spe_adma_chan *chan;
2415                 struct ppc440spe_adma_desc_slot *iter;
2416                 struct dma_cdb *hw_desc;
2417
2418                 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2419                 set_bits(op, &sw_desc->flags);
2420                 sw_desc->src_cnt = src_cnt;
2421                 sw_desc->dst_cnt = dst_cnt;
2422                 /* First descriptor, zero data in the destination and copy it
2423                  * to q page using MULTICAST transfer.
2424                  */
2425                 iter = list_first_entry(&sw_desc->group_list,
2426                                         struct ppc440spe_adma_desc_slot,
2427                                         chain_node);
2428                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2429                 /* set 'next' pointer */
2430                 iter->hw_next = list_entry(iter->chain_node.next,
2431                                            struct ppc440spe_adma_desc_slot,
2432                                            chain_node);
2433                 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2434                 hw_desc = iter->hw_desc;
2435                 hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2436
2437                 ppc440spe_desc_set_dest_addr(iter, chan,
2438                                              DMA_CUED_XOR_BASE, dst[0], 0);
2439                 ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1);
2440                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2441                                             src[0]);
2442                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2443                 iter->unmap_len = len;
2444
2445                 /*
2446                  * Second descriptor, multiply data from the q page
2447                  * and store the result in real destination.
2448                  */
2449                 iter = list_first_entry(&iter->chain_node,
2450                                         struct ppc440spe_adma_desc_slot,
2451                                         chain_node);
2452                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2453                 iter->hw_next = NULL;
2454                 if (flags & DMA_PREP_INTERRUPT)
2455                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
2456                 else
2457                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2458
2459                 hw_desc = iter->hw_desc;
2460                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2461                 ppc440spe_desc_set_src_addr(iter, chan, 0,
2462                                             DMA_CUED_XOR_HB, dst[1]);
2463                 ppc440spe_desc_set_dest_addr(iter, chan,
2464                                              DMA_CUED_XOR_BASE, dst[0], 0);
2465
2466                 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2467                                             DMA_CDB_SG_DST1, scf[0]);
2468                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2469                 iter->unmap_len = len;
2470                 sw_desc->async_tx.flags = flags;
2471         }
2472
2473         spin_unlock_bh(&ppc440spe_chan->lock);
2474
2475         return sw_desc;
2476 }
2477
2478 /**
2479  * ppc440spe_dma01_prep_sum_product -
2480  * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also
2481  * the source.
2482  */
2483 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product(
2484                 struct ppc440spe_adma_chan *ppc440spe_chan,
2485                 dma_addr_t *dst, dma_addr_t *src, int src_cnt,
2486                 const unsigned char *scf, size_t len, unsigned long flags)
2487 {
2488         struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2489         unsigned long op = 0;
2490         int slot_cnt;
2491
2492         set_bit(PPC440SPE_DESC_WXOR, &op);
2493         slot_cnt = 3;
2494
2495         spin_lock_bh(&ppc440spe_chan->lock);
2496
2497         /* WXOR, each descriptor occupies one slot */
2498         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2499         if (sw_desc) {
2500                 struct ppc440spe_adma_chan *chan;
2501                 struct ppc440spe_adma_desc_slot *iter;
2502                 struct dma_cdb *hw_desc;
2503
2504                 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2505                 set_bits(op, &sw_desc->flags);
2506                 sw_desc->src_cnt = src_cnt;
2507                 sw_desc->dst_cnt = 1;
2508                 /* 1st descriptor, src[1] data to q page and zero destination */
2509                 iter = list_first_entry(&sw_desc->group_list,
2510                                         struct ppc440spe_adma_desc_slot,
2511                                         chain_node);
2512                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2513                 iter->hw_next = list_entry(iter->chain_node.next,
2514                                            struct ppc440spe_adma_desc_slot,
2515                                            chain_node);
2516                 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2517                 hw_desc = iter->hw_desc;
2518                 hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2519
2520                 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2521                                              *dst, 0);
2522                 ppc440spe_desc_set_dest_addr(iter, chan, 0,
2523                                              ppc440spe_chan->qdest, 1);
2524                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2525                                             src[1]);
2526                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2527                 iter->unmap_len = len;
2528
2529                 /* 2nd descriptor, multiply src[1] data and store the
2530                  * result in destination */
2531                 iter = list_first_entry(&iter->chain_node,
2532                                         struct ppc440spe_adma_desc_slot,
2533                                         chain_node);
2534                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2535                 /* set 'next' pointer */
2536                 iter->hw_next = list_entry(iter->chain_node.next,
2537                                            struct ppc440spe_adma_desc_slot,
2538                                            chain_node);
2539                 if (flags & DMA_PREP_INTERRUPT)
2540                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
2541                 else
2542                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2543
2544                 hw_desc = iter->hw_desc;
2545                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2546                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2547                                             ppc440spe_chan->qdest);
2548                 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2549                                              *dst, 0);
2550                 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2551                                             DMA_CDB_SG_DST1, scf[1]);
2552                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2553                 iter->unmap_len = len;
2554
2555                 /*
2556                  * 3rd descriptor, multiply src[0] data and xor it
2557                  * with destination
2558                  */
2559                 iter = list_first_entry(&iter->chain_node,
2560                                         struct ppc440spe_adma_desc_slot,
2561                                         chain_node);
2562                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2563                 iter->hw_next = NULL;
2564                 if (flags & DMA_PREP_INTERRUPT)
2565                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
2566                 else
2567                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2568
2569                 hw_desc = iter->hw_desc;
2570                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2571                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2572                                             src[0]);
2573                 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2574                                              *dst, 0);
2575                 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2576                                             DMA_CDB_SG_DST1, scf[0]);
2577                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2578                 iter->unmap_len = len;
2579                 sw_desc->async_tx.flags = flags;
2580         }
2581
2582         spin_unlock_bh(&ppc440spe_chan->lock);
2583
2584         return sw_desc;
2585 }
2586
2587 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq(
2588                 struct ppc440spe_adma_chan *ppc440spe_chan,
2589                 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2590                 const unsigned char *scf, size_t len, unsigned long flags)
2591 {
2592         int slot_cnt;
2593         struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2594         unsigned long op = 0;
2595         unsigned char mult = 1;
2596
2597         pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2598                  __func__, dst_cnt, src_cnt, len);
2599         /*  select operations WXOR/RXOR depending on the
2600          * source addresses of operators and the number
2601          * of destinations (RXOR support only Q-parity calculations)
2602          */
2603         set_bit(PPC440SPE_DESC_WXOR, &op);
2604         if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) {
2605                 /* no active RXOR;
2606                  * do RXOR if:
2607                  * - there are more than 1 source,
2608                  * - len is aligned on 512-byte boundary,
2609                  * - source addresses fit to one of 4 possible regions.
2610                  */
2611                 if (src_cnt > 1 &&
2612                     !(len & MQ0_CF2H_RXOR_BS_MASK) &&
2613                     (src[0] + len) == src[1]) {
2614                         /* may do RXOR R1 R2 */
2615                         set_bit(PPC440SPE_DESC_RXOR, &op);
2616                         if (src_cnt != 2) {
2617                                 /* may try to enhance region of RXOR */
2618                                 if ((src[1] + len) == src[2]) {
2619                                         /* do RXOR R1 R2 R3 */
2620                                         set_bit(PPC440SPE_DESC_RXOR123,
2621                                                 &op);
2622                                 } else if ((src[1] + len * 2) == src[2]) {
2623                                         /* do RXOR R1 R2 R4 */
2624                                         set_bit(PPC440SPE_DESC_RXOR124, &op);
2625                                 } else if ((src[1] + len * 3) == src[2]) {
2626                                         /* do RXOR R1 R2 R5 */
2627                                         set_bit(PPC440SPE_DESC_RXOR125,
2628                                                 &op);
2629                                 } else {
2630                                         /* do RXOR R1 R2 */
2631                                         set_bit(PPC440SPE_DESC_RXOR12,
2632                                                 &op);
2633                                 }
2634                         } else {
2635                                 /* do RXOR R1 R2 */
2636                                 set_bit(PPC440SPE_DESC_RXOR12, &op);
2637                         }
2638                 }
2639
2640                 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2641                         /* can not do this operation with RXOR */
2642                         clear_bit(PPC440SPE_RXOR_RUN,
2643                                 &ppc440spe_rxor_state);
2644                 } else {
2645                         /* can do; set block size right now */
2646                         ppc440spe_desc_set_rxor_block_size(len);
2647                 }
2648         }
2649
2650         /* Number of necessary slots depends on operation type selected */
2651         if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2652                 /*  This is a WXOR only chain. Need descriptors for each
2653                  * source to GF-XOR them with WXOR, and need descriptors
2654                  * for each destination to zero them with WXOR
2655                  */
2656                 slot_cnt = src_cnt;
2657
2658                 if (flags & DMA_PREP_ZERO_P) {
2659                         slot_cnt++;
2660                         set_bit(PPC440SPE_ZERO_P, &op);
2661                 }
2662                 if (flags & DMA_PREP_ZERO_Q) {
2663                         slot_cnt++;
2664                         set_bit(PPC440SPE_ZERO_Q, &op);
2665                 }
2666         } else {
2667                 /*  Need 1/2 descriptor for RXOR operation, and
2668                  * need (src_cnt - (2 or 3)) for WXOR of sources
2669                  * remained (if any)
2670                  */
2671                 slot_cnt = dst_cnt;
2672
2673                 if (flags & DMA_PREP_ZERO_P)
2674                         set_bit(PPC440SPE_ZERO_P, &op);
2675                 if (flags & DMA_PREP_ZERO_Q)
2676                         set_bit(PPC440SPE_ZERO_Q, &op);
2677
2678                 if (test_bit(PPC440SPE_DESC_RXOR12, &op))
2679                         slot_cnt += src_cnt - 2;
2680                 else
2681                         slot_cnt += src_cnt - 3;
2682
2683                 /*  Thus we have either RXOR only chain or
2684                  * mixed RXOR/WXOR
2685                  */
2686                 if (slot_cnt == dst_cnt)
2687                         /* RXOR only chain */
2688                         clear_bit(PPC440SPE_DESC_WXOR, &op);
2689         }
2690
2691         spin_lock_bh(&ppc440spe_chan->lock);
2692         /* for both RXOR/WXOR each descriptor occupies one slot */
2693         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2694         if (sw_desc) {
2695                 ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt,
2696                                 flags, op);
2697
2698                 /* setup dst/src/mult */
2699                 pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n",
2700                          __func__, dst[0], dst[1]);
2701                 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2702                 while (src_cnt--) {
2703                         ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2704                                                   src_cnt);
2705
2706                         /* NOTE: "Multi = 0 is equivalent to = 1" as it
2707                          * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf
2708                          * doesn't work for RXOR with DMA0/1! Instead, multi=0
2709                          * leads to zeroing source data after RXOR.
2710                          * So, for P case set-up mult=1 explicitly.
2711                          */
2712                         if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2713                                 mult = scf[src_cnt];
2714                         ppc440spe_adma_pq_set_src_mult(sw_desc,
2715                                 mult, src_cnt,  dst_cnt - 1);
2716                 }
2717
2718                 /* Setup byte count foreach slot just allocated */
2719                 sw_desc->async_tx.flags = flags;
2720                 list_for_each_entry(iter, &sw_desc->group_list,
2721                                 chain_node) {
2722                         ppc440spe_desc_set_byte_count(iter,
2723                                 ppc440spe_chan, len);
2724                         iter->unmap_len = len;
2725                 }
2726         }
2727         spin_unlock_bh(&ppc440spe_chan->lock);
2728
2729         return sw_desc;
2730 }
2731
2732 static struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq(
2733                 struct ppc440spe_adma_chan *ppc440spe_chan,
2734                 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2735                 const unsigned char *scf, size_t len, unsigned long flags)
2736 {
2737         int slot_cnt, descs_per_op;
2738         struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2739         unsigned long op = 0;
2740         unsigned char mult = 1;
2741
2742         BUG_ON(!dst_cnt);
2743         /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2744                  __func__, dst_cnt, src_cnt, len);*/
2745
2746         spin_lock_bh(&ppc440spe_chan->lock);
2747         descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len);
2748         if (descs_per_op < 0) {
2749                 spin_unlock_bh(&ppc440spe_chan->lock);
2750                 return NULL;
2751         }
2752
2753         /* depending on number of sources we have 1 or 2 RXOR chains */
2754         slot_cnt = descs_per_op * dst_cnt;
2755
2756         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2757         if (sw_desc) {
2758                 op = slot_cnt;
2759                 sw_desc->async_tx.flags = flags;
2760                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2761                         ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt,
2762                                 --op ? 0 : flags);
2763                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2764                                 len);
2765                         iter->unmap_len = len;
2766
2767                         ppc440spe_init_rxor_cursor(&(iter->rxor_cursor));
2768                         iter->rxor_cursor.len = len;
2769                         iter->descs_per_op = descs_per_op;
2770                 }
2771                 op = 0;
2772                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2773                         op++;
2774                         if (op % descs_per_op == 0)
2775                                 ppc440spe_adma_init_dma2rxor_slot(iter, src,
2776                                                                   src_cnt);
2777                         if (likely(!list_is_last(&iter->chain_node,
2778                                                  &sw_desc->group_list))) {
2779                                 /* set 'next' pointer */
2780                                 iter->hw_next =
2781                                         list_entry(iter->chain_node.next,
2782                                                 struct ppc440spe_adma_desc_slot,
2783                                                 chain_node);
2784                                 ppc440spe_xor_set_link(iter, iter->hw_next);
2785                         } else {
2786                                 /* this is the last descriptor. */
2787                                 iter->hw_next = NULL;
2788                         }
2789                 }
2790
2791                 /* fixup head descriptor */
2792                 sw_desc->dst_cnt = dst_cnt;
2793                 if (flags & DMA_PREP_ZERO_P)
2794                         set_bit(PPC440SPE_ZERO_P, &sw_desc->flags);
2795                 if (flags & DMA_PREP_ZERO_Q)
2796                         set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags);
2797
2798                 /* setup dst/src/mult */
2799                 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2800
2801                 while (src_cnt--) {
2802                         /* handle descriptors (if dst_cnt == 2) inside
2803                          * the ppc440spe_adma_pq_set_srcxxx() functions
2804                          */
2805                         ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2806                                                   src_cnt);
2807                         if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2808                                 mult = scf[src_cnt];
2809                         ppc440spe_adma_pq_set_src_mult(sw_desc,
2810                                         mult, src_cnt, dst_cnt - 1);
2811                 }
2812         }
2813         spin_unlock_bh(&ppc440spe_chan->lock);
2814         ppc440spe_desc_set_rxor_block_size(len);
2815         return sw_desc;
2816 }
2817
2818 /**
2819  * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation
2820  */
2821 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq(
2822                 struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
2823                 unsigned int src_cnt, const unsigned char *scf,
2824                 size_t len, unsigned long flags)
2825 {
2826         struct ppc440spe_adma_chan *ppc440spe_chan;
2827         struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2828         int dst_cnt = 0;
2829
2830         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2831
2832         ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id,
2833                                     dst, src, src_cnt));
2834         BUG_ON(!len);
2835         BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
2836         BUG_ON(!src_cnt);
2837
2838         if (src_cnt == 1 && dst[1] == src[0]) {
2839                 dma_addr_t dest[2];
2840
2841                 /* dst[1] is real destination (Q) */
2842                 dest[0] = dst[1];
2843                 /* this is the page to multicast source data to */
2844                 dest[1] = ppc440spe_chan->qdest;
2845                 sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan,
2846                                 dest, 2, src, src_cnt, scf, len, flags);
2847                 return sw_desc ? &sw_desc->async_tx : NULL;
2848         }
2849
2850         if (src_cnt == 2 && dst[1] == src[1]) {
2851                 sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan,
2852                                         &dst[1], src, 2, scf, len, flags);
2853                 return sw_desc ? &sw_desc->async_tx : NULL;
2854         }
2855
2856         if (!(flags & DMA_PREP_PQ_DISABLE_P)) {
2857                 BUG_ON(!dst[0]);
2858                 dst_cnt++;
2859                 flags |= DMA_PREP_ZERO_P;
2860         }
2861
2862         if (!(flags & DMA_PREP_PQ_DISABLE_Q)) {
2863                 BUG_ON(!dst[1]);
2864                 dst_cnt++;
2865                 flags |= DMA_PREP_ZERO_Q;
2866         }
2867
2868         BUG_ON(!dst_cnt);
2869
2870         dev_dbg(ppc440spe_chan->device->common.dev,
2871                 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2872                 ppc440spe_chan->device->id, __func__, src_cnt, len,
2873                 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2874
2875         switch (ppc440spe_chan->device->id) {
2876         case PPC440SPE_DMA0_ID:
2877         case PPC440SPE_DMA1_ID:
2878                 sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan,
2879                                 dst, dst_cnt, src, src_cnt, scf,
2880                                 len, flags);
2881                 break;
2882
2883         case PPC440SPE_XOR_ID:
2884                 sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan,
2885                                 dst, dst_cnt, src, src_cnt, scf,
2886                                 len, flags);
2887                 break;
2888         }
2889
2890         return sw_desc ? &sw_desc->async_tx : NULL;
2891 }
2892
2893 /**
2894  * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for
2895  * a PQ_ZERO_SUM operation
2896  */
2897 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum(
2898                 struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
2899                 unsigned int src_cnt, const unsigned char *scf, size_t len,
2900                 enum sum_check_flags *pqres, unsigned long flags)
2901 {
2902         struct ppc440spe_adma_chan *ppc440spe_chan;
2903         struct ppc440spe_adma_desc_slot *sw_desc, *iter;
2904         dma_addr_t pdest, qdest;
2905         int slot_cnt, slots_per_op, idst, dst_cnt;
2906
2907         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2908
2909         if (flags & DMA_PREP_PQ_DISABLE_P)
2910                 pdest = 0;
2911         else
2912                 pdest = pq[0];
2913
2914         if (flags & DMA_PREP_PQ_DISABLE_Q)
2915                 qdest = 0;
2916         else
2917                 qdest = pq[1];
2918
2919         ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id,
2920                                             src, src_cnt, scf));
2921
2922         /* Always use WXOR for P/Q calculations (two destinations).
2923          * Need 1 or 2 extra slots to verify results are zero.
2924          */
2925         idst = dst_cnt = (pdest && qdest) ? 2 : 1;
2926
2927         /* One additional slot per destination to clone P/Q
2928          * before calculation (we have to preserve destinations).
2929          */
2930         slot_cnt = src_cnt + dst_cnt * 2;
2931         slots_per_op = 1;
2932
2933         spin_lock_bh(&ppc440spe_chan->lock);
2934         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2935                                              slots_per_op);
2936         if (sw_desc) {
2937                 ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt);
2938
2939                 /* Setup byte count for each slot just allocated */
2940                 sw_desc->async_tx.flags = flags;
2941                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2942                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2943                                                       len);
2944                         iter->unmap_len = len;
2945                 }
2946
2947                 if (pdest) {
2948                         struct dma_cdb *hw_desc;
2949                         struct ppc440spe_adma_chan *chan;
2950
2951                         iter = sw_desc->group_head;
2952                         chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2953                         memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2954                         iter->hw_next = list_entry(iter->chain_node.next,
2955                                                 struct ppc440spe_adma_desc_slot,
2956                                                 chain_node);
2957                         hw_desc = iter->hw_desc;
2958                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2959                         iter->src_cnt = 0;
2960                         iter->dst_cnt = 0;
2961                         ppc440spe_desc_set_dest_addr(iter, chan, 0,
2962                                                      ppc440spe_chan->pdest, 0);
2963                         ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest);
2964                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2965                                                       len);
2966                         iter->unmap_len = 0;
2967                         /* override pdest to preserve original P */
2968                         pdest = ppc440spe_chan->pdest;
2969                 }
2970                 if (qdest) {
2971                         struct dma_cdb *hw_desc;
2972                         struct ppc440spe_adma_chan *chan;
2973
2974                         iter = list_first_entry(&sw_desc->group_list,
2975                                                 struct ppc440spe_adma_desc_slot,
2976                                                 chain_node);
2977                         chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2978
2979                         if (pdest) {
2980                                 iter = list_entry(iter->chain_node.next,
2981                                                 struct ppc440spe_adma_desc_slot,
2982                                                 chain_node);
2983                         }
2984
2985                         memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2986                         iter->hw_next = list_entry(iter->chain_node.next,
2987                                                 struct ppc440spe_adma_desc_slot,
2988                                                 chain_node);
2989                         hw_desc = iter->hw_desc;
2990                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2991                         iter->src_cnt = 0;
2992                         iter->dst_cnt = 0;
2993                         ppc440spe_desc_set_dest_addr(iter, chan, 0,
2994                                                      ppc440spe_chan->qdest, 0);
2995                         ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest);
2996                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2997                                                       len);
2998                         iter->unmap_len = 0;
2999                         /* override qdest to preserve original Q */
3000                         qdest = ppc440spe_chan->qdest;
3001                 }
3002
3003                 /* Setup destinations for P/Q ops */
3004                 ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest);
3005
3006                 /* Setup zero QWORDs into DCHECK CDBs */
3007                 idst = dst_cnt;
3008                 list_for_each_entry_reverse(iter, &sw_desc->group_list,
3009                                             chain_node) {
3010                         /*
3011                          * The last CDB corresponds to Q-parity check,
3012                          * the one before last CDB corresponds
3013                          * P-parity check
3014                          */
3015                         if (idst == DMA_DEST_MAX_NUM) {
3016                                 if (idst == dst_cnt) {
3017                                         set_bit(PPC440SPE_DESC_QCHECK,
3018                                                 &iter->flags);
3019                                 } else {
3020                                         set_bit(PPC440SPE_DESC_PCHECK,
3021                                                 &iter->flags);
3022                                 }
3023                         } else {
3024                                 if (qdest) {
3025                                         set_bit(PPC440SPE_DESC_QCHECK,
3026                                                 &iter->flags);
3027                                 } else {
3028                                         set_bit(PPC440SPE_DESC_PCHECK,
3029                                                 &iter->flags);
3030                                 }
3031                         }
3032                         iter->xor_check_result = pqres;
3033
3034                         /*
3035                          * set it to zero, if check fail then result will
3036                          * be updated
3037                          */
3038                         *iter->xor_check_result = 0;
3039                         ppc440spe_desc_set_dcheck(iter, ppc440spe_chan,
3040                                 ppc440spe_qword);
3041
3042                         if (!(--dst_cnt))
3043                                 break;
3044                 }
3045
3046                 /* Setup sources and mults for P/Q ops */
3047                 list_for_each_entry_continue_reverse(iter, &sw_desc->group_list,
3048                                                      chain_node) {
3049                         struct ppc440spe_adma_chan *chan;
3050                         u32 mult_dst;
3051
3052                         chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
3053                         ppc440spe_desc_set_src_addr(iter, chan, 0,
3054                                                     DMA_CUED_XOR_HB,
3055                                                     src[src_cnt - 1]);
3056                         if (qdest) {
3057                                 mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 :
3058                                                            DMA_CDB_SG_DST1;
3059                                 ppc440spe_desc_set_src_mult(iter, chan,
3060                                                             DMA_CUED_MULT1_OFF,
3061                                                             mult_dst,
3062                                                             scf[src_cnt - 1]);
3063                         }
3064                         if (!(--src_cnt))
3065                                 break;
3066                 }
3067         }
3068         spin_unlock_bh(&ppc440spe_chan->lock);
3069         return sw_desc ? &sw_desc->async_tx : NULL;
3070 }
3071
3072 /**
3073  * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for
3074  * XOR ZERO_SUM operation
3075  */
3076 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum(
3077                 struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt,
3078                 size_t len, enum sum_check_flags *result, unsigned long flags)
3079 {
3080         struct dma_async_tx_descriptor *tx;
3081         dma_addr_t pq[2];
3082
3083         /* validate P, disable Q */
3084         pq[0] = src[0];
3085         pq[1] = 0;
3086         flags |= DMA_PREP_PQ_DISABLE_Q;
3087
3088         tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1],
3089                                                 src_cnt - 1, 0, len,
3090                                                 result, flags);
3091         return tx;
3092 }
3093
3094 /**
3095  * ppc440spe_adma_set_dest - set destination address into descriptor
3096  */
3097 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
3098                 dma_addr_t addr, int index)
3099 {
3100         struct ppc440spe_adma_chan *chan;
3101
3102         BUG_ON(index >= sw_desc->dst_cnt);
3103
3104         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3105
3106         switch (chan->device->id) {
3107         case PPC440SPE_DMA0_ID:
3108         case PPC440SPE_DMA1_ID:
3109                 /* to do: support transfers lengths >
3110                  * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT
3111                  */
3112                 ppc440spe_desc_set_dest_addr(sw_desc->group_head,
3113                         chan, 0, addr, index);
3114                 break;
3115         case PPC440SPE_XOR_ID:
3116                 sw_desc = ppc440spe_get_group_entry(sw_desc, index);
3117                 ppc440spe_desc_set_dest_addr(sw_desc,
3118                         chan, 0, addr, index);
3119                 break;
3120         }
3121 }
3122
3123 static void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter,
3124                 struct ppc440spe_adma_chan *chan, dma_addr_t addr)
3125 {
3126         /*  To clear destinations update the descriptor
3127          * (P or Q depending on index) as follows:
3128          * addr is destination (0 corresponds to SG2):
3129          */
3130         ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0);
3131
3132         /* ... and the addr is source: */
3133         ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr);
3134
3135         /* addr is always SG2 then the mult is always DST1 */
3136         ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
3137                                     DMA_CDB_SG_DST1, 1);
3138 }
3139
3140 /**
3141  * ppc440spe_adma_pq_set_dest - set destination address into descriptor
3142  * for the PQXOR operation
3143  */
3144 static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
3145                 dma_addr_t *addrs, unsigned long flags)
3146 {
3147         struct ppc440spe_adma_desc_slot *iter;
3148         struct ppc440spe_adma_chan *chan;
3149         dma_addr_t paddr, qaddr;
3150         dma_addr_t addr = 0, ppath, qpath;
3151         int index = 0, i;
3152
3153         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3154
3155         if (flags & DMA_PREP_PQ_DISABLE_P)
3156                 paddr = 0;
3157         else
3158                 paddr = addrs[0];
3159
3160         if (flags & DMA_PREP_PQ_DISABLE_Q)
3161                 qaddr = 0;
3162         else
3163                 qaddr = addrs[1];
3164
3165         if (!paddr || !qaddr)
3166                 addr = paddr ? paddr : qaddr;
3167
3168         switch (chan->device->id) {
3169         case PPC440SPE_DMA0_ID:
3170         case PPC440SPE_DMA1_ID:
3171                 /* walk through the WXOR source list and set P/Q-destinations
3172                  * for each slot:
3173                  */
3174                 if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3175                         /* This is WXOR-only chain; may have 1/2 zero descs */
3176                         if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3177                                 index++;
3178                         if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3179                                 index++;
3180
3181                         iter = ppc440spe_get_group_entry(sw_desc, index);
3182                         if (addr) {
3183                                 /* one destination */
3184                                 list_for_each_entry_from(iter,
3185                                         &sw_desc->group_list, chain_node)
3186                                         ppc440spe_desc_set_dest_addr(iter, chan,
3187                                                 DMA_CUED_XOR_BASE, addr, 0);
3188                         } else {
3189                                 /* two destinations */
3190                                 list_for_each_entry_from(iter,
3191                                         &sw_desc->group_list, chain_node) {
3192                                         ppc440spe_desc_set_dest_addr(iter, chan,
3193                                                 DMA_CUED_XOR_BASE, paddr, 0);
3194                                         ppc440spe_desc_set_dest_addr(iter, chan,
3195                                                 DMA_CUED_XOR_BASE, qaddr, 1);
3196                                 }
3197                         }
3198
3199                         if (index) {
3200                                 /*  To clear destinations update the descriptor
3201                                  * (1st,2nd, or both depending on flags)
3202                                  */
3203                                 index = 0;
3204                                 if (test_bit(PPC440SPE_ZERO_P,
3205                                                 &sw_desc->flags)) {
3206                                         iter = ppc440spe_get_group_entry(
3207                                                         sw_desc, index++);
3208                                         ppc440spe_adma_pq_zero_op(iter, chan,
3209                                                         paddr);
3210                                 }
3211
3212                                 if (test_bit(PPC440SPE_ZERO_Q,
3213                                                 &sw_desc->flags)) {
3214                                         iter = ppc440spe_get_group_entry(
3215                                                         sw_desc, index++);
3216                                         ppc440spe_adma_pq_zero_op(iter, chan,
3217                                                         qaddr);
3218                                 }
3219
3220                                 return;
3221                         }
3222                 } else {
3223                         /* This is RXOR-only or RXOR/WXOR mixed chain */
3224
3225                         /* If we want to include destination into calculations,
3226                          * then make dest addresses cued with mult=1 (XOR).
3227                          */
3228                         ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
3229                                         DMA_CUED_XOR_HB :
3230                                         DMA_CUED_XOR_BASE |
3231                                                 (1 << DMA_CUED_MULT1_OFF);
3232                         qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
3233                                         DMA_CUED_XOR_HB :
3234                                         DMA_CUED_XOR_BASE |
3235                                                 (1 << DMA_CUED_MULT1_OFF);
3236
3237                         /* Setup destination(s) in RXOR slot(s) */
3238                         iter = ppc440spe_get_group_entry(sw_desc, index++);
3239                         ppc440spe_desc_set_dest_addr(iter, chan,
3240                                                 paddr ? ppath : qpath,
3241                                                 paddr ? paddr : qaddr, 0);
3242                         if (!addr) {
3243                                 /* two destinations */
3244                                 iter = ppc440spe_get_group_entry(sw_desc,
3245                                                                  index++);
3246                                 ppc440spe_desc_set_dest_addr(iter, chan,
3247                                                 qpath, qaddr, 0);
3248                         }
3249
3250                         if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) {
3251                                 /* Setup destination(s) in remaining WXOR
3252                                  * slots
3253                                  */
3254                                 iter = ppc440spe_get_group_entry(sw_desc,
3255                                                                  index);
3256                                 if (addr) {
3257                                         /* one destination */
3258                                         list_for_each_entry_from(iter,
3259                                             &sw_desc->group_list,
3260                                             chain_node)
3261                                                 ppc440spe_desc_set_dest_addr(
3262                                                         iter, chan,
3263                                                         DMA_CUED_XOR_BASE,
3264                                                         addr, 0);
3265
3266                                 } else {
3267                                         /* two destinations */
3268                                         list_for_each_entry_from(iter,
3269                                             &sw_desc->group_list,
3270                                             chain_node) {
3271                                                 ppc440spe_desc_set_dest_addr(
3272                                                         iter, chan,
3273                                                         DMA_CUED_XOR_BASE,
3274                                                         paddr, 0);
3275                                                 ppc440spe_desc_set_dest_addr(
3276                                                         iter, chan,
3277                                                         DMA_CUED_XOR_BASE,
3278                                                         qaddr, 1);
3279                                         }
3280                                 }
3281                         }
3282
3283                 }
3284                 break;
3285
3286         case PPC440SPE_XOR_ID:
3287                 /* DMA2 descriptors have only 1 destination, so there are
3288                  * two chains - one for each dest.
3289                  * If we want to include destination into calculations,
3290                  * then make dest addresses cued with mult=1 (XOR).
3291                  */
3292                 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
3293                                 DMA_CUED_XOR_HB :
3294                                 DMA_CUED_XOR_BASE |
3295                                         (1 << DMA_CUED_MULT1_OFF);
3296
3297                 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
3298                                 DMA_CUED_XOR_HB :
3299                                 DMA_CUED_XOR_BASE |
3300                                         (1 << DMA_CUED_MULT1_OFF);
3301
3302                 iter = ppc440spe_get_group_entry(sw_desc, 0);
3303                 for (i = 0; i < sw_desc->descs_per_op; i++) {
3304                         ppc440spe_desc_set_dest_addr(iter, chan,
3305                                 paddr ? ppath : qpath,
3306                                 paddr ? paddr : qaddr, 0);
3307                         iter = list_entry(iter->chain_node.next,
3308                                           struct ppc440spe_adma_desc_slot,
3309                                           chain_node);
3310                 }
3311
3312                 if (!addr) {
3313                         /* Two destinations; setup Q here */
3314                         iter = ppc440spe_get_group_entry(sw_desc,
3315                                 sw_desc->descs_per_op);
3316                         for (i = 0; i < sw_desc->descs_per_op; i++) {
3317                                 ppc440spe_desc_set_dest_addr(iter,
3318                                         chan, qpath, qaddr, 0);
3319                                 iter = list_entry(iter->chain_node.next,
3320                                                 struct ppc440spe_adma_desc_slot,
3321                                                 chain_node);
3322                         }
3323                 }
3324
3325                 break;
3326         }
3327 }
3328
3329 /**
3330  * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor
3331  * for the PQ_ZERO_SUM operation
3332  */
3333 static void ppc440spe_adma_pqzero_sum_set_dest(
3334                 struct ppc440spe_adma_desc_slot *sw_desc,
3335                 dma_addr_t paddr, dma_addr_t qaddr)
3336 {
3337         struct ppc440spe_adma_desc_slot *iter, *end;
3338         struct ppc440spe_adma_chan *chan;
3339         dma_addr_t addr = 0;
3340         int idx;
3341
3342         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3343
3344         /* walk through the WXOR source list and set P/Q-destinations
3345          * for each slot
3346          */
3347         idx = (paddr && qaddr) ? 2 : 1;
3348         /* set end */
3349         list_for_each_entry_reverse(end, &sw_desc->group_list,
3350                                     chain_node) {
3351                 if (!(--idx))
3352                         break;
3353         }
3354         /* set start */
3355         idx = (paddr && qaddr) ? 2 : 1;
3356         iter = ppc440spe_get_group_entry(sw_desc, idx);
3357
3358         if (paddr && qaddr) {
3359                 /* two destinations */
3360                 list_for_each_entry_from(iter, &sw_desc->group_list,
3361                                          chain_node) {
3362                         if (unlikely(iter == end))
3363                                 break;
3364                         ppc440spe_desc_set_dest_addr(iter, chan,
3365                                                 DMA_CUED_XOR_BASE, paddr, 0);
3366                         ppc440spe_desc_set_dest_addr(iter, chan,
3367                                                 DMA_CUED_XOR_BASE, qaddr, 1);
3368                 }
3369         } else {
3370                 /* one destination */
3371                 addr = paddr ? paddr : qaddr;
3372                 list_for_each_entry_from(iter, &sw_desc->group_list,
3373                                          chain_node) {
3374                         if (unlikely(iter == end))
3375                                 break;
3376                         ppc440spe_desc_set_dest_addr(iter, chan,
3377                                                 DMA_CUED_XOR_BASE, addr, 0);
3378                 }
3379         }
3380
3381         /*  The remaining descriptors are DATACHECK. These have no need in
3382          * destination. Actually, these destinations are used there
3383          * as sources for check operation. So, set addr as source.
3384          */
3385         ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr);
3386
3387         if (!addr) {
3388                 end = list_entry(end->chain_node.next,
3389                                  struct ppc440spe_adma_desc_slot, chain_node);
3390                 ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr);
3391         }
3392 }
3393
3394 /**
3395  * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor
3396  */
3397 static inline void ppc440spe_desc_set_xor_src_cnt(
3398                         struct ppc440spe_adma_desc_slot *desc,
3399                         int src_cnt)
3400 {
3401         struct xor_cb *hw_desc = desc->hw_desc;
3402
3403         hw_desc->cbc &= ~XOR_CDCR_OAC_MSK;
3404         hw_desc->cbc |= src_cnt;
3405 }
3406
3407 /**
3408  * ppc440spe_adma_pq_set_src - set source address into descriptor
3409  */
3410 static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc,
3411                 dma_addr_t addr, int index)
3412 {
3413         struct ppc440spe_adma_chan *chan;
3414         dma_addr_t haddr = 0;
3415         struct ppc440spe_adma_desc_slot *iter = NULL;
3416
3417         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3418
3419         switch (chan->device->id) {
3420         case PPC440SPE_DMA0_ID:
3421         case PPC440SPE_DMA1_ID:
3422                 /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain
3423                  */
3424                 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3425                         /* RXOR-only or RXOR/WXOR operation */
3426                         int iskip = test_bit(PPC440SPE_DESC_RXOR12,
3427                                 &sw_desc->flags) ?  2 : 3;
3428
3429                         if (index == 0) {
3430                                 /* 1st slot (RXOR) */
3431                                 /* setup sources region (R1-2-3, R1-2-4,
3432                                  * or R1-2-5)
3433                                  */
3434                                 if (test_bit(PPC440SPE_DESC_RXOR12,
3435                                                 &sw_desc->flags))
3436                                         haddr = DMA_RXOR12 <<
3437                                                 DMA_CUED_REGION_OFF;
3438                                 else if (test_bit(PPC440SPE_DESC_RXOR123,
3439                                     &sw_desc->flags))
3440                                         haddr = DMA_RXOR123 <<
3441                                                 DMA_CUED_REGION_OFF;
3442                                 else if (test_bit(PPC440SPE_DESC_RXOR124,
3443                                     &sw_desc->flags))
3444                                         haddr = DMA_RXOR124 <<
3445                                                 DMA_CUED_REGION_OFF;
3446                                 else if (test_bit(PPC440SPE_DESC_RXOR125,
3447                                     &sw_desc->flags))
3448                                         haddr = DMA_RXOR125 <<
3449                                                 DMA_CUED_REGION_OFF;
3450                                 else
3451                                         BUG();
3452                                 haddr |= DMA_CUED_XOR_BASE;
3453                                 iter = ppc440spe_get_group_entry(sw_desc, 0);
3454                         } else if (index < iskip) {
3455                                 /* 1st slot (RXOR)
3456                                  * shall actually set source address only once
3457                                  * instead of first <iskip>
3458                                  */
3459                                 iter = NULL;
3460                         } else {
3461                                 /* 2nd/3d and next slots (WXOR);
3462                                  * skip first slot with RXOR
3463                                  */
3464                                 haddr = DMA_CUED_XOR_HB;
3465                                 iter = ppc440spe_get_group_entry(sw_desc,
3466                                     index - iskip + sw_desc->dst_cnt);
3467                         }
3468                 } else {
3469                         int znum = 0;
3470
3471                         /* WXOR-only operation; skip first slots with
3472                          * zeroing destinations
3473                          */
3474                         if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3475                                 znum++;
3476                         if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3477                                 znum++;
3478
3479                         haddr = DMA_CUED_XOR_HB;
3480                         iter = ppc440spe_get_group_entry(sw_desc,
3481                                         index + znum);
3482                 }
3483
3484                 if (likely(iter)) {
3485                         ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr);
3486
3487                         if (!index &&
3488                             test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) &&
3489                             sw_desc->dst_cnt == 2) {
3490                                 /* if we have two destinations for RXOR, then
3491                                  * setup source in the second descr too
3492                                  */
3493                                 iter = ppc440spe_get_group_entry(sw_desc, 1);
3494                                 ppc440spe_desc_set_src_addr(iter, chan, 0,
3495                                         haddr, addr);
3496                         }
3497                 }
3498                 break;
3499
3500         case PPC440SPE_XOR_ID:
3501                 /* DMA2 may do Biskup */
3502                 iter = sw_desc->group_head;
3503                 if (iter->dst_cnt == 2) {
3504                         /* both P & Q calculations required; set P src here */
3505                         ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3506
3507                         /* this is for Q */
3508                         iter = ppc440spe_get_group_entry(sw_desc,
3509                                 sw_desc->descs_per_op);
3510                 }
3511                 ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3512                 break;
3513         }
3514 }
3515
3516 /**
3517  * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor
3518  */
3519 static void ppc440spe_adma_memcpy_xor_set_src(
3520                 struct ppc440spe_adma_desc_slot *sw_desc,
3521                 dma_addr_t addr, int index)
3522 {
3523         struct ppc440spe_adma_chan *chan;
3524
3525         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3526         sw_desc = sw_desc->group_head;
3527
3528         if (likely(sw_desc))
3529                 ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr);
3530 }
3531
3532 /**
3533  * ppc440spe_adma_dma2rxor_inc_addr  -
3534  */
3535 static void ppc440spe_adma_dma2rxor_inc_addr(
3536                 struct ppc440spe_adma_desc_slot *desc,
3537                 struct ppc440spe_rxor *cursor, int index, int src_cnt)
3538 {
3539         cursor->addr_count++;
3540         if (index == src_cnt - 1) {
3541                 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3542         } else if (cursor->addr_count == XOR_MAX_OPS) {
3543                 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3544                 cursor->addr_count = 0;
3545                 cursor->desc_count++;
3546         }
3547 }
3548
3549 /**
3550  * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB
3551  */
3552 static int ppc440spe_adma_dma2rxor_prep_src(
3553                 struct ppc440spe_adma_desc_slot *hdesc,
3554                 struct ppc440spe_rxor *cursor, int index,
3555                 int src_cnt, u32 addr)
3556 {
3557         int rval = 0;
3558         u32 sign;
3559         struct ppc440spe_adma_desc_slot *desc = hdesc;
3560         int i;
3561
3562         for (i = 0; i < cursor->desc_count; i++) {
3563                 desc = list_entry(hdesc->chain_node.next,
3564                                   struct ppc440spe_adma_desc_slot,
3565                                   chain_node);
3566         }
3567
3568         switch (cursor->state) {
3569         case 0:
3570                 if (addr == cursor->addrl + cursor->len) {
3571                         /* direct RXOR */
3572                         cursor->state = 1;
3573                         cursor->xor_count++;
3574                         if (index == src_cnt-1) {
3575                                 ppc440spe_rxor_set_region(desc,
3576                                         cursor->addr_count,
3577                                         DMA_RXOR12 << DMA_CUED_REGION_OFF);
3578                                 ppc440spe_adma_dma2rxor_inc_addr(
3579                                         desc, cursor, index, src_cnt);
3580                         }
3581                 } else if (cursor->addrl == addr + cursor->len) {
3582                         /* reverse RXOR */
3583                         cursor->state = 1;
3584                         cursor->xor_count++;
3585                         set_bit(cursor->addr_count, &desc->reverse_flags[0]);
3586                         if (index == src_cnt-1) {
3587                                 ppc440spe_rxor_set_region(desc,
3588                                         cursor->addr_count,
3589                                         DMA_RXOR12 << DMA_CUED_REGION_OFF);
3590                                 ppc440spe_adma_dma2rxor_inc_addr(
3591                                         desc, cursor, index, src_cnt);
3592                         }
3593                 } else {
3594                         printk(KERN_ERR "Cannot build "
3595                                 "DMA2 RXOR command block.\n");
3596                         BUG();
3597                 }
3598                 break;
3599         case 1:
3600                 sign = test_bit(cursor->addr_count,
3601                                 desc->reverse_flags)
3602                         ? -1 : 1;
3603                 if (index == src_cnt-2 || (sign == -1
3604                         && addr != cursor->addrl - 2*cursor->len)) {
3605                         cursor->state = 0;
3606                         cursor->xor_count = 1;
3607                         cursor->addrl = addr;
3608                         ppc440spe_rxor_set_region(desc,
3609                                 cursor->addr_count,
3610                                 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3611                         ppc440spe_adma_dma2rxor_inc_addr(
3612                                 desc, cursor, index, src_cnt);
3613                 } else if (addr == cursor->addrl + 2*sign*cursor->len) {
3614                         cursor->state = 2;
3615                         cursor->xor_count = 0;
3616                         ppc440spe_rxor_set_region(desc,
3617                                 cursor->addr_count,
3618                                 DMA_RXOR123 << DMA_CUED_REGION_OFF);
3619                         if (index == src_cnt-1) {
3620                                 ppc440spe_adma_dma2rxor_inc_addr(
3621                                         desc, cursor, index, src_cnt);
3622                         }
3623                 } else if (addr == cursor->addrl + 3*cursor->len) {
3624                         cursor->state = 2;
3625                         cursor->xor_count = 0;
3626                         ppc440spe_rxor_set_region(desc,
3627                                 cursor->addr_count,
3628                                 DMA_RXOR124 << DMA_CUED_REGION_OFF);
3629                         if (index == src_cnt-1) {
3630                                 ppc440spe_adma_dma2rxor_inc_addr(
3631                                         desc, cursor, index, src_cnt);
3632                         }
3633                 } else if (addr == cursor->addrl + 4*cursor->len) {
3634                         cursor->state = 2;
3635                         cursor->xor_count = 0;
3636                         ppc440spe_rxor_set_region(desc,
3637                                 cursor->addr_count,
3638                                 DMA_RXOR125 << DMA_CUED_REGION_OFF);
3639                         if (index == src_cnt-1) {
3640                                 ppc440spe_adma_dma2rxor_inc_addr(
3641                                         desc, cursor, index, src_cnt);
3642                         }
3643                 } else {
3644                         cursor->state = 0;
3645                         cursor->xor_count = 1;
3646                         cursor->addrl = addr;
3647                         ppc440spe_rxor_set_region(desc,
3648                                 cursor->addr_count,
3649                                 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3650                         ppc440spe_adma_dma2rxor_inc_addr(
3651                                 desc, cursor, index, src_cnt);
3652                 }
3653                 break;
3654         case 2:
3655                 cursor->state = 0;
3656                 cursor->addrl = addr;
3657                 cursor->xor_count++;
3658                 if (index) {
3659                         ppc440spe_adma_dma2rxor_inc_addr(
3660                                 desc, cursor, index, src_cnt);
3661                 }
3662                 break;
3663         }
3664
3665         return rval;
3666 }
3667
3668 /**
3669  * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that
3670  *      ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3671  */
3672 static void ppc440spe_adma_dma2rxor_set_src(
3673                 struct ppc440spe_adma_desc_slot *desc,
3674                 int index, dma_addr_t addr)
3675 {
3676         struct xor_cb *xcb = desc->hw_desc;
3677         int k = 0, op = 0, lop = 0;
3678
3679         /* get the RXOR operand which corresponds to index addr */
3680         while (op <= index) {
3681                 lop = op;
3682                 if (k == XOR_MAX_OPS) {
3683                         k = 0;
3684                         desc = list_entry(desc->chain_node.next,
3685                                 struct ppc440spe_adma_desc_slot, chain_node);
3686                         xcb = desc->hw_desc;
3687
3688                 }
3689                 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3690                     (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3691                         op += 2;
3692                 else
3693                         op += 3;
3694         }
3695
3696         BUG_ON(k < 1);
3697
3698         if (test_bit(k-1, desc->reverse_flags)) {
3699                 /* reverse operand order; put last op in RXOR group */
3700                 if (index == op - 1)
3701                         ppc440spe_rxor_set_src(desc, k - 1, addr);
3702         } else {
3703                 /* direct operand order; put first op in RXOR group */
3704                 if (index == lop)
3705                         ppc440spe_rxor_set_src(desc, k - 1, addr);
3706         }
3707 }
3708
3709 /**
3710  * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that
3711  *      ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3712  */
3713 static void ppc440spe_adma_dma2rxor_set_mult(
3714                 struct ppc440spe_adma_desc_slot *desc,
3715                 int index, u8 mult)
3716 {
3717         struct xor_cb *xcb = desc->hw_desc;
3718         int k = 0, op = 0, lop = 0;
3719
3720         /* get the RXOR operand which corresponds to index mult */
3721         while (op <= index) {
3722                 lop = op;
3723                 if (k == XOR_MAX_OPS) {
3724                         k = 0;
3725                         desc = list_entry(desc->chain_node.next,
3726                                           struct ppc440spe_adma_desc_slot,
3727                                           chain_node);
3728                         xcb = desc->hw_desc;
3729
3730                 }
3731                 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3732                     (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3733                         op += 2;
3734                 else
3735                         op += 3;
3736         }
3737
3738         BUG_ON(k < 1);
3739         if (test_bit(k-1, desc->reverse_flags)) {
3740                 /* reverse order */
3741                 ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult);
3742         } else {
3743                 /* direct order */
3744                 ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult);
3745         }
3746 }
3747
3748 /**
3749  * ppc440spe_init_rxor_cursor -
3750  */
3751 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor)
3752 {
3753         memset(cursor, 0, sizeof(struct ppc440spe_rxor));
3754         cursor->state = 2;
3755 }
3756
3757 /**
3758  * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into
3759  * descriptor for the PQXOR operation
3760  */
3761 static void ppc440spe_adma_pq_set_src_mult(
3762                 struct ppc440spe_adma_desc_slot *sw_desc,
3763                 unsigned char mult, int index, int dst_pos)
3764 {
3765         struct ppc440spe_adma_chan *chan;
3766         u32 mult_idx, mult_dst;
3767         struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL;
3768
3769         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3770
3771         switch (chan->device->id) {
3772         case PPC440SPE_DMA0_ID:
3773         case PPC440SPE_DMA1_ID:
3774                 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3775                         int region = test_bit(PPC440SPE_DESC_RXOR12,
3776                                         &sw_desc->flags) ? 2 : 3;
3777
3778                         if (index < region) {
3779                                 /* RXOR multipliers */
3780                                 iter = ppc440spe_get_group_entry(sw_desc,
3781                                         sw_desc->dst_cnt - 1);
3782                                 if (sw_desc->dst_cnt == 2)
3783                                         iter1 = ppc440spe_get_group_entry(
3784                                                         sw_desc, 0);
3785
3786                                 mult_idx = DMA_CUED_MULT1_OFF + (index << 3);
3787                                 mult_dst = DMA_CDB_SG_SRC;
3788                         } else {
3789                                 /* WXOR multiplier */
3790                                 iter = ppc440spe_get_group_entry(sw_desc,
3791                                                         index - region +
3792                                                         sw_desc->dst_cnt);
3793                                 mult_idx = DMA_CUED_MULT1_OFF;
3794                                 mult_dst = dst_pos ? DMA_CDB_SG_DST2 :
3795                                                      DMA_CDB_SG_DST1;
3796                         }
3797                 } else {
3798                         int znum = 0;
3799
3800                         /* WXOR-only;
3801                          * skip first slots with destinations (if ZERO_DST has
3802                          * place)
3803                          */
3804                         if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3805                                 znum++;
3806                         if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3807                                 znum++;
3808
3809                         iter = ppc440spe_get_group_entry(sw_desc, index + znum);
3810                         mult_idx = DMA_CUED_MULT1_OFF;
3811                         mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1;
3812                 }
3813
3814                 if (likely(iter)) {
3815                         ppc440spe_desc_set_src_mult(iter, chan,
3816                                 mult_idx, mult_dst, mult);
3817
3818                         if (unlikely(iter1)) {
3819                                 /* if we have two destinations for RXOR, then
3820                                  * we've just set Q mult. Set-up P now.
3821                                  */
3822                                 ppc440spe_desc_set_src_mult(iter1, chan,
3823                                         mult_idx, mult_dst, 1);
3824                         }
3825
3826                 }
3827                 break;
3828
3829         case PPC440SPE_XOR_ID:
3830                 iter = sw_desc->group_head;
3831                 if (sw_desc->dst_cnt == 2) {
3832                         /* both P & Q calculations required; set P mult here */
3833                         ppc440spe_adma_dma2rxor_set_mult(iter, index, 1);
3834
3835                         /* and then set Q mult */
3836                         iter = ppc440spe_get_group_entry(sw_desc,
3837                                sw_desc->descs_per_op);
3838                 }
3839                 ppc440spe_adma_dma2rxor_set_mult(iter, index, mult);
3840                 break;
3841         }
3842 }
3843
3844 /**
3845  * ppc440spe_adma_free_chan_resources - free the resources allocated
3846  */
3847 static void ppc440spe_adma_free_chan_resources(struct dma_chan *chan)
3848 {
3849         struct ppc440spe_adma_chan *ppc440spe_chan;
3850         struct ppc440spe_adma_desc_slot *iter, *_iter;
3851         int in_use_descs = 0;
3852
3853         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3854         ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3855
3856         spin_lock_bh(&ppc440spe_chan->lock);
3857         list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain,
3858                                         chain_node) {
3859                 in_use_descs++;
3860                 list_del(&iter->chain_node);
3861         }
3862         list_for_each_entry_safe_reverse(iter, _iter,
3863                         &ppc440spe_chan->all_slots, slot_node) {
3864                 list_del(&iter->slot_node);
3865                 kfree(iter);
3866                 ppc440spe_chan->slots_allocated--;
3867         }
3868         ppc440spe_chan->last_used = NULL;
3869
3870         dev_dbg(ppc440spe_chan->device->common.dev,
3871                 "ppc440spe adma%d %s slots_allocated %d\n",
3872                 ppc440spe_chan->device->id,
3873                 __func__, ppc440spe_chan->slots_allocated);
3874         spin_unlock_bh(&ppc440spe_chan->lock);
3875
3876         /* one is ok since we left it on there on purpose */
3877         if (in_use_descs > 1)
3878                 printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n",
3879                         in_use_descs - 1);
3880 }
3881
3882 /**
3883  * ppc440spe_adma_tx_status - poll the status of an ADMA transaction
3884  * @chan: ADMA channel handle
3885  * @cookie: ADMA transaction identifier
3886  * @txstate: a holder for the current state of the channel
3887  */
3888 static enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan,
3889                         dma_cookie_t cookie, struct dma_tx_state *txstate)
3890 {
3891         struct ppc440spe_adma_chan *ppc440spe_chan;
3892         enum dma_status ret;
3893
3894         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3895         ret = dma_cookie_status(chan, cookie, txstate);
3896         if (ret == DMA_SUCCESS)
3897                 return ret;
3898
3899         ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3900
3901         return dma_cookie_status(chan, cookie, txstate);
3902 }
3903
3904 /**
3905  * ppc440spe_adma_eot_handler - end of transfer interrupt handler
3906  */
3907 static irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data)
3908 {
3909         struct ppc440spe_adma_chan *chan = data;
3910
3911         dev_dbg(chan->device->common.dev,
3912                 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3913
3914         tasklet_schedule(&chan->irq_tasklet);
3915         ppc440spe_adma_device_clear_eot_status(chan);
3916
3917         return IRQ_HANDLED;
3918 }
3919
3920 /**
3921  * ppc440spe_adma_err_handler - DMA error interrupt handler;
3922  *      do the same things as a eot handler
3923  */
3924 static irqreturn_t ppc440spe_adma_err_handler(int irq, void *data)
3925 {
3926         struct ppc440spe_adma_chan *chan = data;
3927
3928         dev_dbg(chan->device->common.dev,
3929                 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3930
3931         tasklet_schedule(&chan->irq_tasklet);
3932         ppc440spe_adma_device_clear_eot_status(chan);
3933
3934         return IRQ_HANDLED;
3935 }
3936
3937 /**
3938  * ppc440spe_test_callback - called when test operation has been done
3939  */
3940 static void ppc440spe_test_callback(void *unused)
3941 {
3942         complete(&ppc440spe_r6_test_comp);
3943 }
3944
3945 /**
3946  * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w
3947  */
3948 static void ppc440spe_adma_issue_pending(struct dma_chan *chan)
3949 {
3950         struct ppc440spe_adma_chan *ppc440spe_chan;
3951
3952         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3953         dev_dbg(ppc440spe_chan->device->common.dev,
3954                 "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id,
3955                 __func__, ppc440spe_chan->pending);
3956
3957         if (ppc440spe_chan->pending) {
3958                 ppc440spe_chan->pending = 0;
3959                 ppc440spe_chan_append(ppc440spe_chan);
3960         }
3961 }
3962
3963 /**
3964  * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines
3965  *      use FIFOs (as opposite to chains used in XOR) so this is a XOR
3966  *      specific operation)
3967  */
3968 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan)
3969 {
3970         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
3971         dma_cookie_t cookie;
3972         int slot_cnt, slots_per_op;
3973
3974         dev_dbg(chan->device->common.dev,
3975                 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3976
3977         spin_lock_bh(&chan->lock);
3978         slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op);
3979         sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op);
3980         if (sw_desc) {
3981                 group_start = sw_desc->group_head;
3982                 list_splice_init(&sw_desc->group_list, &chan->chain);
3983                 async_tx_ack(&sw_desc->async_tx);
3984                 ppc440spe_desc_init_null_xor(group_start);
3985
3986                 cookie = dma_cookie_assign(&sw_desc->async_tx);
3987
3988                 /* initialize the completed cookie to be less than
3989                  * the most recently used cookie
3990                  */
3991                 chan->common.completed_cookie = cookie - 1;
3992
3993                 /* channel should not be busy */
3994                 BUG_ON(ppc440spe_chan_is_busy(chan));
3995
3996                 /* set the descriptor address */
3997                 ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc);
3998
3999                 /* run the descriptor */
4000                 ppc440spe_chan_run(chan);
4001         } else
4002                 printk(KERN_ERR "ppc440spe adma%d"
4003                         " failed to allocate null descriptor\n",
4004                         chan->device->id);
4005         spin_unlock_bh(&chan->lock);
4006 }
4007
4008 /**
4009  * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully.
4010  *      For this we just perform one WXOR operation with the same source
4011  *      and destination addresses, the GF-multiplier is 1; so if RAID-6
4012  *      capabilities are enabled then we'll get src/dst filled with zero.
4013  */
4014 static int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan)
4015 {
4016         struct ppc440spe_adma_desc_slot *sw_desc, *iter;
4017         struct page *pg;
4018         char *a;
4019         dma_addr_t dma_addr, addrs[2];
4020         unsigned long op = 0;
4021         int rval = 0;
4022
4023         set_bit(PPC440SPE_DESC_WXOR, &op);
4024
4025         pg = alloc_page(GFP_KERNEL);
4026         if (!pg)
4027                 return -ENOMEM;
4028
4029         spin_lock_bh(&chan->lock);
4030         sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1);
4031         if (sw_desc) {
4032                 /* 1 src, 1 dsr, int_ena, WXOR */
4033                 ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op);
4034                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
4035                         ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE);
4036                         iter->unmap_len = PAGE_SIZE;
4037                 }
4038         } else {
4039                 rval = -EFAULT;
4040                 spin_unlock_bh(&chan->lock);
4041                 goto exit;
4042         }
4043         spin_unlock_bh(&chan->lock);
4044
4045         /* Fill the test page with ones */
4046         memset(page_address(pg), 0xFF, PAGE_SIZE);
4047         dma_addr = dma_map_page(chan->device->dev, pg, 0,
4048                                 PAGE_SIZE, DMA_BIDIRECTIONAL);
4049
4050         /* Setup addresses */
4051         ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0);
4052         ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0);
4053         addrs[0] = dma_addr;
4054         addrs[1] = 0;
4055         ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q);
4056
4057         async_tx_ack(&sw_desc->async_tx);
4058         sw_desc->async_tx.callback = ppc440spe_test_callback;
4059         sw_desc->async_tx.callback_param = NULL;
4060
4061         init_completion(&ppc440spe_r6_test_comp);
4062
4063         ppc440spe_adma_tx_submit(&sw_desc->async_tx);
4064         ppc440spe_adma_issue_pending(&chan->common);
4065
4066         wait_for_completion(&ppc440spe_r6_test_comp);
4067
4068         /* Now check if the test page is zeroed */
4069         a = page_address(pg);
4070         if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) {
4071                 /* page is zero - RAID-6 enabled */
4072                 rval = 0;
4073         } else {
4074                 /* RAID-6 was not enabled */
4075                 rval = -EINVAL;
4076         }
4077 exit:
4078         __free_page(pg);
4079         return rval;
4080 }
4081
4082 static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev)
4083 {
4084         switch (adev->id) {
4085         case PPC440SPE_DMA0_ID:
4086         case PPC440SPE_DMA1_ID:
4087                 dma_cap_set(DMA_MEMCPY, adev->common.cap_mask);
4088                 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
4089                 dma_cap_set(DMA_PQ, adev->common.cap_mask);
4090                 dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask);
4091                 dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask);
4092                 break;
4093         case PPC440SPE_XOR_ID:
4094                 dma_cap_set(DMA_XOR, adev->common.cap_mask);
4095                 dma_cap_set(DMA_PQ, adev->common.cap_mask);
4096                 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
4097                 adev->common.cap_mask = adev->common.cap_mask;
4098                 break;
4099         }
4100
4101         /* Set base routines */
4102         adev->common.device_alloc_chan_resources =
4103                                 ppc440spe_adma_alloc_chan_resources;
4104         adev->common.device_free_chan_resources =
4105                                 ppc440spe_adma_free_chan_resources;
4106         adev->common.device_tx_status = ppc440spe_adma_tx_status;
4107         adev->common.device_issue_pending = ppc440spe_adma_issue_pending;
4108
4109         /* Set prep routines based on capability */
4110         if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) {
4111                 adev->common.device_prep_dma_memcpy =
4112                         ppc440spe_adma_prep_dma_memcpy;
4113         }
4114         if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) {
4115                 adev->common.max_xor = XOR_MAX_OPS;
4116                 adev->common.device_prep_dma_xor =
4117                         ppc440spe_adma_prep_dma_xor;
4118         }
4119         if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) {
4120                 switch (adev->id) {
4121                 case PPC440SPE_DMA0_ID:
4122                         dma_set_maxpq(&adev->common,
4123                                 DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0);
4124                         break;
4125                 case PPC440SPE_DMA1_ID:
4126                         dma_set_maxpq(&adev->common,
4127                                 DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0);
4128                         break;
4129                 case PPC440SPE_XOR_ID:
4130                         adev->common.max_pq = XOR_MAX_OPS * 3;
4131                         break;
4132                 }
4133                 adev->common.device_prep_dma_pq =
4134                         ppc440spe_adma_prep_dma_pq;
4135         }
4136         if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) {
4137                 switch (adev->id) {
4138                 case PPC440SPE_DMA0_ID:
4139                         adev->common.max_pq = DMA0_FIFO_SIZE /
4140                                                 sizeof(struct dma_cdb);
4141                         break;
4142                 case PPC440SPE_DMA1_ID:
4143                         adev->common.max_pq = DMA1_FIFO_SIZE /
4144                                                 sizeof(struct dma_cdb);
4145                         break;
4146                 }
4147                 adev->common.device_prep_dma_pq_val =
4148                         ppc440spe_adma_prep_dma_pqzero_sum;
4149         }
4150         if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) {
4151                 switch (adev->id) {
4152                 case PPC440SPE_DMA0_ID:
4153                         adev->common.max_xor = DMA0_FIFO_SIZE /
4154                                                 sizeof(struct dma_cdb);
4155                         break;
4156                 case PPC440SPE_DMA1_ID:
4157                         adev->common.max_xor = DMA1_FIFO_SIZE /
4158                                                 sizeof(struct dma_cdb);
4159                         break;
4160                 }
4161                 adev->common.device_prep_dma_xor_val =
4162                         ppc440spe_adma_prep_dma_xor_zero_sum;
4163         }
4164         if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) {
4165                 adev->common.device_prep_dma_interrupt =
4166                         ppc440spe_adma_prep_dma_interrupt;
4167         }
4168         pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: "
4169           "( %s%s%s%s%s%s%s)\n",
4170           dev_name(adev->dev),
4171           dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "",
4172           dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "",
4173           dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "",
4174           dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "",
4175           dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "",
4176           dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : "");
4177 }
4178
4179 static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev,
4180                                      struct ppc440spe_adma_chan *chan,
4181                                      int *initcode)
4182 {
4183         struct platform_device *ofdev;
4184         struct device_node *np;
4185         int ret;
4186
4187         ofdev = container_of(adev->dev, struct platform_device, dev);
4188         np = ofdev->dev.of_node;
4189         if (adev->id != PPC440SPE_XOR_ID) {
4190                 adev->err_irq = irq_of_parse_and_map(np, 1);
4191                 if (adev->err_irq == NO_IRQ) {
4192                         dev_warn(adev->dev, "no err irq resource?\n");
4193                         *initcode = PPC_ADMA_INIT_IRQ2;
4194                         adev->err_irq = -ENXIO;
4195                 } else
4196                         atomic_inc(&ppc440spe_adma_err_irq_ref);
4197         } else {
4198                 adev->err_irq = -ENXIO;
4199         }
4200
4201         adev->irq = irq_of_parse_and_map(np, 0);
4202         if (adev->irq == NO_IRQ) {
4203                 dev_err(adev->dev, "no irq resource\n");
4204                 *initcode = PPC_ADMA_INIT_IRQ1;
4205                 ret = -ENXIO;
4206                 goto err_irq_map;
4207         }
4208         dev_dbg(adev->dev, "irq %d, err irq %d\n",
4209                 adev->irq, adev->err_irq);
4210
4211         ret = request_irq(adev->irq, ppc440spe_adma_eot_handler,
4212                           0, dev_driver_string(adev->dev), chan);
4213         if (ret) {
4214                 dev_err(adev->dev, "can't request irq %d\n",
4215                         adev->irq);
4216                 *initcode = PPC_ADMA_INIT_IRQ1;
4217                 ret = -EIO;
4218                 goto err_req1;
4219         }
4220
4221         /* only DMA engines have a separate error IRQ
4222          * so it's Ok if err_irq < 0 in XOR engine case.
4223          */
4224         if (adev->err_irq > 0) {
4225                 /* both DMA engines share common error IRQ */
4226                 ret = request_irq(adev->err_irq,
4227                                   ppc440spe_adma_err_handler,
4228                                   IRQF_SHARED,
4229                                   dev_driver_string(adev->dev),
4230                                   chan);
4231                 if (ret) {
4232                         dev_err(adev->dev, "can't request irq %d\n",
4233                                 adev->err_irq);
4234                         *initcode = PPC_ADMA_INIT_IRQ2;
4235                         ret = -EIO;
4236                         goto err_req2;
4237                 }
4238         }
4239
4240         if (adev->id == PPC440SPE_XOR_ID) {
4241                 /* enable XOR engine interrupts */
4242                 iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
4243                             XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT,
4244                             &adev->xor_reg->ier);
4245         } else {
4246                 u32 mask, enable;
4247
4248                 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
4249                 if (!np) {
4250                         pr_err("%s: can't find I2O device tree node\n",
4251                                 __func__);
4252                         ret = -ENODEV;
4253                         goto err_req2;
4254                 }
4255                 adev->i2o_reg = of_iomap(np, 0);
4256                 if (!adev->i2o_reg) {
4257                         pr_err("%s: failed to map I2O registers\n", __func__);
4258                         of_node_put(np);
4259                         ret = -EINVAL;
4260                         goto err_req2;
4261                 }
4262                 of_node_put(np);
4263                 /* Unmask 'CS FIFO Attention' interrupts and
4264                  * enable generating interrupts on errors
4265                  */
4266                 enable = (adev->id == PPC440SPE_DMA0_ID) ?
4267                          ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
4268                          ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
4269                 mask = ioread32(&adev->i2o_reg->iopim) & enable;
4270                 iowrite32(mask, &adev->i2o_reg->iopim);
4271         }
4272         return 0;
4273
4274 err_req2:
4275         free_irq(adev->irq, chan);
4276 err_req1:
4277         irq_dispose_mapping(adev->irq);
4278 err_irq_map:
4279         if (adev->err_irq > 0) {
4280                 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref))
4281                         irq_dispose_mapping(adev->err_irq);
4282         }
4283         return ret;
4284 }
4285
4286 static void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev,
4287                                         struct ppc440spe_adma_chan *chan)
4288 {
4289         u32 mask, disable;
4290
4291         if (adev->id == PPC440SPE_XOR_ID) {
4292                 /* disable XOR engine interrupts */
4293                 mask = ioread32be(&adev->xor_reg->ier);
4294                 mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
4295                           XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT);
4296                 iowrite32be(mask, &adev->xor_reg->ier);
4297         } else {
4298                 /* disable DMAx engine interrupts */
4299                 disable = (adev->id == PPC440SPE_DMA0_ID) ?
4300                           (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
4301                           (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
4302                 mask = ioread32(&adev->i2o_reg->iopim) | disable;
4303                 iowrite32(mask, &adev->i2o_reg->iopim);
4304         }
4305         free_irq(adev->irq, chan);
4306         irq_dispose_mapping(adev->irq);
4307         if (adev->err_irq > 0) {
4308                 free_irq(adev->err_irq, chan);
4309                 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) {
4310                         irq_dispose_mapping(adev->err_irq);
4311                         iounmap(adev->i2o_reg);
4312                 }
4313         }
4314 }
4315
4316 /**
4317  * ppc440spe_adma_probe - probe the asynch device
4318  */
4319 static int ppc440spe_adma_probe(struct platform_device *ofdev)
4320 {
4321         struct device_node *np = ofdev->dev.of_node;
4322         struct resource res;
4323         struct ppc440spe_adma_device *adev;
4324         struct ppc440spe_adma_chan *chan;
4325         struct ppc_dma_chan_ref *ref, *_ref;
4326         int ret = 0, initcode = PPC_ADMA_INIT_OK;
4327         const u32 *idx;
4328         int len;
4329         void *regs;
4330         u32 id, pool_size;
4331
4332         if (of_device_is_compatible(np, "amcc,xor-accelerator")) {
4333                 id = PPC440SPE_XOR_ID;
4334                 /* As far as the XOR engine is concerned, it does not
4335                  * use FIFOs but uses linked list. So there is no dependency
4336                  * between pool size to allocate and the engine configuration.
4337                  */
4338                 pool_size = PAGE_SIZE << 1;
4339         } else {
4340                 /* it is DMA0 or DMA1 */
4341                 idx = of_get_property(np, "cell-index", &len);
4342                 if (!idx || (len != sizeof(u32))) {
4343                         dev_err(&ofdev->dev, "Device node %s has missing "
4344                                 "or invalid cell-index property\n",
4345                                 np->full_name);
4346                         return -EINVAL;
4347                 }
4348                 id = *idx;
4349                 /* DMA0,1 engines use FIFO to maintain CDBs, so we
4350                  * should allocate the pool accordingly to size of this
4351                  * FIFO. Thus, the pool size depends on the FIFO depth:
4352                  * how much CDBs pointers the FIFO may contain then so
4353                  * much CDBs we should provide in the pool.
4354                  * That is
4355                  *   CDB size = 32B;
4356                  *   CDBs number = (DMA0_FIFO_SIZE >> 3);
4357                  *   Pool size = CDBs number * CDB size =
4358                  *      = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2.
4359                  */
4360                 pool_size = (id == PPC440SPE_DMA0_ID) ?
4361                             DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4362                 pool_size <<= 2;
4363         }
4364
4365         if (of_address_to_resource(np, 0, &res)) {
4366                 dev_err(&ofdev->dev, "failed to get memory resource\n");
4367                 initcode = PPC_ADMA_INIT_MEMRES;
4368                 ret = -ENODEV;
4369                 goto out;
4370         }
4371
4372         if (!request_mem_region(res.start, resource_size(&res),
4373                                 dev_driver_string(&ofdev->dev))) {
4374                 dev_err(&ofdev->dev, "failed to request memory region %pR\n",
4375                         &res);
4376                 initcode = PPC_ADMA_INIT_MEMREG;
4377                 ret = -EBUSY;
4378                 goto out;
4379         }
4380
4381         /* create a device */
4382         adev = kzalloc(sizeof(*adev), GFP_KERNEL);
4383         if (!adev) {
4384                 dev_err(&ofdev->dev, "failed to allocate device\n");
4385                 initcode = PPC_ADMA_INIT_ALLOC;
4386                 ret = -ENOMEM;
4387                 goto err_adev_alloc;
4388         }
4389
4390         adev->id = id;
4391         adev->pool_size = pool_size;
4392         /* allocate coherent memory for hardware descriptors */
4393         adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev,
4394                                         adev->pool_size, &adev->dma_desc_pool,
4395                                         GFP_KERNEL);
4396         if (adev->dma_desc_pool_virt == NULL) {
4397                 dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent "
4398                         "memory for hardware descriptors\n",
4399                         adev->pool_size);
4400                 initcode = PPC_ADMA_INIT_COHERENT;
4401                 ret = -ENOMEM;
4402                 goto err_dma_alloc;
4403         }
4404         dev_dbg(&ofdev->dev, "allocated descriptor pool virt 0x%p phys 0x%llx\n",
4405                 adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool);
4406
4407         regs = ioremap(res.start, resource_size(&res));
4408         if (!regs) {
4409                 dev_err(&ofdev->dev, "failed to ioremap regs!\n");
4410                 goto err_regs_alloc;
4411         }
4412
4413         if (adev->id == PPC440SPE_XOR_ID) {
4414                 adev->xor_reg = regs;
4415                 /* Reset XOR */
4416                 iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr);
4417                 iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr);
4418         } else {
4419                 size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ?
4420                                    DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4421                 adev->dma_reg = regs;
4422                 /* DMAx_FIFO_SIZE is defined in bytes,
4423                  * <fsiz> - is defined in number of CDB pointers (8byte).
4424                  * DMA FIFO Length = CSlength + CPlength, where
4425                  * CSlength = CPlength = (fsiz + 1) * 8.
4426                  */
4427                 iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2),
4428                           &adev->dma_reg->fsiz);
4429                 /* Configure DMA engine */
4430                 iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN,
4431                           &adev->dma_reg->cfg);
4432                 /* Clear Status */
4433                 iowrite32(~0, &adev->dma_reg->dsts);
4434         }
4435
4436         adev->dev = &ofdev->dev;
4437         adev->common.dev = &ofdev->dev;
4438         INIT_LIST_HEAD(&adev->common.channels);
4439         platform_set_drvdata(ofdev, adev);
4440
4441         /* create a channel */
4442         chan = kzalloc(sizeof(*chan), GFP_KERNEL);
4443         if (!chan) {
4444                 dev_err(&ofdev->dev, "can't allocate channel structure\n");
4445                 initcode = PPC_ADMA_INIT_CHANNEL;
4446                 ret = -ENOMEM;
4447                 goto err_chan_alloc;
4448         }
4449
4450         spin_lock_init(&chan->lock);
4451         INIT_LIST_HEAD(&chan->chain);
4452         INIT_LIST_HEAD(&chan->all_slots);
4453         chan->device = adev;
4454         chan->common.device = &adev->common;
4455         dma_cookie_init(&chan->common);
4456         list_add_tail(&chan->common.device_node, &adev->common.channels);
4457         tasklet_init(&chan->irq_tasklet, ppc440spe_adma_tasklet,
4458                      (unsigned long)chan);
4459
4460         /* allocate and map helper pages for async validation or
4461          * async_mult/async_sum_product operations on DMA0/1.
4462          */
4463         if (adev->id != PPC440SPE_XOR_ID) {
4464                 chan->pdest_page = alloc_page(GFP_KERNEL);
4465                 chan->qdest_page = alloc_page(GFP_KERNEL);
4466                 if (!chan->pdest_page ||
4467                     !chan->qdest_page) {
4468                         if (chan->pdest_page)
4469                                 __free_page(chan->pdest_page);
4470                         if (chan->qdest_page)
4471                                 __free_page(chan->qdest_page);
4472                         ret = -ENOMEM;
4473                         goto err_page_alloc;
4474                 }
4475                 chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0,
4476                                            PAGE_SIZE, DMA_BIDIRECTIONAL);
4477                 chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0,
4478                                            PAGE_SIZE, DMA_BIDIRECTIONAL);
4479         }
4480
4481         ref = kmalloc(sizeof(*ref), GFP_KERNEL);
4482         if (ref) {
4483                 ref->chan = &chan->common;
4484                 INIT_LIST_HEAD(&ref->node);
4485                 list_add_tail(&ref->node, &ppc440spe_adma_chan_list);
4486         } else {
4487                 dev_err(&ofdev->dev, "failed to allocate channel reference!\n");
4488                 ret = -ENOMEM;
4489                 goto err_ref_alloc;
4490         }
4491
4492         ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode);
4493         if (ret)
4494                 goto err_irq;
4495
4496         ppc440spe_adma_init_capabilities(adev);
4497
4498         ret = dma_async_device_register(&adev->common);
4499         if (ret) {
4500                 initcode = PPC_ADMA_INIT_REGISTER;
4501                 dev_err(&ofdev->dev, "failed to register dma device\n");
4502                 goto err_dev_reg;
4503         }
4504
4505         goto out;
4506
4507 err_dev_reg:
4508         ppc440spe_adma_release_irqs(adev, chan);
4509 err_irq:
4510         list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) {
4511                 if (chan == to_ppc440spe_adma_chan(ref->chan)) {
4512                         list_del(&ref->node);
4513                         kfree(ref);
4514                 }
4515         }
4516 err_ref_alloc:
4517         if (adev->id != PPC440SPE_XOR_ID) {
4518                 dma_unmap_page(&ofdev->dev, chan->pdest,
4519                                PAGE_SIZE, DMA_BIDIRECTIONAL);
4520                 dma_unmap_page(&ofdev->dev, chan->qdest,
4521                                PAGE_SIZE, DMA_BIDIRECTIONAL);
4522                 __free_page(chan->pdest_page);
4523                 __free_page(chan->qdest_page);
4524         }
4525 err_page_alloc:
4526         kfree(chan);
4527 err_chan_alloc:
4528         if (adev->id == PPC440SPE_XOR_ID)
4529                 iounmap(adev->xor_reg);
4530         else
4531                 iounmap(adev->dma_reg);
4532 err_regs_alloc:
4533         dma_free_coherent(adev->dev, adev->pool_size,
4534                           adev->dma_desc_pool_virt,
4535                           adev->dma_desc_pool);
4536 err_dma_alloc:
4537         kfree(adev);
4538 err_adev_alloc:
4539         release_mem_region(res.start, resource_size(&res));
4540 out:
4541         if (id < PPC440SPE_ADMA_ENGINES_NUM)
4542                 ppc440spe_adma_devices[id] = initcode;
4543
4544         return ret;
4545 }
4546
4547 /**
4548  * ppc440spe_adma_remove - remove the asynch device
4549  */
4550 static int ppc440spe_adma_remove(struct platform_device *ofdev)
4551 {
4552         struct ppc440spe_adma_device *adev = platform_get_drvdata(ofdev);
4553         struct device_node *np = ofdev->dev.of_node;
4554         struct resource res;
4555         struct dma_chan *chan, *_chan;
4556         struct ppc_dma_chan_ref *ref, *_ref;
4557         struct ppc440spe_adma_chan *ppc440spe_chan;
4558
4559         if (adev->id < PPC440SPE_ADMA_ENGINES_NUM)
4560                 ppc440spe_adma_devices[adev->id] = -1;
4561
4562         dma_async_device_unregister(&adev->common);
4563
4564         list_for_each_entry_safe(chan, _chan, &adev->common.channels,
4565                                  device_node) {
4566                 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
4567                 ppc440spe_adma_release_irqs(adev, ppc440spe_chan);
4568                 tasklet_kill(&ppc440spe_chan->irq_tasklet);
4569                 if (adev->id != PPC440SPE_XOR_ID) {
4570                         dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest,
4571                                         PAGE_SIZE, DMA_BIDIRECTIONAL);
4572                         dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest,
4573                                         PAGE_SIZE, DMA_BIDIRECTIONAL);
4574                         __free_page(ppc440spe_chan->pdest_page);
4575                         __free_page(ppc440spe_chan->qdest_page);
4576                 }
4577                 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list,
4578                                          node) {
4579                         if (ppc440spe_chan ==
4580                             to_ppc440spe_adma_chan(ref->chan)) {
4581                                 list_del(&ref->node);
4582                                 kfree(ref);
4583                         }
4584                 }
4585                 list_del(&chan->device_node);
4586                 kfree(ppc440spe_chan);
4587         }
4588
4589         dma_free_coherent(adev->dev, adev->pool_size,
4590                           adev->dma_desc_pool_virt, adev->dma_desc_pool);
4591         if (adev->id == PPC440SPE_XOR_ID)
4592                 iounmap(adev->xor_reg);
4593         else
4594                 iounmap(adev->dma_reg);
4595         of_address_to_resource(np, 0, &res);
4596         release_mem_region(res.start, resource_size(&res));
4597         kfree(adev);
4598         return 0;
4599 }
4600
4601 /*
4602  * /sys driver interface to enable h/w RAID-6 capabilities
4603  * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/
4604  * directory are "devices", "enable" and "poly".
4605  * "devices" shows available engines.
4606  * "enable" is used to enable RAID-6 capabilities or to check
4607  * whether these has been activated.
4608  * "poly" allows setting/checking used polynomial (for PPC440SPe only).
4609  */
4610
4611 static ssize_t show_ppc440spe_devices(struct device_driver *dev, char *buf)
4612 {
4613         ssize_t size = 0;
4614         int i;
4615
4616         for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) {
4617                 if (ppc440spe_adma_devices[i] == -1)
4618                         continue;
4619                 size += snprintf(buf + size, PAGE_SIZE - size,
4620                                  "PPC440SP(E)-ADMA.%d: %s\n", i,
4621                                  ppc_adma_errors[ppc440spe_adma_devices[i]]);
4622         }
4623         return size;
4624 }
4625
4626 static ssize_t show_ppc440spe_r6enable(struct device_driver *dev, char *buf)
4627 {
4628         return snprintf(buf, PAGE_SIZE,
4629                         "PPC440SP(e) RAID-6 capabilities are %sABLED.\n",
4630                         ppc440spe_r6_enabled ? "EN" : "DIS");
4631 }
4632
4633 static ssize_t store_ppc440spe_r6enable(struct device_driver *dev,
4634                                         const char *buf, size_t count)
4635 {
4636         unsigned long val;
4637
4638         if (!count || count > 11)
4639                 return -EINVAL;
4640
4641         if (!ppc440spe_r6_tchan)
4642                 return -EFAULT;
4643
4644         /* Write a key */
4645         sscanf(buf, "%lx", &val);
4646         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val);
4647         isync();
4648
4649         /* Verify whether it really works now */
4650         if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) {
4651                 pr_info("PPC440SP(e) RAID-6 has been activated "
4652                         "successfully\n");
4653                 ppc440spe_r6_enabled = 1;
4654         } else {
4655                 pr_info("PPC440SP(e) RAID-6 hasn't been activated!"
4656                         " Error key ?\n");
4657                 ppc440spe_r6_enabled = 0;
4658         }
4659         return count;
4660 }
4661
4662 static ssize_t show_ppc440spe_r6poly(struct device_driver *dev, char *buf)
4663 {
4664         ssize_t size = 0;
4665         u32 reg;
4666
4667 #ifdef CONFIG_440SP
4668         /* 440SP has fixed polynomial */
4669         reg = 0x4d;
4670 #else
4671         reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4672         reg >>= MQ0_CFBHL_POLY;
4673         reg &= 0xFF;
4674 #endif
4675
4676         size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver "
4677                         "uses 0x1%02x polynomial.\n", reg);
4678         return size;
4679 }
4680
4681 static ssize_t store_ppc440spe_r6poly(struct device_driver *dev,
4682                                       const char *buf, size_t count)
4683 {
4684         unsigned long reg, val;
4685
4686 #ifdef CONFIG_440SP
4687         /* 440SP uses default 0x14D polynomial only */
4688         return -EINVAL;
4689 #endif
4690
4691         if (!count || count > 6)
4692                 return -EINVAL;
4693
4694         /* e.g., 0x14D or 0x11D */
4695         sscanf(buf, "%lx", &val);
4696
4697         if (val & ~0x1FF)
4698                 return -EINVAL;
4699
4700         val &= 0xFF;
4701         reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4702         reg &= ~(0xFF << MQ0_CFBHL_POLY);
4703         reg |= val << MQ0_CFBHL_POLY;
4704         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg);
4705
4706         return count;
4707 }
4708
4709 static DRIVER_ATTR(devices, S_IRUGO, show_ppc440spe_devices, NULL);
4710 static DRIVER_ATTR(enable, S_IRUGO | S_IWUSR, show_ppc440spe_r6enable,
4711                    store_ppc440spe_r6enable);
4712 static DRIVER_ATTR(poly, S_IRUGO | S_IWUSR, show_ppc440spe_r6poly,
4713                    store_ppc440spe_r6poly);
4714
4715 /*
4716  * Common initialisation for RAID engines; allocate memory for
4717  * DMAx FIFOs, perform configuration common for all DMA engines.
4718  * Further DMA engine specific configuration is done at probe time.
4719  */
4720 static int ppc440spe_configure_raid_devices(void)
4721 {
4722         struct device_node *np;
4723         struct resource i2o_res;
4724         struct i2o_regs __iomem *i2o_reg;
4725         dcr_host_t i2o_dcr_host;
4726         unsigned int dcr_base, dcr_len;
4727         int i, ret;
4728
4729         np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
4730         if (!np) {
4731                 pr_err("%s: can't find I2O device tree node\n",
4732                         __func__);
4733                 return -ENODEV;
4734         }
4735
4736         if (of_address_to_resource(np, 0, &i2o_res)) {
4737                 of_node_put(np);
4738                 return -EINVAL;
4739         }
4740
4741         i2o_reg = of_iomap(np, 0);
4742         if (!i2o_reg) {
4743                 pr_err("%s: failed to map I2O registers\n", __func__);
4744                 of_node_put(np);
4745                 return -EINVAL;
4746         }
4747
4748         /* Get I2O DCRs base */
4749         dcr_base = dcr_resource_start(np, 0);
4750         dcr_len = dcr_resource_len(np, 0);
4751         if (!dcr_base && !dcr_len) {
4752                 pr_err("%s: can't get DCR registers base/len!\n",
4753                         np->full_name);
4754                 of_node_put(np);
4755                 iounmap(i2o_reg);
4756                 return -ENODEV;
4757         }
4758
4759         i2o_dcr_host = dcr_map(np, dcr_base, dcr_len);
4760         if (!DCR_MAP_OK(i2o_dcr_host)) {
4761                 pr_err("%s: failed to map DCRs!\n", np->full_name);
4762                 of_node_put(np);
4763                 iounmap(i2o_reg);
4764                 return -ENODEV;
4765         }
4766         of_node_put(np);
4767
4768         /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share
4769          * the base address of FIFO memory space.
4770          * Actually we need twice more physical memory than programmed in the
4771          * <fsiz> register (because there are two FIFOs for each DMA: CP and CS)
4772          */
4773         ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1,
4774                                          GFP_KERNEL);
4775         if (!ppc440spe_dma_fifo_buf) {
4776                 pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__);
4777                 iounmap(i2o_reg);
4778                 dcr_unmap(i2o_dcr_host, dcr_len);
4779                 return -ENOMEM;
4780         }
4781
4782         /*
4783          * Configure h/w
4784          */
4785         /* Reset I2O/DMA */
4786         mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA);
4787         mtdcri(SDR0, DCRN_SDR0_SRST, 0);
4788
4789         /* Setup the base address of mmaped registers */
4790         dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32));
4791         dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) |
4792                                                 I2O_REG_ENABLE);
4793         dcr_unmap(i2o_dcr_host, dcr_len);
4794
4795         /* Setup FIFO memory space base address */
4796         iowrite32(0, &i2o_reg->ifbah);
4797         iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal);
4798
4799         /* set zero FIFO size for I2O, so the whole
4800          * ppc440spe_dma_fifo_buf is used by DMAs.
4801          * DMAx_FIFOs will be configured while probe.
4802          */
4803         iowrite32(0, &i2o_reg->ifsiz);
4804         iounmap(i2o_reg);
4805
4806         /* To prepare WXOR/RXOR functionality we need access to
4807          * Memory Queue Module DCRs (finally it will be enabled
4808          * via /sys interface of the ppc440spe ADMA driver).
4809          */
4810         np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe");
4811         if (!np) {
4812                 pr_err("%s: can't find MQ device tree node\n",
4813                         __func__);
4814                 ret = -ENODEV;
4815                 goto out_free;
4816         }
4817
4818         /* Get MQ DCRs base */
4819         dcr_base = dcr_resource_start(np, 0);
4820         dcr_len = dcr_resource_len(np, 0);
4821         if (!dcr_base && !dcr_len) {
4822                 pr_err("%s: can't get DCR registers base/len!\n",
4823                         np->full_name);
4824                 ret = -ENODEV;
4825                 goto out_mq;
4826         }
4827
4828         ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len);
4829         if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) {
4830                 pr_err("%s: failed to map DCRs!\n", np->full_name);
4831                 ret = -ENODEV;
4832                 goto out_mq;
4833         }
4834         of_node_put(np);
4835         ppc440spe_mq_dcr_len = dcr_len;
4836
4837         /* Set HB alias */
4838         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB);
4839
4840         /* Set:
4841          * - LL transaction passing limit to 1;
4842          * - Memory controller cycle limit to 1;
4843          * - Galois Polynomial to 0x14d (default)
4844          */
4845         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL,
4846                   (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) |
4847                   (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY));
4848
4849         atomic_set(&ppc440spe_adma_err_irq_ref, 0);
4850         for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++)
4851                 ppc440spe_adma_devices[i] = -1;
4852
4853         return 0;
4854
4855 out_mq:
4856         of_node_put(np);
4857 out_free:
4858         kfree(ppc440spe_dma_fifo_buf);
4859         return ret;
4860 }
4861
4862 static const struct of_device_id ppc440spe_adma_of_match[] = {
4863         { .compatible   = "ibm,dma-440spe", },
4864         { .compatible   = "amcc,xor-accelerator", },
4865         {},
4866 };
4867 MODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match);
4868
4869 static struct platform_driver ppc440spe_adma_driver = {
4870         .probe = ppc440spe_adma_probe,
4871         .remove = ppc440spe_adma_remove,
4872         .driver = {
4873                 .name = "PPC440SP(E)-ADMA",
4874                 .owner = THIS_MODULE,
4875                 .of_match_table = ppc440spe_adma_of_match,
4876         },
4877 };
4878
4879 static __init int ppc440spe_adma_init(void)
4880 {
4881         int ret;
4882
4883         ret = ppc440spe_configure_raid_devices();
4884         if (ret)
4885                 return ret;
4886
4887         ret = platform_driver_register(&ppc440spe_adma_driver);
4888         if (ret) {
4889                 pr_err("%s: failed to register platform driver\n",
4890                         __func__);
4891                 goto out_reg;
4892         }
4893
4894         /* Initialization status */
4895         ret = driver_create_file(&ppc440spe_adma_driver.driver,
4896                                  &driver_attr_devices);
4897         if (ret)
4898                 goto out_dev;
4899
4900         /* RAID-6 h/w enable entry */
4901         ret = driver_create_file(&ppc440spe_adma_driver.driver,
4902                                  &driver_attr_enable);
4903         if (ret)
4904                 goto out_en;
4905
4906         /* GF polynomial to use */
4907         ret = driver_create_file(&ppc440spe_adma_driver.driver,
4908                                  &driver_attr_poly);
4909         if (!ret)
4910                 return ret;
4911
4912         driver_remove_file(&ppc440spe_adma_driver.driver,
4913                            &driver_attr_enable);
4914 out_en:
4915         driver_remove_file(&ppc440spe_adma_driver.driver,
4916                            &driver_attr_devices);
4917 out_dev:
4918         /* User will not be able to enable h/w RAID-6 */
4919         pr_err("%s: failed to create RAID-6 driver interface\n",
4920                 __func__);
4921         platform_driver_unregister(&ppc440spe_adma_driver);
4922 out_reg:
4923         dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
4924         kfree(ppc440spe_dma_fifo_buf);
4925         return ret;
4926 }
4927
4928 static void __exit ppc440spe_adma_exit(void)
4929 {
4930         driver_remove_file(&ppc440spe_adma_driver.driver,
4931                            &driver_attr_poly);
4932         driver_remove_file(&ppc440spe_adma_driver.driver,
4933                            &driver_attr_enable);
4934         driver_remove_file(&ppc440spe_adma_driver.driver,
4935                            &driver_attr_devices);
4936         platform_driver_unregister(&ppc440spe_adma_driver);
4937         dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
4938         kfree(ppc440spe_dma_fifo_buf);
4939 }
4940
4941 arch_initcall(ppc440spe_adma_init);
4942 module_exit(ppc440spe_adma_exit);
4943
4944 MODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>");
4945 MODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver");
4946 MODULE_LICENSE("GPL");