drivers/crypto/talitos.c

   1 /*
   2  * talitos - Freescale Integrated Security Engine (SEC) device driver
   3  *
   4  * Copyright (c) 2008 Freescale Semiconductor, Inc.
   5  *
   6  * Scatterlist Crypto API glue code copied from files with the following:
   7  * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
   8  *
   9  * Crypto algorithm registration code copied from hifn driver:
  10  * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
  11  * All rights reserved.
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License as published by
  15  * the Free Software Foundation; either version 2 of the License, or
  16  * (at your option) any later version.
  17  *
  18  * This program is distributed in the hope that it will be useful,
  19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21  * GNU General Public License for more details.
  22  *
  23  * You should have received a copy of the GNU General Public License
  24  * along with this program; if not, write to the Free Software
  25  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  26  */
  27
  28 #include <linux/kernel.h>
  29 #include <linux/module.h>
  30 #include <linux/mod_devicetable.h>
  31 #include <linux/device.h>
  32 #include <linux/interrupt.h>
  33 #include <linux/crypto.h>
  34 #include <linux/hw_random.h>
  35 #include <linux/of_platform.h>
  36 #include <linux/dma-mapping.h>
  37 #include <linux/io.h>
  38 #include <linux/spinlock.h>
  39 #include <linux/rtnetlink.h>
  40
  41 #include <crypto/algapi.h>
  42 #include <crypto/aes.h>
  43 #include <crypto/des.h>
  44 #include <crypto/sha.h>
  45 #include <crypto/aead.h>
  46 #include <crypto/authenc.h>
  47
  48 #include "talitos.h"
  49
  50 #define TALITOS_TIMEOUT 100000
  51 #define TALITOS_MAX_DATA_LEN 65535
  52
  53 #define DESC_TYPE(desc_hdr) ((be32_to_cpu(desc_hdr) >> 3) & 0x1f)
  54 #define PRIMARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 28) & 0xf)
  55 #define SECONDARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 16) & 0xf)
  56
  57 /* descriptor pointer entry */
  58 struct talitos_ptr {
  59         __be16 len;     /* length */
  60         u8 j_extent;    /* jump to sg link table and/or extent */
  61         u8 eptr;        /* extended address */
  62         __be32 ptr;     /* address */
  63 };
  64
  65 /* descriptor */
  66 struct talitos_desc {
  67         __be32 hdr;                     /* header high bits */
  68         __be32 hdr_lo;                  /* header low bits */
  69         struct talitos_ptr ptr[7];      /* ptr/len pair array */
  70 };
  71
  72 /**
  73  * talitos_request - descriptor submission request
  74  * @desc: descriptor pointer (kernel virtual)
  75  * @dma_desc: descriptor's physical bus address
  76  * @callback: whom to call when descriptor processing is done
  77  * @context: caller context (optional)
  78  */
  79 struct talitos_request {
  80         struct talitos_desc *desc;
  81         dma_addr_t dma_desc;
  82         void (*callback) (struct device *dev, struct talitos_desc *desc,
  83                           void *context, int error);
  84         void *context;
  85 };
  86
  87 struct talitos_private {
  88         struct device *dev;
  89         struct of_device *ofdev;
  90         void __iomem *reg;
  91         int irq;
  92
  93         /* SEC version geometry (from device tree node) */
  94         unsigned int num_channels;
  95         unsigned int chfifo_len;
  96         unsigned int exec_units;
  97         unsigned int desc_types;
  98
  99         /* SEC Compatibility info */
 100         unsigned long features;
 101
 102         /* next channel to be assigned next incoming descriptor */
 103         atomic_t last_chan;
 104
 105         /* per-channel number of requests pending in channel h/w fifo */
 106         atomic_t *submit_count;
 107
 108         /* per-channel request fifo */
 109         struct talitos_request **fifo;
 110
 111         /*
 112          * length of the request fifo
 113          * fifo_len is chfifo_len rounded up to next power of 2
 114          * so we can use bitwise ops to wrap
 115          */
 116         unsigned int fifo_len;
 117
 118         /* per-channel index to next free descriptor request */
 119         int *head;
 120
 121         /* per-channel index to next in-progress/done descriptor request */
 122         int *tail;
 123
 124         /* per-channel request submission (head) and release (tail) locks */
 125         spinlock_t *head_lock;
 126         spinlock_t *tail_lock;
 127
 128         /* request callback tasklet */
 129         struct tasklet_struct done_task;
 130         struct tasklet_struct error_task;
 131
 132         /* list of registered algorithms */
 133         struct list_head alg_list;
 134
 135         /* hwrng device */
 136         struct hwrng rng;
 137 };
 138
 139 /* .features flag */
 140 #define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001
 141
 142 /*
 143  * map virtual single (contiguous) pointer to h/w descriptor pointer
 144  */
 145 static void map_single_talitos_ptr(struct device *dev,
 146                                    struct talitos_ptr *talitos_ptr,
 147                                    unsigned short len, void *data,
 148                                    unsigned char extent,
 149                                    enum dma_data_direction dir)
 150 {
 151         talitos_ptr->len = cpu_to_be16(len);
 152         talitos_ptr->ptr = cpu_to_be32(dma_map_single(dev, data, len, dir));
 153         talitos_ptr->j_extent = extent;
 154 }
 155
 156 /*
 157  * unmap bus single (contiguous) h/w descriptor pointer
 158  */
 159 static void unmap_single_talitos_ptr(struct device *dev,
 160                                      struct talitos_ptr *talitos_ptr,
 161                                      enum dma_data_direction dir)
 162 {
 163         dma_unmap_single(dev, be32_to_cpu(talitos_ptr->ptr),
 164                          be16_to_cpu(talitos_ptr->len), dir);
 165 }
 166
 167 static int reset_channel(struct device *dev, int ch)
 168 {
 169         struct talitos_private *priv = dev_get_drvdata(dev);
 170         unsigned int timeout = TALITOS_TIMEOUT;
 171
 172         setbits32(priv->reg + TALITOS_CCCR(ch), TALITOS_CCCR_RESET);
 173
 174         while ((in_be32(priv->reg + TALITOS_CCCR(ch)) & TALITOS_CCCR_RESET)
 175                && --timeout)
 176                 cpu_relax();
 177
 178         if (timeout == 0) {
 179                 dev_err(dev, "failed to reset channel %d\n", ch);
 180                 return -EIO;
 181         }
 182
 183         /* set done writeback and IRQ */
 184         setbits32(priv->reg + TALITOS_CCCR_LO(ch), TALITOS_CCCR_LO_CDWE |
 185                   TALITOS_CCCR_LO_CDIE);
 186
 187         return 0;
 188 }
 189
 190 static int reset_device(struct device *dev)
 191 {
 192         struct talitos_private *priv = dev_get_drvdata(dev);
 193         unsigned int timeout = TALITOS_TIMEOUT;
 194
 195         setbits32(priv->reg + TALITOS_MCR, TALITOS_MCR_SWR);
 196
 197         while ((in_be32(priv->reg + TALITOS_MCR) & TALITOS_MCR_SWR)
 198                && --timeout)
 199                 cpu_relax();
 200
 201         if (timeout == 0) {
 202                 dev_err(dev, "failed to reset device\n");
 203                 return -EIO;
 204         }
 205
 206         return 0;
 207 }
 208
 209 /*
 210  * Reset and initialize the device
 211  */
 212 static int init_device(struct device *dev)
 213 {
 214         struct talitos_private *priv = dev_get_drvdata(dev);
 215         int ch, err;
 216
 217         /*
 218          * Master reset
 219          * errata documentation: warning: certain SEC interrupts
 220          * are not fully cleared by writing the MCR:SWR bit,
 221          * set bit twice to completely reset
 222          */
 223         err = reset_device(dev);
 224         if (err)
 225                 return err;
 226
 227         err = reset_device(dev);
 228         if (err)
 229                 return err;
 230
 231         /* reset channels */
 232         for (ch = 0; ch < priv->num_channels; ch++) {
 233                 err = reset_channel(dev, ch);
 234                 if (err)
 235                         return err;
 236         }
 237
 238         /* enable channel done and error interrupts */
 239         setbits32(priv->reg + TALITOS_IMR, TALITOS_IMR_INIT);
 240         setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT);
 241
 242         return 0;
 243 }
 244
 245 /**
 246  * talitos_submit - submits a descriptor to the device for processing
 247  * @dev:        the SEC device to be used
 248  * @desc:       the descriptor to be processed by the device
 249  * @callback:   whom to call when processing is complete
 250  * @context:    a handle for use by caller (optional)
 251  *
 252  * desc must contain valid dma-mapped (bus physical) address pointers.
 253  * callback must check err and feedback in descriptor header
 254  * for device processing status.
 255  */
 256 static int talitos_submit(struct device *dev, struct talitos_desc *desc,
 257                           void (*callback)(struct device *dev,
 258                                            struct talitos_desc *desc,
 259                                            void *context, int error),
 260                           void *context)
 261 {
 262         struct talitos_private *priv = dev_get_drvdata(dev);
 263         struct talitos_request *request;
 264         unsigned long flags, ch;
 265         int head;
 266
 267         /* select done notification */
 268         desc->hdr |= DESC_HDR_DONE_NOTIFY;
 269
 270         /* emulate SEC's round-robin channel fifo polling scheme */
 271         ch = atomic_inc_return(&priv->last_chan) & (priv->num_channels - 1);
 272
 273         spin_lock_irqsave(&priv->head_lock[ch], flags);
 274
 275         if (!atomic_inc_not_zero(&priv->submit_count[ch])) {
 276                 /* h/w fifo is full */
 277                 spin_unlock_irqrestore(&priv->head_lock[ch], flags);
 278                 return -EAGAIN;
 279         }
 280
 281         head = priv->head[ch];
 282         request = &priv->fifo[ch][head];
 283
 284         /* map descriptor and save caller data */
 285         request->dma_desc = dma_map_single(dev, desc, sizeof(*desc),
 286                                            DMA_BIDIRECTIONAL);
 287         request->callback = callback;
 288         request->context = context;
 289
 290         /* increment fifo head */
 291         priv->head[ch] = (priv->head[ch] + 1) & (priv->fifo_len - 1);
 292
 293         smp_wmb();
 294         request->desc = desc;
 295
 296         /* GO! */
 297         wmb();
 298         out_be32(priv->reg + TALITOS_FF_LO(ch), request->dma_desc);
 299
 300         spin_unlock_irqrestore(&priv->head_lock[ch], flags);
 301
 302         return -EINPROGRESS;
 303 }
 304
 305 /*
 306  * process what was done, notify callback of error if not
 307  */
 308 static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
 309 {
 310         struct talitos_private *priv = dev_get_drvdata(dev);
 311         struct talitos_request *request, saved_req;
 312         unsigned long flags;
 313         int tail, status;
 314
 315         spin_lock_irqsave(&priv->tail_lock[ch], flags);
 316
 317         tail = priv->tail[ch];
 318         while (priv->fifo[ch][tail].desc) {
 319                 request = &priv->fifo[ch][tail];
 320
 321                 /* descriptors with their done bits set don't get the error */
 322                 rmb();
 323                 if ((request->desc->hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
 324                         status = 0;
 325                 else
 326                         if (!error)
 327                                 break;
 328                         else
 329                                 status = error;
 330
 331                 dma_unmap_single(dev, request->dma_desc,
 332                         sizeof(struct talitos_desc), DMA_BIDIRECTIONAL);
 333
 334                 /* copy entries so we can call callback outside lock */
 335                 saved_req.desc = request->desc;
 336                 saved_req.callback = request->callback;
 337                 saved_req.context = request->context;
 338
 339                 /* release request entry in fifo */
 340                 smp_wmb();
 341                 request->desc = NULL;
 342
 343                 /* increment fifo tail */
 344                 priv->tail[ch] = (tail + 1) & (priv->fifo_len - 1);
 345
 346                 spin_unlock_irqrestore(&priv->tail_lock[ch], flags);
 347
 348                 atomic_dec(&priv->submit_count[ch]);
 349
 350                 saved_req.callback(dev, saved_req.desc, saved_req.context,
 351                                    status);
 352                 /* channel may resume processing in single desc error case */
 353                 if (error && !reset_ch && status == error)
 354                         return;
 355                 spin_lock_irqsave(&priv->tail_lock[ch], flags);
 356                 tail = priv->tail[ch];
 357         }
 358
 359         spin_unlock_irqrestore(&priv->tail_lock[ch], flags);
 360 }
 361
 362 /*
 363  * process completed requests for channels that have done status
 364  */
 365 static void talitos_done(unsigned long data)
 366 {
 367         struct device *dev = (struct device *)data;
 368         struct talitos_private *priv = dev_get_drvdata(dev);
 369         int ch;
 370
 371         for (ch = 0; ch < priv->num_channels; ch++)
 372                 flush_channel(dev, ch, 0, 0);
 373 }
 374
 375 /*
 376  * locate current (offending) descriptor
 377  */
 378 static struct talitos_desc *current_desc(struct device *dev, int ch)
 379 {
 380         struct talitos_private *priv = dev_get_drvdata(dev);
 381         int tail = priv->tail[ch];
 382         dma_addr_t cur_desc;
 383
 384         cur_desc = in_be32(priv->reg + TALITOS_CDPR_LO(ch));
 385
 386         while (priv->fifo[ch][tail].dma_desc != cur_desc) {
 387                 tail = (tail + 1) & (priv->fifo_len - 1);
 388                 if (tail == priv->tail[ch]) {
 389                         dev_err(dev, "couldn't locate current descriptor\n");
 390                         return NULL;
 391                 }
 392         }
 393
 394         return priv->fifo[ch][tail].desc;
 395 }
 396
 397 /*
 398  * user diagnostics; report root cause of error based on execution unit status
 399  */
 400 static void report_eu_error(struct device *dev, int ch, struct talitos_desc *desc)
 401 {
 402         struct talitos_private *priv = dev_get_drvdata(dev);
 403         int i;
 404
 405         switch (desc->hdr & DESC_HDR_SEL0_MASK) {
 406         case DESC_HDR_SEL0_AFEU:
 407                 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
 408                         in_be32(priv->reg + TALITOS_AFEUISR),
 409                         in_be32(priv->reg + TALITOS_AFEUISR_LO));
 410                 break;
 411         case DESC_HDR_SEL0_DEU:
 412                 dev_err(dev, "DEUISR 0x%08x_%08x\n",
 413                         in_be32(priv->reg + TALITOS_DEUISR),
 414                         in_be32(priv->reg + TALITOS_DEUISR_LO));
 415                 break;
 416         case DESC_HDR_SEL0_MDEUA:
 417         case DESC_HDR_SEL0_MDEUB:
 418                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
 419                         in_be32(priv->reg + TALITOS_MDEUISR),
 420                         in_be32(priv->reg + TALITOS_MDEUISR_LO));
 421                 break;
 422         case DESC_HDR_SEL0_RNG:
 423                 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
 424                         in_be32(priv->reg + TALITOS_RNGUISR),
 425                         in_be32(priv->reg + TALITOS_RNGUISR_LO));
 426                 break;
 427         case DESC_HDR_SEL0_PKEU:
 428                 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
 429                         in_be32(priv->reg + TALITOS_PKEUISR),
 430                         in_be32(priv->reg + TALITOS_PKEUISR_LO));
 431                 break;
 432         case DESC_HDR_SEL0_AESU:
 433                 dev_err(dev, "AESUISR 0x%08x_%08x\n",
 434                         in_be32(priv->reg + TALITOS_AESUISR),
 435                         in_be32(priv->reg + TALITOS_AESUISR_LO));
 436                 break;
 437         case DESC_HDR_SEL0_CRCU:
 438                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
 439                         in_be32(priv->reg + TALITOS_CRCUISR),
 440                         in_be32(priv->reg + TALITOS_CRCUISR_LO));
 441                 break;
 442         case DESC_HDR_SEL0_KEU:
 443                 dev_err(dev, "KEUISR 0x%08x_%08x\n",
 444                         in_be32(priv->reg + TALITOS_KEUISR),
 445                         in_be32(priv->reg + TALITOS_KEUISR_LO));
 446                 break;
 447         }
 448
 449         switch (desc->hdr & DESC_HDR_SEL1_MASK) {
 450         case DESC_HDR_SEL1_MDEUA:
 451         case DESC_HDR_SEL1_MDEUB:
 452                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
 453                         in_be32(priv->reg + TALITOS_MDEUISR),
 454                         in_be32(priv->reg + TALITOS_MDEUISR_LO));
 455                 break;
 456         case DESC_HDR_SEL1_CRCU:
 457                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
 458                         in_be32(priv->reg + TALITOS_CRCUISR),
 459                         in_be32(priv->reg + TALITOS_CRCUISR_LO));
 460                 break;
 461         }
 462
 463         for (i = 0; i < 8; i++)
 464                 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
 465                         in_be32(priv->reg + TALITOS_DESCBUF(ch) + 8*i),
 466                         in_be32(priv->reg + TALITOS_DESCBUF_LO(ch) + 8*i));
 467 }
 468
 469 /*
 470  * recover from error interrupts
 471  */
 472 static void talitos_error(unsigned long data)
 473 {
 474         struct device *dev = (struct device *)data;
 475         struct talitos_private *priv = dev_get_drvdata(dev);
 476         unsigned int timeout = TALITOS_TIMEOUT;
 477         int ch, error, reset_dev = 0, reset_ch = 0;
 478         u32 isr, isr_lo, v, v_lo;
 479
 480         isr = in_be32(priv->reg + TALITOS_ISR);
 481         isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);
 482
 483         for (ch = 0; ch < priv->num_channels; ch++) {
 484                 /* skip channels without errors */
 485                 if (!(isr & (1 << (ch * 2 + 1))))
 486                         continue;
 487
 488                 error = -EINVAL;
 489
 490                 v = in_be32(priv->reg + TALITOS_CCPSR(ch));
 491                 v_lo = in_be32(priv->reg + TALITOS_CCPSR_LO(ch));
 492
 493                 if (v_lo & TALITOS_CCPSR_LO_DOF) {
 494                         dev_err(dev, "double fetch fifo overflow error\n");
 495                         error = -EAGAIN;
 496                         reset_ch = 1;
 497                 }
 498                 if (v_lo & TALITOS_CCPSR_LO_SOF) {
 499                         /* h/w dropped descriptor */
 500                         dev_err(dev, "single fetch fifo overflow error\n");
 501                         error = -EAGAIN;
 502                 }
 503                 if (v_lo & TALITOS_CCPSR_LO_MDTE)
 504                         dev_err(dev, "master data transfer error\n");
 505                 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
 506                         dev_err(dev, "s/g data length zero error\n");
 507                 if (v_lo & TALITOS_CCPSR_LO_FPZ)
 508                         dev_err(dev, "fetch pointer zero error\n");
 509                 if (v_lo & TALITOS_CCPSR_LO_IDH)
 510                         dev_err(dev, "illegal descriptor header error\n");
 511                 if (v_lo & TALITOS_CCPSR_LO_IEU)
 512                         dev_err(dev, "invalid execution unit error\n");
 513                 if (v_lo & TALITOS_CCPSR_LO_EU)
 514                         report_eu_error(dev, ch, current_desc(dev, ch));
 515                 if (v_lo & TALITOS_CCPSR_LO_GB)
 516                         dev_err(dev, "gather boundary error\n");
 517                 if (v_lo & TALITOS_CCPSR_LO_GRL)
 518                         dev_err(dev, "gather return/length error\n");
 519                 if (v_lo & TALITOS_CCPSR_LO_SB)
 520                         dev_err(dev, "scatter boundary error\n");
 521                 if (v_lo & TALITOS_CCPSR_LO_SRL)
 522                         dev_err(dev, "scatter return/length error\n");
 523
 524                 flush_channel(dev, ch, error, reset_ch);
 525
 526                 if (reset_ch) {
 527                         reset_channel(dev, ch);
 528                 } else {
 529                         setbits32(priv->reg + TALITOS_CCCR(ch),
 530                                   TALITOS_CCCR_CONT);
 531                         setbits32(priv->reg + TALITOS_CCCR_LO(ch), 0);
 532                         while ((in_be32(priv->reg + TALITOS_CCCR(ch)) &
 533                                TALITOS_CCCR_CONT) && --timeout)
 534                                 cpu_relax();
 535                         if (timeout == 0) {
 536                                 dev_err(dev, "failed to restart channel %d\n",
 537                                         ch);
 538                                 reset_dev = 1;
 539                         }
 540                 }
 541         }
 542         if (reset_dev || isr & ~TALITOS_ISR_CHERR || isr_lo) {
 543                 dev_err(dev, "done overflow, internal time out, or rngu error: "
 544                         "ISR 0x%08x_%08x\n", isr, isr_lo);
 545
 546                 /* purge request queues */
 547                 for (ch = 0; ch < priv->num_channels; ch++)
 548                         flush_channel(dev, ch, -EIO, 1);
 549
 550                 /* reset and reinitialize the device */
 551                 init_device(dev);
 552         }
 553 }
 554
 555 static irqreturn_t talitos_interrupt(int irq, void *data)
 556 {
 557         struct device *dev = data;
 558         struct talitos_private *priv = dev_get_drvdata(dev);
 559         u32 isr, isr_lo;
 560
 561         isr = in_be32(priv->reg + TALITOS_ISR);
 562         isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);
 563
 564         /* ack */
 565         out_be32(priv->reg + TALITOS_ICR, isr);
 566         out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);
 567
 568         if (unlikely((isr & ~TALITOS_ISR_CHDONE) || isr_lo))
 569                 talitos_error((unsigned long)data);
 570         else
 571                 if (likely(isr & TALITOS_ISR_CHDONE))
 572                         tasklet_schedule(&priv->done_task);
 573
 574         return (isr || isr_lo) ? IRQ_HANDLED : IRQ_NONE;
 575 }
 576
 577 /*
 578  * hwrng
 579  */
 580 static int talitos_rng_data_present(struct hwrng *rng, int wait)
 581 {
 582         struct device *dev = (struct device *)rng->priv;
 583         struct talitos_private *priv = dev_get_drvdata(dev);
 584         u32 ofl;
 585         int i;
 586
 587         for (i = 0; i < 20; i++) {
 588                 ofl = in_be32(priv->reg + TALITOS_RNGUSR_LO) &
 589                       TALITOS_RNGUSR_LO_OFL;
 590                 if (ofl || !wait)
 591                         break;
 592                 udelay(10);
 593         }
 594
 595         return !!ofl;
 596 }
 597
 598 static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
 599 {
 600         struct device *dev = (struct device *)rng->priv;
 601         struct talitos_private *priv = dev_get_drvdata(dev);
 602
 603         /* rng fifo requires 64-bit accesses */
 604         *data = in_be32(priv->reg + TALITOS_RNGU_FIFO);
 605         *data = in_be32(priv->reg + TALITOS_RNGU_FIFO_LO);
 606
 607         return sizeof(u32);
 608 }
 609
 610 static int talitos_rng_init(struct hwrng *rng)
 611 {
 612         struct device *dev = (struct device *)rng->priv;
 613         struct talitos_private *priv = dev_get_drvdata(dev);
 614         unsigned int timeout = TALITOS_TIMEOUT;
 615
 616         setbits32(priv->reg + TALITOS_RNGURCR_LO, TALITOS_RNGURCR_LO_SR);
 617         while (!(in_be32(priv->reg + TALITOS_RNGUSR_LO) & TALITOS_RNGUSR_LO_RD)
 618                && --timeout)
 619                 cpu_relax();
 620         if (timeout == 0) {
 621                 dev_err(dev, "failed to reset rng hw\n");
 622                 return -ENODEV;
 623         }
 624
 625         /* start generating */
 626         setbits32(priv->reg + TALITOS_RNGUDSR_LO, 0);
 627
 628         return 0;
 629 }
 630
 631 static int talitos_register_rng(struct device *dev)
 632 {
 633         struct talitos_private *priv = dev_get_drvdata(dev);
 634
 635         priv->rng.name          = dev_driver_string(dev),
 636         priv->rng.init          = talitos_rng_init,
 637         priv->rng.data_present  = talitos_rng_data_present,
 638         priv->rng.data_read     = talitos_rng_data_read,
 639         priv->rng.priv          = (unsigned long)dev;
 640
 641         return hwrng_register(&priv->rng);
 642 }
 643
 644 static void talitos_unregister_rng(struct device *dev)
 645 {
 646         struct talitos_private *priv = dev_get_drvdata(dev);
 647
 648         hwrng_unregister(&priv->rng);
 649 }
 650
 651 /*
 652  * crypto alg
 653  */
 654 #define TALITOS_CRA_PRIORITY            3000
 655 #define TALITOS_MAX_KEY_SIZE            64
 656 #define TALITOS_MAX_IV_LENGTH           16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
 657
 658 #define MD5_DIGEST_SIZE   16
 659
 660 struct talitos_ctx {
 661         struct device *dev;
 662         __be32 desc_hdr_template;
 663         u8 key[TALITOS_MAX_KEY_SIZE];
 664         u8 iv[TALITOS_MAX_IV_LENGTH];
 665         unsigned int keylen;
 666         unsigned int enckeylen;
 667         unsigned int authkeylen;
 668         unsigned int authsize;
 669 };
 670
 671 static int aead_authenc_setauthsize(struct crypto_aead *authenc,
 672                                                  unsigned int authsize)
 673 {
 674         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 675
 676         ctx->authsize = authsize;
 677
 678         return 0;
 679 }
 680
 681 static int aead_authenc_setkey(struct crypto_aead *authenc,
 682                                             const u8 *key, unsigned int keylen)
 683 {
 684         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 685         struct rtattr *rta = (void *)key;
 686         struct crypto_authenc_key_param *param;
 687         unsigned int authkeylen;
 688         unsigned int enckeylen;
 689
 690         if (!RTA_OK(rta, keylen))
 691                 goto badkey;
 692
 693         if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
 694                 goto badkey;
 695
 696         if (RTA_PAYLOAD(rta) < sizeof(*param))
 697                 goto badkey;
 698
 699         param = RTA_DATA(rta);
 700         enckeylen = be32_to_cpu(param->enckeylen);
 701
 702         key += RTA_ALIGN(rta->rta_len);
 703         keylen -= RTA_ALIGN(rta->rta_len);
 704
 705         if (keylen < enckeylen)
 706                 goto badkey;
 707
 708         authkeylen = keylen - enckeylen;
 709
 710         if (keylen > TALITOS_MAX_KEY_SIZE)
 711                 goto badkey;
 712
 713         memcpy(&ctx->key, key, keylen);
 714
 715         ctx->keylen = keylen;
 716         ctx->enckeylen = enckeylen;
 717         ctx->authkeylen = authkeylen;
 718
 719         return 0;
 720
 721 badkey:
 722         crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
 723         return -EINVAL;
 724 }
 725
 726 /*
 727  * ipsec_esp_edesc - s/w-extended ipsec_esp descriptor
 728  * @src_nents: number of segments in input scatterlist
 729  * @dst_nents: number of segments in output scatterlist
 730  * @dma_len: length of dma mapped link_tbl space
 731  * @dma_link_tbl: bus physical address of link_tbl
 732  * @desc: h/w descriptor
 733  * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1)
 734  *
 735  * if decrypting (with authcheck), or either one of src_nents or dst_nents
 736  * is greater than 1, an integrity check value is concatenated to the end
 737  * of link_tbl data
 738  */
 739 struct ipsec_esp_edesc {
 740         int src_nents;
 741         int dst_nents;
 742         int dma_len;
 743         dma_addr_t dma_link_tbl;
 744         struct talitos_desc desc;
 745         struct talitos_ptr link_tbl[0];
 746 };
 747
 748 static void ipsec_esp_unmap(struct device *dev,
 749                             struct ipsec_esp_edesc *edesc,
 750                             struct aead_request *areq)
 751 {
 752         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE);
 753         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);
 754         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
 755         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
 756
 757         dma_unmap_sg(dev, areq->assoc, 1, DMA_TO_DEVICE);
 758
 759         if (areq->src != areq->dst) {
 760                 dma_unmap_sg(dev, areq->src, edesc->src_nents ? : 1,
 761                              DMA_TO_DEVICE);
 762                 dma_unmap_sg(dev, areq->dst, edesc->dst_nents ? : 1,
 763                              DMA_FROM_DEVICE);
 764         } else {
 765                 dma_unmap_sg(dev, areq->src, edesc->src_nents ? : 1,
 766                              DMA_BIDIRECTIONAL);
 767         }
 768
 769         if (edesc->dma_len)
 770                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
 771                                  DMA_BIDIRECTIONAL);
 772 }
 773
 774 /*
 775  * ipsec_esp descriptor callbacks
 776  */
 777 static void ipsec_esp_encrypt_done(struct device *dev,
 778                                    struct talitos_desc *desc, void *context,
 779                                    int err)
 780 {
 781         struct aead_request *areq = context;
 782         struct ipsec_esp_edesc *edesc =
 783                  container_of(desc, struct ipsec_esp_edesc, desc);
 784         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
 785         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 786         struct scatterlist *sg;
 787         void *icvdata;
 788
 789         ipsec_esp_unmap(dev, edesc, areq);
 790
 791         /* copy the generated ICV to dst */
 792         if (edesc->dma_len) {
 793                 icvdata = &edesc->link_tbl[edesc->src_nents +
 794                                            edesc->dst_nents + 2];
 795                 sg = sg_last(areq->dst, edesc->dst_nents);
 796                 memcpy((char *)sg_virt(sg) + sg->length - ctx->authsize,
 797                        icvdata, ctx->authsize);
 798         }
 799
 800         kfree(edesc);
 801
 802         aead_request_complete(areq, err);
 803 }
 804
 805 static void ipsec_esp_decrypt_done(struct device *dev,
 806                                    struct talitos_desc *desc, void *context,
 807                                    int err)
 808 {
 809         struct aead_request *req = context;
 810         struct ipsec_esp_edesc *edesc =
 811                  container_of(desc, struct ipsec_esp_edesc, desc);
 812         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
 813         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 814         struct scatterlist *sg;
 815         void *icvdata;
 816
 817         ipsec_esp_unmap(dev, edesc, req);
 818
 819         if (!err) {
 820                 /* auth check */
 821                 if (edesc->dma_len)
 822                         icvdata = &edesc->link_tbl[edesc->src_nents +
 823                                                    edesc->dst_nents + 2];
 824                 else
 825                         icvdata = &edesc->link_tbl[0];
 826
 827                 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
 828                 err = memcmp(icvdata, (char *)sg_virt(sg) + sg->length -
 829                              ctx->authsize, ctx->authsize) ? -EBADMSG : 0;
 830         }
 831
 832         kfree(edesc);
 833
 834         aead_request_complete(req, err);
 835 }
 836
 837 /*
 838  * convert scatterlist to SEC h/w link table format
 839  * stop at cryptlen bytes
 840  */
 841 static int sg_to_link_tbl(struct scatterlist *sg, int sg_count,
 842                            int cryptlen, struct talitos_ptr *link_tbl_ptr)
 843 {
 844         int n_sg = sg_count;
 845
 846         while (n_sg--) {
 847                 link_tbl_ptr->ptr = cpu_to_be32(sg_dma_address(sg));
 848                 link_tbl_ptr->len = cpu_to_be16(sg_dma_len(sg));
 849                 link_tbl_ptr->j_extent = 0;
 850                 link_tbl_ptr++;
 851                 cryptlen -= sg_dma_len(sg);
 852                 sg = sg_next(sg);
 853         }
 854
 855         /* adjust (decrease) last one (or two) entry's len to cryptlen */
 856         link_tbl_ptr--;
 857         while (be16_to_cpu(link_tbl_ptr->len) <= (-cryptlen)) {
 858                 /* Empty this entry, and move to previous one */
 859                 cryptlen += be16_to_cpu(link_tbl_ptr->len);
 860                 link_tbl_ptr->len = 0;
 861                 sg_count--;
 862                 link_tbl_ptr--;
 863         }
 864         link_tbl_ptr->len = cpu_to_be16(be16_to_cpu(link_tbl_ptr->len)
 865                                         + cryptlen);
 866
 867         /* tag end of link table */
 868         link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
 869
 870         return sg_count;
 871 }
 872
 873 /*
 874  * fill in and submit ipsec_esp descriptor
 875  */
 876 static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq,
 877                      u8 *giv, u64 seq,
 878                      void (*callback) (struct device *dev,
 879                                        struct talitos_desc *desc,
 880                                        void *context, int error))
 881 {
 882         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
 883         struct talitos_ctx *ctx = crypto_aead_ctx(aead);
 884         struct device *dev = ctx->dev;
 885         struct talitos_desc *desc = &edesc->desc;
 886         unsigned int cryptlen = areq->cryptlen;
 887         unsigned int authsize = ctx->authsize;
 888         unsigned int ivsize;
 889         int sg_count, ret;
 890
 891         /* hmac key */
 892         map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
 893                                0, DMA_TO_DEVICE);
 894         /* hmac data */
 895         map_single_talitos_ptr(dev, &desc->ptr[1], sg_virt(areq->src) -
 896                                sg_virt(areq->assoc), sg_virt(areq->assoc), 0,
 897                                DMA_TO_DEVICE);
 898         /* cipher iv */
 899         ivsize = crypto_aead_ivsize(aead);
 900         map_single_talitos_ptr(dev, &desc->ptr[2], ivsize, giv ?: areq->iv, 0,
 901                                DMA_TO_DEVICE);
 902
 903         /* cipher key */
 904         map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
 905                                (char *)&ctx->key + ctx->authkeylen, 0,
 906                                DMA_TO_DEVICE);
 907
 908         /*
 909          * cipher in
 910          * map and adjust cipher len to aead request cryptlen.
 911          * extent is bytes of HMAC postpended to ciphertext,
 912          * typically 12 for ipsec
 913          */
 914         desc->ptr[4].len = cpu_to_be16(cryptlen);
 915         desc->ptr[4].j_extent = authsize;
 916
 917         if (areq->src == areq->dst)
 918                 sg_count = dma_map_sg(dev, areq->src, edesc->src_nents ? : 1,
 919                                       DMA_BIDIRECTIONAL);
 920         else
 921                 sg_count = dma_map_sg(dev, areq->src, edesc->src_nents ? : 1,
 922                                       DMA_TO_DEVICE);
 923
 924         if (sg_count == 1) {
 925                 desc->ptr[4].ptr = cpu_to_be32(sg_dma_address(areq->src));
 926         } else {
 927                 sg_count = sg_to_link_tbl(areq->src, sg_count, cryptlen,
 928                                           &edesc->link_tbl[0]);
 929                 if (sg_count > 1) {
 930                         struct talitos_ptr *link_tbl_ptr =
 931                                 &edesc->link_tbl[sg_count-1];
 932                         struct scatterlist *sg;
 933                         struct talitos_private *priv = dev_get_drvdata(dev);
 934
 935                         desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
 936                         desc->ptr[4].ptr = cpu_to_be32(edesc->dma_link_tbl);
 937                         dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
 938                                                    edesc->dma_len, DMA_BIDIRECTIONAL);
 939                         /* If necessary for this SEC revision,
 940                          * add a link table entry for ICV.
 941                          */
 942                         if ((priv->features &
 943                              TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT) &&
 944                             (edesc->desc.hdr & DESC_HDR_MODE0_ENCRYPT) == 0) {
 945                                 link_tbl_ptr->j_extent = 0;
 946                                 link_tbl_ptr++;
 947                                 link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
 948                                 link_tbl_ptr->len = cpu_to_be16(authsize);
 949                                 sg = sg_last(areq->src, edesc->src_nents ? : 1);
 950                                 link_tbl_ptr->ptr = cpu_to_be32(
 951                                                 (char *)sg_dma_address(sg)
 952                                                 + sg->length - authsize);
 953                         }
 954                 } else {
 955                         /* Only one segment now, so no link tbl needed */
 956                         desc->ptr[4].ptr = cpu_to_be32(sg_dma_address(areq->src));
 957                 }
 958         }
 959
 960         /* cipher out */
 961         desc->ptr[5].len = cpu_to_be16(cryptlen);
 962         desc->ptr[5].j_extent = authsize;
 963
 964         if (areq->src != areq->dst) {
 965                 sg_count = dma_map_sg(dev, areq->dst, edesc->dst_nents ? : 1,
 966                                       DMA_FROM_DEVICE);
 967         }
 968
 969         if (sg_count == 1) {
 970                 desc->ptr[5].ptr = cpu_to_be32(sg_dma_address(areq->dst));
 971         } else {
 972                 struct talitos_ptr *link_tbl_ptr =
 973                         &edesc->link_tbl[edesc->src_nents + 1];
 974
 975                 desc->ptr[5].ptr = cpu_to_be32((struct talitos_ptr *)
 976                                                edesc->dma_link_tbl +
 977                                                edesc->src_nents + 1);
 978                 if (areq->src == areq->dst) {
 979                         memcpy(link_tbl_ptr, &edesc->link_tbl[0],
 980                                edesc->src_nents * sizeof(struct talitos_ptr));
 981                 } else {
 982                         sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen,
 983                                                   link_tbl_ptr);
 984                 }
 985                 /* Add an entry to the link table for ICV data */
 986                 link_tbl_ptr += sg_count - 1;
 987                 link_tbl_ptr->j_extent = 0;
 988                 sg_count++;
 989                 link_tbl_ptr++;
 990                 link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
 991                 link_tbl_ptr->len = cpu_to_be16(authsize);
 992
 993                 /* icv data follows link tables */
 994                 link_tbl_ptr->ptr = cpu_to_be32((struct talitos_ptr *)
 995                                                 edesc->dma_link_tbl +
 996                                                 edesc->src_nents +
 997                                                 edesc->dst_nents + 2);
 998
 999                 desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP;
1000                 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
1001                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1002         }
1003
1004         /* iv out */
1005         map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, 0,
1006                                DMA_FROM_DEVICE);
1007
1008         ret = talitos_submit(dev, desc, callback, areq);
1009         if (ret != -EINPROGRESS) {
1010                 ipsec_esp_unmap(dev, edesc, areq);
1011                 kfree(edesc);
1012         }
1013         return ret;
1014 }
1015
1016
1017 /*
1018  * derive number of elements in scatterlist
1019  */
1020 static int sg_count(struct scatterlist *sg_list, int nbytes)
1021 {
1022         struct scatterlist *sg = sg_list;
1023         int sg_nents = 0;
1024
1025         while (nbytes) {
1026                 sg_nents++;
1027                 nbytes -= sg->length;
1028                 sg = sg_next(sg);
1029         }
1030
1031         return sg_nents;
1032 }
1033
1034 /*
1035  * allocate and map the ipsec_esp extended descriptor
1036  */
1037 static struct ipsec_esp_edesc *ipsec_esp_edesc_alloc(struct aead_request *areq,
1038                                                      int icv_stashing)
1039 {
1040         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1041         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1042         struct ipsec_esp_edesc *edesc;
1043         int src_nents, dst_nents, alloc_len, dma_len;
1044         gfp_t flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1045                       GFP_ATOMIC;
1046
1047         if (areq->cryptlen + ctx->authsize > TALITOS_MAX_DATA_LEN) {
1048                 dev_err(ctx->dev, "cryptlen exceeds h/w max limit\n");
1049                 return ERR_PTR(-EINVAL);
1050         }
1051
1052         src_nents = sg_count(areq->src, areq->cryptlen + ctx->authsize);
1053         src_nents = (src_nents == 1) ? 0 : src_nents;
1054
1055         if (areq->dst == areq->src) {
1056                 dst_nents = src_nents;
1057         } else {
1058                 dst_nents = sg_count(areq->dst, areq->cryptlen + ctx->authsize);
1059                 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1060         }
1061
1062         /*
1063          * allocate space for base edesc plus the link tables,
1064          * allowing for two separate entries for ICV and generated ICV (+ 2),
1065          * and the ICV data itself
1066          */
1067         alloc_len = sizeof(struct ipsec_esp_edesc);
1068         if (src_nents || dst_nents) {
1069                 dma_len = (src_nents + dst_nents + 2) *
1070                                  sizeof(struct talitos_ptr) + ctx->authsize;
1071                 alloc_len += dma_len;
1072         } else {
1073                 dma_len = 0;
1074                 alloc_len += icv_stashing ? ctx->authsize : 0;
1075         }
1076
1077         edesc = kmalloc(alloc_len, GFP_DMA | flags);
1078         if (!edesc) {
1079                 dev_err(ctx->dev, "could not allocate edescriptor\n");
1080                 return ERR_PTR(-ENOMEM);
1081         }
1082
1083         edesc->src_nents = src_nents;
1084         edesc->dst_nents = dst_nents;
1085         edesc->dma_len = dma_len;
1086         edesc->dma_link_tbl = dma_map_single(ctx->dev, &edesc->link_tbl[0],
1087                                              edesc->dma_len, DMA_BIDIRECTIONAL);
1088
1089         return edesc;
1090 }
1091
1092 static int aead_authenc_encrypt(struct aead_request *req)
1093 {
1094         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1095         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1096         struct ipsec_esp_edesc *edesc;
1097
1098         /* allocate extended descriptor */
1099         edesc = ipsec_esp_edesc_alloc(req, 0);
1100         if (IS_ERR(edesc))
1101                 return PTR_ERR(edesc);
1102
1103         /* set encrypt */
1104         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1105
1106         return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_encrypt_done);
1107 }
1108
1109 static int aead_authenc_decrypt(struct aead_request *req)
1110 {
1111         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1112         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1113         unsigned int authsize = ctx->authsize;
1114         struct ipsec_esp_edesc *edesc;
1115         struct scatterlist *sg;
1116         void *icvdata;
1117
1118         req->cryptlen -= authsize;
1119
1120         /* allocate extended descriptor */
1121         edesc = ipsec_esp_edesc_alloc(req, 1);
1122         if (IS_ERR(edesc))
1123                 return PTR_ERR(edesc);
1124
1125         /* stash incoming ICV for later cmp with ICV generated by the h/w */
1126         if (edesc->dma_len)
1127                 icvdata = &edesc->link_tbl[edesc->src_nents +
1128                                            edesc->dst_nents + 2];
1129         else
1130                 icvdata = &edesc->link_tbl[0];
1131
1132         sg = sg_last(req->src, edesc->src_nents ? : 1);
1133
1134         memcpy(icvdata, (char *)sg_virt(sg) + sg->length - ctx->authsize,
1135                ctx->authsize);
1136
1137         /* decrypt */
1138         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1139
1140         return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_decrypt_done);
1141 }
1142
1143 static int aead_authenc_givencrypt(
1144         struct aead_givcrypt_request *req)
1145 {
1146         struct aead_request *areq = &req->areq;
1147         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1148         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1149         struct ipsec_esp_edesc *edesc;
1150
1151         /* allocate extended descriptor */
1152         edesc = ipsec_esp_edesc_alloc(areq, 0);
1153         if (IS_ERR(edesc))
1154                 return PTR_ERR(edesc);
1155
1156         /* set encrypt */
1157         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1158
1159         memcpy(req->giv, ctx->iv, crypto_aead_ivsize(authenc));
1160         /* avoid consecutive packets going out with same IV */
1161         *(__be64 *)req->giv ^= cpu_to_be64(req->seq);
1162
1163         return ipsec_esp(edesc, areq, req->giv, req->seq,
1164                          ipsec_esp_encrypt_done);
1165 }
1166
1167 struct talitos_alg_template {
1168         char name[CRYPTO_MAX_ALG_NAME];
1169         char driver_name[CRYPTO_MAX_ALG_NAME];
1170         unsigned int blocksize;
1171         struct aead_alg aead;
1172         struct device *dev;
1173         __be32 desc_hdr_template;
1174 };
1175
1176 static struct talitos_alg_template driver_algs[] = {
1177         /* single-pass ipsec_esp descriptor */
1178         {
1179                 .name = "authenc(hmac(sha1),cbc(aes))",
1180                 .driver_name = "authenc-hmac-sha1-cbc-aes-talitos",
1181                 .blocksize = AES_BLOCK_SIZE,
1182                 .aead = {
1183                         .setkey = aead_authenc_setkey,
1184                         .setauthsize = aead_authenc_setauthsize,
1185                         .encrypt = aead_authenc_encrypt,
1186                         .decrypt = aead_authenc_decrypt,
1187                         .givencrypt = aead_authenc_givencrypt,
1188                         .geniv = "<built-in>",
1189                         .ivsize = AES_BLOCK_SIZE,
1190                         .maxauthsize = SHA1_DIGEST_SIZE,
1191                         },
1192                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1193                                      DESC_HDR_SEL0_AESU |
1194                                      DESC_HDR_MODE0_AESU_CBC |
1195                                      DESC_HDR_SEL1_MDEUA |
1196                                      DESC_HDR_MODE1_MDEU_INIT |
1197                                      DESC_HDR_MODE1_MDEU_PAD |
1198                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
1199         },
1200         {
1201                 .name = "authenc(hmac(sha1),cbc(des3_ede))",
1202                 .driver_name = "authenc-hmac-sha1-cbc-3des-talitos",
1203                 .blocksize = DES3_EDE_BLOCK_SIZE,
1204                 .aead = {
1205                         .setkey = aead_authenc_setkey,
1206                         .setauthsize = aead_authenc_setauthsize,
1207                         .encrypt = aead_authenc_encrypt,
1208                         .decrypt = aead_authenc_decrypt,
1209                         .givencrypt = aead_authenc_givencrypt,
1210                         .geniv = "<built-in>",
1211                         .ivsize = DES3_EDE_BLOCK_SIZE,
1212                         .maxauthsize = SHA1_DIGEST_SIZE,
1213                         },
1214                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1215                                      DESC_HDR_SEL0_DEU |
1216                                      DESC_HDR_MODE0_DEU_CBC |
1217                                      DESC_HDR_MODE0_DEU_3DES |
1218                                      DESC_HDR_SEL1_MDEUA |
1219                                      DESC_HDR_MODE1_MDEU_INIT |
1220                                      DESC_HDR_MODE1_MDEU_PAD |
1221                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
1222         },
1223         {
1224                 .name = "authenc(hmac(sha256),cbc(aes))",
1225                 .driver_name = "authenc-hmac-sha256-cbc-aes-talitos",
1226                 .blocksize = AES_BLOCK_SIZE,
1227                 .aead = {
1228                         .setkey = aead_authenc_setkey,
1229                         .setauthsize = aead_authenc_setauthsize,
1230                         .encrypt = aead_authenc_encrypt,
1231                         .decrypt = aead_authenc_decrypt,
1232                         .givencrypt = aead_authenc_givencrypt,
1233                         .geniv = "<built-in>",
1234                         .ivsize = AES_BLOCK_SIZE,
1235                         .maxauthsize = SHA256_DIGEST_SIZE,
1236                         },
1237                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1238                                      DESC_HDR_SEL0_AESU |
1239                                      DESC_HDR_MODE0_AESU_CBC |
1240                                      DESC_HDR_SEL1_MDEUA |
1241                                      DESC_HDR_MODE1_MDEU_INIT |
1242                                      DESC_HDR_MODE1_MDEU_PAD |
1243                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
1244         },
1245         {
1246                 .name = "authenc(hmac(sha256),cbc(des3_ede))",
1247                 .driver_name = "authenc-hmac-sha256-cbc-3des-talitos",
1248                 .blocksize = DES3_EDE_BLOCK_SIZE,
1249                 .aead = {
1250                         .setkey = aead_authenc_setkey,
1251                         .setauthsize = aead_authenc_setauthsize,
1252                         .encrypt = aead_authenc_encrypt,
1253                         .decrypt = aead_authenc_decrypt,
1254                         .givencrypt = aead_authenc_givencrypt,
1255                         .geniv = "<built-in>",
1256                         .ivsize = DES3_EDE_BLOCK_SIZE,
1257                         .maxauthsize = SHA256_DIGEST_SIZE,
1258                         },
1259                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1260                                      DESC_HDR_SEL0_DEU |
1261                                      DESC_HDR_MODE0_DEU_CBC |
1262                                      DESC_HDR_MODE0_DEU_3DES |
1263                                      DESC_HDR_SEL1_MDEUA |
1264                                      DESC_HDR_MODE1_MDEU_INIT |
1265                                      DESC_HDR_MODE1_MDEU_PAD |
1266                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
1267         },
1268         {
1269                 .name = "authenc(hmac(md5),cbc(aes))",
1270                 .driver_name = "authenc-hmac-md5-cbc-aes-talitos",
1271                 .blocksize = AES_BLOCK_SIZE,
1272                 .aead = {
1273                         .setkey = aead_authenc_setkey,
1274                         .setauthsize = aead_authenc_setauthsize,
1275                         .encrypt = aead_authenc_encrypt,
1276                         .decrypt = aead_authenc_decrypt,
1277                         .givencrypt = aead_authenc_givencrypt,
1278                         .geniv = "<built-in>",
1279                         .ivsize = AES_BLOCK_SIZE,
1280                         .maxauthsize = MD5_DIGEST_SIZE,
1281                         },
1282                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1283                                      DESC_HDR_SEL0_AESU |
1284                                      DESC_HDR_MODE0_AESU_CBC |
1285                                      DESC_HDR_SEL1_MDEUA |
1286                                      DESC_HDR_MODE1_MDEU_INIT |
1287                                      DESC_HDR_MODE1_MDEU_PAD |
1288                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
1289         },
1290         {
1291                 .name = "authenc(hmac(md5),cbc(des3_ede))",
1292                 .driver_name = "authenc-hmac-md5-cbc-3des-talitos",
1293                 .blocksize = DES3_EDE_BLOCK_SIZE,
1294                 .aead = {
1295                         .setkey = aead_authenc_setkey,
1296                         .setauthsize = aead_authenc_setauthsize,
1297                         .encrypt = aead_authenc_encrypt,
1298                         .decrypt = aead_authenc_decrypt,
1299                         .givencrypt = aead_authenc_givencrypt,
1300                         .geniv = "<built-in>",
1301                         .ivsize = DES3_EDE_BLOCK_SIZE,
1302                         .maxauthsize = MD5_DIGEST_SIZE,
1303                         },
1304                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1305                                      DESC_HDR_SEL0_DEU |
1306                                      DESC_HDR_MODE0_DEU_CBC |
1307                                      DESC_HDR_MODE0_DEU_3DES |
1308                                      DESC_HDR_SEL1_MDEUA |
1309                                      DESC_HDR_MODE1_MDEU_INIT |
1310                                      DESC_HDR_MODE1_MDEU_PAD |
1311                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
1312         }
1313 };
1314
1315 struct talitos_crypto_alg {
1316         struct list_head entry;
1317         struct device *dev;
1318         __be32 desc_hdr_template;
1319         struct crypto_alg crypto_alg;
1320 };
1321
1322 static int talitos_cra_init(struct crypto_tfm *tfm)
1323 {
1324         struct crypto_alg *alg = tfm->__crt_alg;
1325         struct talitos_crypto_alg *talitos_alg =
1326                  container_of(alg, struct talitos_crypto_alg, crypto_alg);
1327         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
1328
1329         /* update context with ptr to dev */
1330         ctx->dev = talitos_alg->dev;
1331         /* copy descriptor header template value */
1332         ctx->desc_hdr_template = talitos_alg->desc_hdr_template;
1333
1334         /* random first IV */
1335         get_random_bytes(ctx->iv, TALITOS_MAX_IV_LENGTH);
1336
1337         return 0;
1338 }
1339
1340 /*
1341  * given the alg's descriptor header template, determine whether descriptor
1342  * type and primary/secondary execution units required match the hw
1343  * capabilities description provided in the device tree node.
1344  */
1345 static int hw_supports(struct device *dev, __be32 desc_hdr_template)
1346 {
1347         struct talitos_private *priv = dev_get_drvdata(dev);
1348         int ret;
1349
1350         ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
1351               (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
1352
1353         if (SECONDARY_EU(desc_hdr_template))
1354                 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
1355                               & priv->exec_units);
1356
1357         return ret;
1358 }
1359
1360 static int __devexit talitos_remove(struct of_device *ofdev)
1361 {
1362         struct device *dev = &ofdev->dev;
1363         struct talitos_private *priv = dev_get_drvdata(dev);
1364         struct talitos_crypto_alg *t_alg, *n;
1365         int i;
1366
1367         list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
1368                 crypto_unregister_alg(&t_alg->crypto_alg);
1369                 list_del(&t_alg->entry);
1370                 kfree(t_alg);
1371         }
1372
1373         if (hw_supports(dev, DESC_HDR_SEL0_RNG))
1374                 talitos_unregister_rng(dev);
1375
1376         kfree(priv->submit_count);
1377         kfree(priv->tail);
1378         kfree(priv->head);
1379
1380         if (priv->fifo)
1381                 for (i = 0; i < priv->num_channels; i++)
1382                         kfree(priv->fifo[i]);
1383
1384         kfree(priv->fifo);
1385         kfree(priv->head_lock);
1386         kfree(priv->tail_lock);
1387
1388         if (priv->irq != NO_IRQ) {
1389                 free_irq(priv->irq, dev);
1390                 irq_dispose_mapping(priv->irq);
1391         }
1392
1393         tasklet_kill(&priv->done_task);
1394         tasklet_kill(&priv->error_task);
1395
1396         iounmap(priv->reg);
1397
1398         dev_set_drvdata(dev, NULL);
1399
1400         kfree(priv);
1401
1402         return 0;
1403 }
1404
1405 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
1406                                                     struct talitos_alg_template
1407                                                            *template)
1408 {
1409         struct talitos_crypto_alg *t_alg;
1410         struct crypto_alg *alg;
1411
1412         t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL);
1413         if (!t_alg)
1414                 return ERR_PTR(-ENOMEM);
1415
1416         alg = &t_alg->crypto_alg;
1417
1418         snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name);
1419         snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1420                  template->driver_name);
1421         alg->cra_module = THIS_MODULE;
1422         alg->cra_init = talitos_cra_init;
1423         alg->cra_priority = TALITOS_CRA_PRIORITY;
1424         alg->cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC;
1425         alg->cra_blocksize = template->blocksize;
1426         alg->cra_alignmask = 0;
1427         alg->cra_type = &crypto_aead_type;
1428         alg->cra_ctxsize = sizeof(struct talitos_ctx);
1429         alg->cra_u.aead = template->aead;
1430
1431         t_alg->desc_hdr_template = template->desc_hdr_template;
1432         t_alg->dev = dev;
1433
1434         return t_alg;
1435 }
1436
1437 static int talitos_probe(struct of_device *ofdev,
1438                          const struct of_device_id *match)
1439 {
1440         struct device *dev = &ofdev->dev;
1441         struct device_node *np = ofdev->node;
1442         struct talitos_private *priv;
1443         const unsigned int *prop;
1444         int i, err;
1445
1446         priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL);
1447         if (!priv)
1448                 return -ENOMEM;
1449
1450         dev_set_drvdata(dev, priv);
1451
1452         priv->ofdev = ofdev;
1453
1454         INIT_LIST_HEAD(&priv->alg_list);
1455
1456         tasklet_init(&priv->done_task, talitos_done, (unsigned long)dev);
1457         tasklet_init(&priv->error_task, talitos_error, (unsigned long)dev);
1458
1459         priv->irq = irq_of_parse_and_map(np, 0);
1460
1461         if (priv->irq == NO_IRQ) {
1462                 dev_err(dev, "failed to map irq\n");
1463                 err = -EINVAL;
1464                 goto err_out;
1465         }
1466
1467         /* get the irq line */
1468         err = request_irq(priv->irq, talitos_interrupt, 0,
1469                           dev_driver_string(dev), dev);
1470         if (err) {
1471                 dev_err(dev, "failed to request irq %d\n", priv->irq);
1472                 irq_dispose_mapping(priv->irq);
1473                 priv->irq = NO_IRQ;
1474                 goto err_out;
1475         }
1476
1477         priv->reg = of_iomap(np, 0);
1478         if (!priv->reg) {
1479                 dev_err(dev, "failed to of_iomap\n");
1480                 err = -ENOMEM;
1481                 goto err_out;
1482         }
1483
1484         /* get SEC version capabilities from device tree */
1485         prop = of_get_property(np, "fsl,num-channels", NULL);
1486         if (prop)
1487                 priv->num_channels = *prop;
1488
1489         prop = of_get_property(np, "fsl,channel-fifo-len", NULL);
1490         if (prop)
1491                 priv->chfifo_len = *prop;
1492
1493         prop = of_get_property(np, "fsl,exec-units-mask", NULL);
1494         if (prop)
1495                 priv->exec_units = *prop;
1496
1497         prop = of_get_property(np, "fsl,descriptor-types-mask", NULL);
1498         if (prop)
1499                 priv->desc_types = *prop;
1500
1501         if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
1502             !priv->exec_units || !priv->desc_types) {
1503                 dev_err(dev, "invalid property data in device tree node\n");
1504                 err = -EINVAL;
1505                 goto err_out;
1506         }
1507
1508         if (of_device_is_compatible(np, "fsl,sec3.0"))
1509                 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
1510
1511         priv->head_lock = kmalloc(sizeof(spinlock_t) * priv->num_channels,
1512                                   GFP_KERNEL);
1513         priv->tail_lock = kmalloc(sizeof(spinlock_t) * priv->num_channels,
1514                                   GFP_KERNEL);
1515         if (!priv->head_lock || !priv->tail_lock) {
1516                 dev_err(dev, "failed to allocate fifo locks\n");
1517                 err = -ENOMEM;
1518                 goto err_out;
1519         }
1520
1521         for (i = 0; i < priv->num_channels; i++) {
1522                 spin_lock_init(&priv->head_lock[i]);
1523                 spin_lock_init(&priv->tail_lock[i]);
1524         }
1525
1526         priv->fifo = kmalloc(sizeof(struct talitos_request *) *
1527                              priv->num_channels, GFP_KERNEL);
1528         if (!priv->fifo) {
1529                 dev_err(dev, "failed to allocate request fifo\n");
1530                 err = -ENOMEM;
1531                 goto err_out;
1532         }
1533
1534         priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
1535
1536         for (i = 0; i < priv->num_channels; i++) {
1537                 priv->fifo[i] = kzalloc(sizeof(struct talitos_request) *
1538                                         priv->fifo_len, GFP_KERNEL);
1539                 if (!priv->fifo[i]) {
1540                         dev_err(dev, "failed to allocate request fifo %d\n", i);
1541                         err = -ENOMEM;
1542                         goto err_out;
1543                 }
1544         }
1545
1546         priv->submit_count = kmalloc(sizeof(atomic_t) * priv->num_channels,
1547                                      GFP_KERNEL);
1548         if (!priv->submit_count) {
1549                 dev_err(dev, "failed to allocate fifo submit count space\n");
1550                 err = -ENOMEM;
1551                 goto err_out;
1552         }
1553         for (i = 0; i < priv->num_channels; i++)
1554                 atomic_set(&priv->submit_count[i], -priv->chfifo_len);
1555
1556         priv->head = kzalloc(sizeof(int) * priv->num_channels, GFP_KERNEL);
1557         priv->tail = kzalloc(sizeof(int) * priv->num_channels, GFP_KERNEL);
1558         if (!priv->head || !priv->tail) {
1559                 dev_err(dev, "failed to allocate request index space\n");
1560                 err = -ENOMEM;
1561                 goto err_out;
1562         }
1563
1564         /* reset and initialize the h/w */
1565         err = init_device(dev);
1566         if (err) {
1567                 dev_err(dev, "failed to initialize device\n");
1568                 goto err_out;
1569         }
1570
1571         /* register the RNG, if available */
1572         if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
1573                 err = talitos_register_rng(dev);
1574                 if (err) {
1575                         dev_err(dev, "failed to register hwrng: %d\n", err);
1576                         goto err_out;
1577                 } else
1578                         dev_info(dev, "hwrng\n");
1579         }
1580
1581         /* register crypto algorithms the device supports */
1582         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
1583                 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
1584                         struct talitos_crypto_alg *t_alg;
1585
1586                         t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
1587                         if (IS_ERR(t_alg)) {
1588                                 err = PTR_ERR(t_alg);
1589                                 goto err_out;
1590                         }
1591
1592                         err = crypto_register_alg(&t_alg->crypto_alg);
1593                         if (err) {
1594                                 dev_err(dev, "%s alg registration failed\n",
1595                                         t_alg->crypto_alg.cra_driver_name);
1596                                 kfree(t_alg);
1597                         } else {
1598                                 list_add_tail(&t_alg->entry, &priv->alg_list);
1599                                 dev_info(dev, "%s\n",
1600                                          t_alg->crypto_alg.cra_driver_name);
1601                         }
1602                 }
1603         }
1604
1605         return 0;
1606
1607 err_out:
1608         talitos_remove(ofdev);
1609
1610         return err;
1611 }
1612
1613 static struct of_device_id talitos_match[] = {
1614         {
1615                 .compatible = "fsl,sec2.0",
1616         },
1617         {},
1618 };
1619 MODULE_DEVICE_TABLE(of, talitos_match);
1620
1621 static struct of_platform_driver talitos_driver = {
1622         .name = "talitos",
1623         .match_table = talitos_match,
1624         .probe = talitos_probe,
1625         .remove = __devexit_p(talitos_remove),
1626 };
1627
1628 static int __init talitos_init(void)
1629 {
1630         return of_register_platform_driver(&talitos_driver);
1631 }
1632 module_init(talitos_init);
1633
1634 static void __exit talitos_exit(void)
1635 {
1636         of_unregister_platform_driver(&talitos_driver);
1637 }
1638 module_exit(talitos_exit);
1639
1640 MODULE_LICENSE("GPL");
1641 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
1642 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");