drivers/crypto/omap-aes.c

   1 /*
   2  * Cryptographic API.
   3  *
   4  * Support for OMAP AES HW acceleration.
   5  *
   6  * Copyright (c) 2010 Nokia Corporation
   7  * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 as published
  11  * by the Free Software Foundation.
  12  *
  13  */
  14
  15 #define pr_fmt(fmt) "%s: " fmt, __func__
  16
  17 #include <linux/err.h>
  18 #include <linux/module.h>
  19 #include <linux/init.h>
  20 #include <linux/errno.h>
  21 #include <linux/kernel.h>
  22 #include <linux/clk.h>
  23 #include <linux/platform_device.h>
  24 #include <linux/scatterlist.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/io.h>
  27 #include <linux/crypto.h>
  28 #include <linux/interrupt.h>
  29 #include <crypto/scatterwalk.h>
  30 #include <crypto/aes.h>
  31
  32 #include <linux/omap-dma.h>
  33
  34 /* OMAP TRM gives bitfields as start:end, where start is the higher bit
  35    number. For example 7:0 */
  36 #define FLD_MASK(start, end)    (((1 << ((start) - (end) + 1)) - 1) << (end))
  37 #define FLD_VAL(val, start, end) (((val) << (end)) & FLD_MASK(start, end))
  38
  39 #define AES_REG_KEY(x)                  (0x1C - ((x ^ 0x01) * 0x04))
  40 #define AES_REG_IV(x)                   (0x20 + ((x) * 0x04))
  41
  42 #define AES_REG_CTRL                    0x30
  43 #define AES_REG_CTRL_CTR_WIDTH          (1 << 7)
  44 #define AES_REG_CTRL_CTR                (1 << 6)
  45 #define AES_REG_CTRL_CBC                (1 << 5)
  46 #define AES_REG_CTRL_KEY_SIZE           (3 << 3)
  47 #define AES_REG_CTRL_DIRECTION          (1 << 2)
  48 #define AES_REG_CTRL_INPUT_READY        (1 << 1)
  49 #define AES_REG_CTRL_OUTPUT_READY       (1 << 0)
  50
  51 #define AES_REG_DATA                    0x34
  52 #define AES_REG_DATA_N(x)               (0x34 + ((x) * 0x04))
  53
  54 #define AES_REG_REV                     0x44
  55 #define AES_REG_REV_MAJOR               0xF0
  56 #define AES_REG_REV_MINOR               0x0F
  57
  58 #define AES_REG_MASK                    0x48
  59 #define AES_REG_MASK_SIDLE              (1 << 6)
  60 #define AES_REG_MASK_START              (1 << 5)
  61 #define AES_REG_MASK_DMA_OUT_EN         (1 << 3)
  62 #define AES_REG_MASK_DMA_IN_EN          (1 << 2)
  63 #define AES_REG_MASK_SOFTRESET          (1 << 1)
  64 #define AES_REG_AUTOIDLE                (1 << 0)
  65
  66 #define AES_REG_SYSSTATUS               0x4C
  67 #define AES_REG_SYSSTATUS_RESETDONE     (1 << 0)
  68
  69 #define DEFAULT_TIMEOUT         (5*HZ)
  70
  71 #define FLAGS_MODE_MASK         0x000f
  72 #define FLAGS_ENCRYPT           BIT(0)
  73 #define FLAGS_CBC               BIT(1)
  74 #define FLAGS_GIV               BIT(2)
  75
  76 #define FLAGS_INIT              BIT(4)
  77 #define FLAGS_FAST              BIT(5)
  78 #define FLAGS_BUSY              BIT(6)
  79
  80 struct omap_aes_ctx {
  81         struct omap_aes_dev *dd;
  82
  83         int             keylen;
  84         u32             key[AES_KEYSIZE_256 / sizeof(u32)];
  85         unsigned long   flags;
  86 };
  87
  88 struct omap_aes_reqctx {
  89         unsigned long mode;
  90 };
  91
  92 #define OMAP_AES_QUEUE_LENGTH   1
  93 #define OMAP_AES_CACHE_SIZE     0
  94
  95 struct omap_aes_dev {
  96         struct list_head        list;
  97         unsigned long           phys_base;
  98         void __iomem            *io_base;
  99         struct clk              *iclk;
 100         struct omap_aes_ctx     *ctx;
 101         struct device           *dev;
 102         unsigned long           flags;
 103         int                     err;
 104
 105         spinlock_t              lock;
 106         struct crypto_queue     queue;
 107
 108         struct tasklet_struct   done_task;
 109         struct tasklet_struct   queue_task;
 110
 111         struct ablkcipher_request       *req;
 112         size_t                          total;
 113         struct scatterlist              *in_sg;
 114         size_t                          in_offset;
 115         struct scatterlist              *out_sg;
 116         size_t                          out_offset;
 117
 118         size_t                  buflen;
 119         void                    *buf_in;
 120         size_t                  dma_size;
 121         int                     dma_in;
 122         int                     dma_lch_in;
 123         dma_addr_t              dma_addr_in;
 124         void                    *buf_out;
 125         int                     dma_out;
 126         int                     dma_lch_out;
 127         dma_addr_t              dma_addr_out;
 128 };
 129
 130 /* keep registered devices data here */
 131 static LIST_HEAD(dev_list);
 132 static DEFINE_SPINLOCK(list_lock);
 133
 134 static inline u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset)
 135 {
 136         return __raw_readl(dd->io_base + offset);
 137 }
 138
 139 static inline void omap_aes_write(struct omap_aes_dev *dd, u32 offset,
 140                                   u32 value)
 141 {
 142         __raw_writel(value, dd->io_base + offset);
 143 }
 144
 145 static inline void omap_aes_write_mask(struct omap_aes_dev *dd, u32 offset,
 146                                         u32 value, u32 mask)
 147 {
 148         u32 val;
 149
 150         val = omap_aes_read(dd, offset);
 151         val &= ~mask;
 152         val |= value;
 153         omap_aes_write(dd, offset, val);
 154 }
 155
 156 static void omap_aes_write_n(struct omap_aes_dev *dd, u32 offset,
 157                                         u32 *value, int count)
 158 {
 159         for (; count--; value++, offset += 4)
 160                 omap_aes_write(dd, offset, *value);
 161 }
 162
 163 static int omap_aes_wait(struct omap_aes_dev *dd, u32 offset, u32 bit)
 164 {
 165         unsigned long timeout = jiffies + DEFAULT_TIMEOUT;
 166
 167         while (!(omap_aes_read(dd, offset) & bit)) {
 168                 if (time_is_before_jiffies(timeout)) {
 169                         dev_err(dd->dev, "omap-aes timeout\n");
 170                         return -ETIMEDOUT;
 171                 }
 172         }
 173         return 0;
 174 }
 175
 176 static int omap_aes_hw_init(struct omap_aes_dev *dd)
 177 {
 178         /*
 179          * clocks are enabled when request starts and disabled when finished.
 180          * It may be long delays between requests.
 181          * Device might go to off mode to save power.
 182          */
 183         clk_enable(dd->iclk);
 184
 185         if (!(dd->flags & FLAGS_INIT)) {
 186                 /* is it necessary to reset before every operation? */
 187                 omap_aes_write_mask(dd, AES_REG_MASK, AES_REG_MASK_SOFTRESET,
 188                                         AES_REG_MASK_SOFTRESET);
 189                 /*
 190                  * prevent OCP bus error (SRESP) in case an access to the module
 191                  * is performed while the module is coming out of soft reset
 192                  */
 193                 __asm__ __volatile__("nop");
 194                 __asm__ __volatile__("nop");
 195
 196                 if (omap_aes_wait(dd, AES_REG_SYSSTATUS,
 197                                 AES_REG_SYSSTATUS_RESETDONE))
 198                         return -ETIMEDOUT;
 199
 200                 dd->flags |= FLAGS_INIT;
 201                 dd->err = 0;
 202         }
 203
 204         return 0;
 205 }
 206
 207 static int omap_aes_write_ctrl(struct omap_aes_dev *dd)
 208 {
 209         unsigned int key32;
 210         int i, err;
 211         u32 val, mask;
 212
 213         err = omap_aes_hw_init(dd);
 214         if (err)
 215                 return err;
 216
 217         val = 0;
 218         if (dd->dma_lch_out >= 0)
 219                 val |= AES_REG_MASK_DMA_OUT_EN;
 220         if (dd->dma_lch_in >= 0)
 221                 val |= AES_REG_MASK_DMA_IN_EN;
 222
 223         mask = AES_REG_MASK_DMA_IN_EN | AES_REG_MASK_DMA_OUT_EN;
 224
 225         omap_aes_write_mask(dd, AES_REG_MASK, val, mask);
 226
 227         key32 = dd->ctx->keylen / sizeof(u32);
 228
 229         /* it seems a key should always be set even if it has not changed */
 230         for (i = 0; i < key32; i++) {
 231                 omap_aes_write(dd, AES_REG_KEY(i),
 232                         __le32_to_cpu(dd->ctx->key[i]));
 233         }
 234
 235         if ((dd->flags & FLAGS_CBC) && dd->req->info)
 236                 omap_aes_write_n(dd, AES_REG_IV(0), dd->req->info, 4);
 237
 238         val = FLD_VAL(((dd->ctx->keylen >> 3) - 1), 4, 3);
 239         if (dd->flags & FLAGS_CBC)
 240                 val |= AES_REG_CTRL_CBC;
 241         if (dd->flags & FLAGS_ENCRYPT)
 242                 val |= AES_REG_CTRL_DIRECTION;
 243
 244         mask = AES_REG_CTRL_CBC | AES_REG_CTRL_DIRECTION |
 245                         AES_REG_CTRL_KEY_SIZE;
 246
 247         omap_aes_write_mask(dd, AES_REG_CTRL, val, mask);
 248
 249         /* IN */
 250         omap_set_dma_dest_params(dd->dma_lch_in, 0, OMAP_DMA_AMODE_CONSTANT,
 251                                  dd->phys_base + AES_REG_DATA, 0, 4);
 252
 253         omap_set_dma_dest_burst_mode(dd->dma_lch_in, OMAP_DMA_DATA_BURST_4);
 254         omap_set_dma_src_burst_mode(dd->dma_lch_in, OMAP_DMA_DATA_BURST_4);
 255
 256         /* OUT */
 257         omap_set_dma_src_params(dd->dma_lch_out, 0, OMAP_DMA_AMODE_CONSTANT,
 258                                 dd->phys_base + AES_REG_DATA, 0, 4);
 259
 260         omap_set_dma_src_burst_mode(dd->dma_lch_out, OMAP_DMA_DATA_BURST_4);
 261         omap_set_dma_dest_burst_mode(dd->dma_lch_out, OMAP_DMA_DATA_BURST_4);
 262
 263         return 0;
 264 }
 265
 266 static struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_ctx *ctx)
 267 {
 268         struct omap_aes_dev *dd = NULL, *tmp;
 269
 270         spin_lock_bh(&list_lock);
 271         if (!ctx->dd) {
 272                 list_for_each_entry(tmp, &dev_list, list) {
 273                         /* FIXME: take fist available aes core */
 274                         dd = tmp;
 275                         break;
 276                 }
 277                 ctx->dd = dd;
 278         } else {
 279                 /* already found before */
 280                 dd = ctx->dd;
 281         }
 282         spin_unlock_bh(&list_lock);
 283
 284         return dd;
 285 }
 286
 287 static void omap_aes_dma_callback(int lch, u16 ch_status, void *data)
 288 {
 289         struct omap_aes_dev *dd = data;
 290
 291         if (ch_status != OMAP_DMA_BLOCK_IRQ) {
 292                 pr_err("omap-aes DMA error status: 0x%hx\n", ch_status);
 293                 dd->err = -EIO;
 294                 dd->flags &= ~FLAGS_INIT; /* request to re-initialize */
 295         } else if (lch == dd->dma_lch_in) {
 296                 return;
 297         }
 298
 299         /* dma_lch_out - completed */
 300         tasklet_schedule(&dd->done_task);
 301 }
 302
 303 static int omap_aes_dma_init(struct omap_aes_dev *dd)
 304 {
 305         int err = -ENOMEM;
 306
 307         dd->dma_lch_out = -1;
 308         dd->dma_lch_in = -1;
 309
 310         dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, OMAP_AES_CACHE_SIZE);
 311         dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, OMAP_AES_CACHE_SIZE);
 312         dd->buflen = PAGE_SIZE << OMAP_AES_CACHE_SIZE;
 313         dd->buflen &= ~(AES_BLOCK_SIZE - 1);
 314
 315         if (!dd->buf_in || !dd->buf_out) {
 316                 dev_err(dd->dev, "unable to alloc pages.\n");
 317                 goto err_alloc;
 318         }
 319
 320         /* MAP here */
 321         dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in, dd->buflen,
 322                                          DMA_TO_DEVICE);
 323         if (dma_mapping_error(dd->dev, dd->dma_addr_in)) {
 324                 dev_err(dd->dev, "dma %d bytes error\n", dd->buflen);
 325                 err = -EINVAL;
 326                 goto err_map_in;
 327         }
 328
 329         dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out, dd->buflen,
 330                                           DMA_FROM_DEVICE);
 331         if (dma_mapping_error(dd->dev, dd->dma_addr_out)) {
 332                 dev_err(dd->dev, "dma %d bytes error\n", dd->buflen);
 333                 err = -EINVAL;
 334                 goto err_map_out;
 335         }
 336
 337         err = omap_request_dma(dd->dma_in, "omap-aes-rx",
 338                                omap_aes_dma_callback, dd, &dd->dma_lch_in);
 339         if (err) {
 340                 dev_err(dd->dev, "Unable to request DMA channel\n");
 341                 goto err_dma_in;
 342         }
 343         err = omap_request_dma(dd->dma_out, "omap-aes-tx",
 344                                omap_aes_dma_callback, dd, &dd->dma_lch_out);
 345         if (err) {
 346                 dev_err(dd->dev, "Unable to request DMA channel\n");
 347                 goto err_dma_out;
 348         }
 349
 350         return 0;
 351
 352 err_dma_out:
 353         omap_free_dma(dd->dma_lch_in);
 354 err_dma_in:
 355         dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen,
 356                          DMA_FROM_DEVICE);
 357 err_map_out:
 358         dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE);
 359 err_map_in:
 360         free_pages((unsigned long)dd->buf_out, OMAP_AES_CACHE_SIZE);
 361         free_pages((unsigned long)dd->buf_in, OMAP_AES_CACHE_SIZE);
 362 err_alloc:
 363         if (err)
 364                 pr_err("error: %d\n", err);
 365         return err;
 366 }
 367
 368 static void omap_aes_dma_cleanup(struct omap_aes_dev *dd)
 369 {
 370         omap_free_dma(dd->dma_lch_out);
 371         omap_free_dma(dd->dma_lch_in);
 372         dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen,
 373                          DMA_FROM_DEVICE);
 374         dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE);
 375         free_pages((unsigned long)dd->buf_out, OMAP_AES_CACHE_SIZE);
 376         free_pages((unsigned long)dd->buf_in, OMAP_AES_CACHE_SIZE);
 377 }
 378
 379 static void sg_copy_buf(void *buf, struct scatterlist *sg,
 380                               unsigned int start, unsigned int nbytes, int out)
 381 {
 382         struct scatter_walk walk;
 383
 384         if (!nbytes)
 385                 return;
 386
 387         scatterwalk_start(&walk, sg);
 388         scatterwalk_advance(&walk, start);
 389         scatterwalk_copychunks(buf, &walk, nbytes, out);
 390         scatterwalk_done(&walk, out, 0);
 391 }
 392
 393 static int sg_copy(struct scatterlist **sg, size_t *offset, void *buf,
 394                    size_t buflen, size_t total, int out)
 395 {
 396         unsigned int count, off = 0;
 397
 398         while (buflen && total) {
 399                 count = min((*sg)->length - *offset, total);
 400                 count = min(count, buflen);
 401
 402                 if (!count)
 403                         return off;
 404
 405                 /*
 406                  * buflen and total are AES_BLOCK_SIZE size aligned,
 407                  * so count should be also aligned
 408                  */
 409
 410                 sg_copy_buf(buf + off, *sg, *offset, count, out);
 411
 412                 off += count;
 413                 buflen -= count;
 414                 *offset += count;
 415                 total -= count;
 416
 417                 if (*offset == (*sg)->length) {
 418                         *sg = sg_next(*sg);
 419                         if (*sg)
 420                                 *offset = 0;
 421                         else
 422                                 total = 0;
 423                 }
 424         }
 425
 426         return off;
 427 }
 428
 429 static int omap_aes_crypt_dma(struct crypto_tfm *tfm, dma_addr_t dma_addr_in,
 430                                dma_addr_t dma_addr_out, int length)
 431 {
 432         struct omap_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 433         struct omap_aes_dev *dd = ctx->dd;
 434         int len32;
 435
 436         pr_debug("len: %d\n", length);
 437
 438         dd->dma_size = length;
 439
 440         if (!(dd->flags & FLAGS_FAST))
 441                 dma_sync_single_for_device(dd->dev, dma_addr_in, length,
 442                                            DMA_TO_DEVICE);
 443
 444         len32 = DIV_ROUND_UP(length, sizeof(u32));
 445
 446         /* IN */
 447         omap_set_dma_transfer_params(dd->dma_lch_in, OMAP_DMA_DATA_TYPE_S32,
 448                                      len32, 1, OMAP_DMA_SYNC_PACKET, dd->dma_in,
 449                                         OMAP_DMA_DST_SYNC);
 450
 451         omap_set_dma_src_params(dd->dma_lch_in, 0, OMAP_DMA_AMODE_POST_INC,
 452                                 dma_addr_in, 0, 0);
 453
 454         /* OUT */
 455         omap_set_dma_transfer_params(dd->dma_lch_out, OMAP_DMA_DATA_TYPE_S32,
 456                                      len32, 1, OMAP_DMA_SYNC_PACKET,
 457                                         dd->dma_out, OMAP_DMA_SRC_SYNC);
 458
 459         omap_set_dma_dest_params(dd->dma_lch_out, 0, OMAP_DMA_AMODE_POST_INC,
 460                                  dma_addr_out, 0, 0);
 461
 462         omap_start_dma(dd->dma_lch_in);
 463         omap_start_dma(dd->dma_lch_out);
 464
 465         /* start DMA or disable idle mode */
 466         omap_aes_write_mask(dd, AES_REG_MASK, AES_REG_MASK_START,
 467                             AES_REG_MASK_START);
 468
 469         return 0;
 470 }
 471
 472 static int omap_aes_crypt_dma_start(struct omap_aes_dev *dd)
 473 {
 474         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(
 475                                         crypto_ablkcipher_reqtfm(dd->req));
 476         int err, fast = 0, in, out;
 477         size_t count;
 478         dma_addr_t addr_in, addr_out;
 479
 480         pr_debug("total: %d\n", dd->total);
 481
 482         if (sg_is_last(dd->in_sg) && sg_is_last(dd->out_sg)) {
 483                 /* check for alignment */
 484                 in = IS_ALIGNED((u32)dd->in_sg->offset, sizeof(u32));
 485                 out = IS_ALIGNED((u32)dd->out_sg->offset, sizeof(u32));
 486
 487                 fast = in && out;
 488         }
 489
 490         if (fast)  {
 491                 count = min(dd->total, sg_dma_len(dd->in_sg));
 492                 count = min(count, sg_dma_len(dd->out_sg));
 493
 494                 if (count != dd->total) {
 495                         pr_err("request length != buffer length\n");
 496                         return -EINVAL;
 497                 }
 498
 499                 pr_debug("fast\n");
 500
 501                 err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 502                 if (!err) {
 503                         dev_err(dd->dev, "dma_map_sg() error\n");
 504                         return -EINVAL;
 505                 }
 506
 507                 err = dma_map_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
 508                 if (!err) {
 509                         dev_err(dd->dev, "dma_map_sg() error\n");
 510                         dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 511                         return -EINVAL;
 512                 }
 513
 514                 addr_in = sg_dma_address(dd->in_sg);
 515                 addr_out = sg_dma_address(dd->out_sg);
 516
 517                 dd->flags |= FLAGS_FAST;
 518
 519         } else {
 520                 /* use cache buffers */
 521                 count = sg_copy(&dd->in_sg, &dd->in_offset, dd->buf_in,
 522                                  dd->buflen, dd->total, 0);
 523
 524                 addr_in = dd->dma_addr_in;
 525                 addr_out = dd->dma_addr_out;
 526
 527                 dd->flags &= ~FLAGS_FAST;
 528
 529         }
 530
 531         dd->total -= count;
 532
 533         err = omap_aes_crypt_dma(tfm, addr_in, addr_out, count);
 534         if (err) {
 535                 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 536                 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE);
 537         }
 538
 539         return err;
 540 }
 541
 542 static void omap_aes_finish_req(struct omap_aes_dev *dd, int err)
 543 {
 544         struct ablkcipher_request *req = dd->req;
 545
 546         pr_debug("err: %d\n", err);
 547
 548         clk_disable(dd->iclk);
 549         dd->flags &= ~FLAGS_BUSY;
 550
 551         req->base.complete(&req->base, err);
 552 }
 553
 554 static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd)
 555 {
 556         int err = 0;
 557         size_t count;
 558
 559         pr_debug("total: %d\n", dd->total);
 560
 561         omap_aes_write_mask(dd, AES_REG_MASK, 0, AES_REG_MASK_START);
 562
 563         omap_stop_dma(dd->dma_lch_in);
 564         omap_stop_dma(dd->dma_lch_out);
 565
 566         if (dd->flags & FLAGS_FAST) {
 567                 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
 568                 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 569         } else {
 570                 dma_sync_single_for_device(dd->dev, dd->dma_addr_out,
 571                                            dd->dma_size, DMA_FROM_DEVICE);
 572
 573                 /* copy data */
 574                 count = sg_copy(&dd->out_sg, &dd->out_offset, dd->buf_out,
 575                                  dd->buflen, dd->dma_size, 1);
 576                 if (count != dd->dma_size) {
 577                         err = -EINVAL;
 578                         pr_err("not all data converted: %u\n", count);
 579                 }
 580         }
 581
 582         return err;
 583 }
 584
 585 static int omap_aes_handle_queue(struct omap_aes_dev *dd,
 586                                struct ablkcipher_request *req)
 587 {
 588         struct crypto_async_request *async_req, *backlog;
 589         struct omap_aes_ctx *ctx;
 590         struct omap_aes_reqctx *rctx;
 591         unsigned long flags;
 592         int err, ret = 0;
 593
 594         spin_lock_irqsave(&dd->lock, flags);
 595         if (req)
 596                 ret = ablkcipher_enqueue_request(&dd->queue, req);
 597         if (dd->flags & FLAGS_BUSY) {
 598                 spin_unlock_irqrestore(&dd->lock, flags);
 599                 return ret;
 600         }
 601         backlog = crypto_get_backlog(&dd->queue);
 602         async_req = crypto_dequeue_request(&dd->queue);
 603         if (async_req)
 604                 dd->flags |= FLAGS_BUSY;
 605         spin_unlock_irqrestore(&dd->lock, flags);
 606
 607         if (!async_req)
 608                 return ret;
 609
 610         if (backlog)
 611                 backlog->complete(backlog, -EINPROGRESS);
 612
 613         req = ablkcipher_request_cast(async_req);
 614
 615         /* assign new request to device */
 616         dd->req = req;
 617         dd->total = req->nbytes;
 618         dd->in_offset = 0;
 619         dd->in_sg = req->src;
 620         dd->out_offset = 0;
 621         dd->out_sg = req->dst;
 622
 623         rctx = ablkcipher_request_ctx(req);
 624         ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
 625         rctx->mode &= FLAGS_MODE_MASK;
 626         dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
 627
 628         dd->ctx = ctx;
 629         ctx->dd = dd;
 630
 631         err = omap_aes_write_ctrl(dd);
 632         if (!err)
 633                 err = omap_aes_crypt_dma_start(dd);
 634         if (err) {
 635                 /* aes_task will not finish it, so do it here */
 636                 omap_aes_finish_req(dd, err);
 637                 tasklet_schedule(&dd->queue_task);
 638         }
 639
 640         return ret; /* return ret, which is enqueue return value */
 641 }
 642
 643 static void omap_aes_done_task(unsigned long data)
 644 {
 645         struct omap_aes_dev *dd = (struct omap_aes_dev *)data;
 646         int err;
 647
 648         pr_debug("enter\n");
 649
 650         err = omap_aes_crypt_dma_stop(dd);
 651
 652         err = dd->err ? : err;
 653
 654         if (dd->total && !err) {
 655                 err = omap_aes_crypt_dma_start(dd);
 656                 if (!err)
 657                         return; /* DMA started. Not fininishing. */
 658         }
 659
 660         omap_aes_finish_req(dd, err);
 661         omap_aes_handle_queue(dd, NULL);
 662
 663         pr_debug("exit\n");
 664 }
 665
 666 static void omap_aes_queue_task(unsigned long data)
 667 {
 668         struct omap_aes_dev *dd = (struct omap_aes_dev *)data;
 669
 670         omap_aes_handle_queue(dd, NULL);
 671 }
 672
 673 static int omap_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
 674 {
 675         struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx(
 676                         crypto_ablkcipher_reqtfm(req));
 677         struct omap_aes_reqctx *rctx = ablkcipher_request_ctx(req);
 678         struct omap_aes_dev *dd;
 679
 680         pr_debug("nbytes: %d, enc: %d, cbc: %d\n", req->nbytes,
 681                   !!(mode & FLAGS_ENCRYPT),
 682                   !!(mode & FLAGS_CBC));
 683
 684         if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) {
 685                 pr_err("request size is not exact amount of AES blocks\n");
 686                 return -EINVAL;
 687         }
 688
 689         dd = omap_aes_find_dev(ctx);
 690         if (!dd)
 691                 return -ENODEV;
 692
 693         rctx->mode = mode;
 694
 695         return omap_aes_handle_queue(dd, req);
 696 }
 697
 698 /* ********************** ALG API ************************************ */
 699
 700 static int omap_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 701                            unsigned int keylen)
 702 {
 703         struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 704
 705         if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
 706                    keylen != AES_KEYSIZE_256)
 707                 return -EINVAL;
 708
 709         pr_debug("enter, keylen: %d\n", keylen);
 710
 711         memcpy(ctx->key, key, keylen);
 712         ctx->keylen = keylen;
 713
 714         return 0;
 715 }
 716
 717 static int omap_aes_ecb_encrypt(struct ablkcipher_request *req)
 718 {
 719         return omap_aes_crypt(req, FLAGS_ENCRYPT);
 720 }
 721
 722 static int omap_aes_ecb_decrypt(struct ablkcipher_request *req)
 723 {
 724         return omap_aes_crypt(req, 0);
 725 }
 726
 727 static int omap_aes_cbc_encrypt(struct ablkcipher_request *req)
 728 {
 729         return omap_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC);
 730 }
 731
 732 static int omap_aes_cbc_decrypt(struct ablkcipher_request *req)
 733 {
 734         return omap_aes_crypt(req, FLAGS_CBC);
 735 }
 736
 737 static int omap_aes_cra_init(struct crypto_tfm *tfm)
 738 {
 739         pr_debug("enter\n");
 740
 741         tfm->crt_ablkcipher.reqsize = sizeof(struct omap_aes_reqctx);
 742
 743         return 0;
 744 }
 745
 746 static void omap_aes_cra_exit(struct crypto_tfm *tfm)
 747 {
 748         pr_debug("enter\n");
 749 }
 750
 751 /* ********************** ALGS ************************************ */
 752
 753 static struct crypto_alg algs[] = {
 754 {
 755         .cra_name               = "ecb(aes)",
 756         .cra_driver_name        = "ecb-aes-omap",
 757         .cra_priority           = 100,
 758         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
 759                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
 760                                   CRYPTO_ALG_ASYNC,
 761         .cra_blocksize          = AES_BLOCK_SIZE,
 762         .cra_ctxsize            = sizeof(struct omap_aes_ctx),
 763         .cra_alignmask          = 0,
 764         .cra_type               = &crypto_ablkcipher_type,
 765         .cra_module             = THIS_MODULE,
 766         .cra_init               = omap_aes_cra_init,
 767         .cra_exit               = omap_aes_cra_exit,
 768         .cra_u.ablkcipher = {
 769                 .min_keysize    = AES_MIN_KEY_SIZE,
 770                 .max_keysize    = AES_MAX_KEY_SIZE,
 771                 .setkey         = omap_aes_setkey,
 772                 .encrypt        = omap_aes_ecb_encrypt,
 773                 .decrypt        = omap_aes_ecb_decrypt,
 774         }
 775 },
 776 {
 777         .cra_name               = "cbc(aes)",
 778         .cra_driver_name        = "cbc-aes-omap",
 779         .cra_priority           = 100,
 780         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
 781                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
 782                                   CRYPTO_ALG_ASYNC,
 783         .cra_blocksize          = AES_BLOCK_SIZE,
 784         .cra_ctxsize            = sizeof(struct omap_aes_ctx),
 785         .cra_alignmask          = 0,
 786         .cra_type               = &crypto_ablkcipher_type,
 787         .cra_module             = THIS_MODULE,
 788         .cra_init               = omap_aes_cra_init,
 789         .cra_exit               = omap_aes_cra_exit,
 790         .cra_u.ablkcipher = {
 791                 .min_keysize    = AES_MIN_KEY_SIZE,
 792                 .max_keysize    = AES_MAX_KEY_SIZE,
 793                 .ivsize         = AES_BLOCK_SIZE,
 794                 .setkey         = omap_aes_setkey,
 795                 .encrypt        = omap_aes_cbc_encrypt,
 796                 .decrypt        = omap_aes_cbc_decrypt,
 797         }
 798 }
 799 };
 800
 801 static int omap_aes_probe(struct platform_device *pdev)
 802 {
 803         struct device *dev = &pdev->dev;
 804         struct omap_aes_dev *dd;
 805         struct resource *res;
 806         int err = -ENOMEM, i, j;
 807         u32 reg;
 808
 809         dd = kzalloc(sizeof(struct omap_aes_dev), GFP_KERNEL);
 810         if (dd == NULL) {
 811                 dev_err(dev, "unable to alloc data struct.\n");
 812                 goto err_data;
 813         }
 814         dd->dev = dev;
 815         platform_set_drvdata(pdev, dd);
 816
 817         spin_lock_init(&dd->lock);
 818         crypto_init_queue(&dd->queue, OMAP_AES_QUEUE_LENGTH);
 819
 820         /* Get the base address */
 821         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 822         if (!res) {
 823                 dev_err(dev, "invalid resource type\n");
 824                 err = -ENODEV;
 825                 goto err_res;
 826         }
 827         dd->phys_base = res->start;
 828
 829         /* Get the DMA */
 830         res = platform_get_resource(pdev, IORESOURCE_DMA, 0);
 831         if (!res)
 832                 dev_info(dev, "no DMA info\n");
 833         else
 834                 dd->dma_out = res->start;
 835
 836         /* Get the DMA */
 837         res = platform_get_resource(pdev, IORESOURCE_DMA, 1);
 838         if (!res)
 839                 dev_info(dev, "no DMA info\n");
 840         else
 841                 dd->dma_in = res->start;
 842
 843         /* Initializing the clock */
 844         dd->iclk = clk_get(dev, "ick");
 845         if (IS_ERR(dd->iclk)) {
 846                 dev_err(dev, "clock intialization failed.\n");
 847                 err = PTR_ERR(dd->iclk);
 848                 goto err_res;
 849         }
 850
 851         dd->io_base = ioremap(dd->phys_base, SZ_4K);
 852         if (!dd->io_base) {
 853                 dev_err(dev, "can't ioremap\n");
 854                 err = -ENOMEM;
 855                 goto err_io;
 856         }
 857
 858         clk_enable(dd->iclk);
 859         reg = omap_aes_read(dd, AES_REG_REV);
 860         dev_info(dev, "OMAP AES hw accel rev: %u.%u\n",
 861                  (reg & AES_REG_REV_MAJOR) >> 4, reg & AES_REG_REV_MINOR);
 862         clk_disable(dd->iclk);
 863
 864         tasklet_init(&dd->done_task, omap_aes_done_task, (unsigned long)dd);
 865         tasklet_init(&dd->queue_task, omap_aes_queue_task, (unsigned long)dd);
 866
 867         err = omap_aes_dma_init(dd);
 868         if (err)
 869                 goto err_dma;
 870
 871         INIT_LIST_HEAD(&dd->list);
 872         spin_lock(&list_lock);
 873         list_add_tail(&dd->list, &dev_list);
 874         spin_unlock(&list_lock);
 875
 876         for (i = 0; i < ARRAY_SIZE(algs); i++) {
 877                 pr_debug("i: %d\n", i);
 878                 err = crypto_register_alg(&algs[i]);
 879                 if (err)
 880                         goto err_algs;
 881         }
 882
 883         pr_info("probe() done\n");
 884
 885         return 0;
 886 err_algs:
 887         for (j = 0; j < i; j++)
 888                 crypto_unregister_alg(&algs[j]);
 889         omap_aes_dma_cleanup(dd);
 890 err_dma:
 891         tasklet_kill(&dd->done_task);
 892         tasklet_kill(&dd->queue_task);
 893         iounmap(dd->io_base);
 894 err_io:
 895         clk_put(dd->iclk);
 896 err_res:
 897         kfree(dd);
 898         dd = NULL;
 899 err_data:
 900         dev_err(dev, "initialization failed.\n");
 901         return err;
 902 }
 903
 904 static int omap_aes_remove(struct platform_device *pdev)
 905 {
 906         struct omap_aes_dev *dd = platform_get_drvdata(pdev);
 907         int i;
 908
 909         if (!dd)
 910                 return -ENODEV;
 911
 912         spin_lock(&list_lock);
 913         list_del(&dd->list);
 914         spin_unlock(&list_lock);
 915
 916         for (i = 0; i < ARRAY_SIZE(algs); i++)
 917                 crypto_unregister_alg(&algs[i]);
 918
 919         tasklet_kill(&dd->done_task);
 920         tasklet_kill(&dd->queue_task);
 921         omap_aes_dma_cleanup(dd);
 922         iounmap(dd->io_base);
 923         clk_put(dd->iclk);
 924         kfree(dd);
 925         dd = NULL;
 926
 927         return 0;
 928 }
 929
 930 static struct platform_driver omap_aes_driver = {
 931         .probe  = omap_aes_probe,
 932         .remove = omap_aes_remove,
 933         .driver = {
 934                 .name   = "omap-aes",
 935                 .owner  = THIS_MODULE,
 936         },
 937 };
 938
 939 static int __init omap_aes_mod_init(void)
 940 {
 941         pr_info("loading %s driver\n", "omap-aes");
 942
 943         return  platform_driver_register(&omap_aes_driver);
 944 }
 945
 946 static void __exit omap_aes_mod_exit(void)
 947 {
 948         platform_driver_unregister(&omap_aes_driver);
 949 }
 950
 951 module_init(omap_aes_mod_init);
 952 module_exit(omap_aes_mod_exit);
 953
 954 MODULE_DESCRIPTION("OMAP AES hw acceleration support.");
 955 MODULE_LICENSE("GPL v2");
 956 MODULE_AUTHOR("Dmitry Kasatkin");
 957