drivers/crypto/omap-aes.c

   1 /*
   2  * Cryptographic API.
   3  *
   4  * Support for OMAP AES HW acceleration.
   5  *
   6  * Copyright (c) 2010 Nokia Corporation
   7  * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 as published
  11  * by the Free Software Foundation.
  12  *
  13  */
  14
  15 #define pr_fmt(fmt) "%s: " fmt, __func__
  16
  17 #include <linux/err.h>
  18 #include <linux/module.h>
  19 #include <linux/init.h>
  20 #include <linux/errno.h>
  21 #include <linux/kernel.h>
  22 #include <linux/clk.h>
  23 #include <linux/platform_device.h>
  24 #include <linux/scatterlist.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/io.h>
  27 #include <linux/crypto.h>
  28 #include <linux/interrupt.h>
  29 #include <crypto/scatterwalk.h>
  30 #include <crypto/aes.h>
  31
  32 #include <plat/cpu.h>
  33 #include <plat/dma.h>
  34
  35 /* OMAP TRM gives bitfields as start:end, where start is the higher bit
  36    number. For example 7:0 */
  37 #define FLD_MASK(start, end)    (((1 << ((start) - (end) + 1)) - 1) << (end))
  38 #define FLD_VAL(val, start, end) (((val) << (end)) & FLD_MASK(start, end))
  39
  40 #define AES_REG_KEY(x)                  (0x1C - ((x ^ 0x01) * 0x04))
  41 #define AES_REG_IV(x)                   (0x20 + ((x) * 0x04))
  42
  43 #define AES_REG_CTRL                    0x30
  44 #define AES_REG_CTRL_CTR_WIDTH          (1 << 7)
  45 #define AES_REG_CTRL_CTR                (1 << 6)
  46 #define AES_REG_CTRL_CBC                (1 << 5)
  47 #define AES_REG_CTRL_KEY_SIZE           (3 << 3)
  48 #define AES_REG_CTRL_DIRECTION          (1 << 2)
  49 #define AES_REG_CTRL_INPUT_READY        (1 << 1)
  50 #define AES_REG_CTRL_OUTPUT_READY       (1 << 0)
  51
  52 #define AES_REG_DATA                    0x34
  53 #define AES_REG_DATA_N(x)               (0x34 + ((x) * 0x04))
  54
  55 #define AES_REG_REV                     0x44
  56 #define AES_REG_REV_MAJOR               0xF0
  57 #define AES_REG_REV_MINOR               0x0F
  58
  59 #define AES_REG_MASK                    0x48
  60 #define AES_REG_MASK_SIDLE              (1 << 6)
  61 #define AES_REG_MASK_START              (1 << 5)
  62 #define AES_REG_MASK_DMA_OUT_EN         (1 << 3)
  63 #define AES_REG_MASK_DMA_IN_EN          (1 << 2)
  64 #define AES_REG_MASK_SOFTRESET          (1 << 1)
  65 #define AES_REG_AUTOIDLE                (1 << 0)
  66
  67 #define AES_REG_SYSSTATUS               0x4C
  68 #define AES_REG_SYSSTATUS_RESETDONE     (1 << 0)
  69
  70 #define DEFAULT_TIMEOUT         (5*HZ)
  71
  72 #define FLAGS_MODE_MASK         0x000f
  73 #define FLAGS_ENCRYPT           BIT(0)
  74 #define FLAGS_CBC               BIT(1)
  75 #define FLAGS_GIV               BIT(2)
  76
  77 #define FLAGS_NEW_KEY           BIT(4)
  78 #define FLAGS_NEW_IV            BIT(5)
  79 #define FLAGS_INIT              BIT(6)
  80 #define FLAGS_FAST              BIT(7)
  81 #define FLAGS_BUSY              BIT(8)
  82
  83 struct omap_aes_ctx {
  84         struct omap_aes_dev *dd;
  85
  86         int             keylen;
  87         u32             key[AES_KEYSIZE_256 / sizeof(u32)];
  88         unsigned long   flags;
  89 };
  90
  91 struct omap_aes_reqctx {
  92         unsigned long mode;
  93 };
  94
  95 #define OMAP_AES_QUEUE_LENGTH   1
  96 #define OMAP_AES_CACHE_SIZE     0
  97
  98 struct omap_aes_dev {
  99         struct list_head        list;
 100         unsigned long           phys_base;
 101         void __iomem            *io_base;
 102         struct clk              *iclk;
 103         struct omap_aes_ctx     *ctx;
 104         struct device           *dev;
 105         unsigned long           flags;
 106
 107         u32                     *iv;
 108         u32                     ctrl;
 109
 110         spinlock_t                      lock;
 111         struct crypto_queue             queue;
 112
 113         struct tasklet_struct           task;
 114
 115         struct ablkcipher_request       *req;
 116         size_t                          total;
 117         struct scatterlist              *in_sg;
 118         size_t                          in_offset;
 119         struct scatterlist              *out_sg;
 120         size_t                          out_offset;
 121
 122         size_t                  buflen;
 123         void                    *buf_in;
 124         size_t                  dma_size;
 125         int                     dma_in;
 126         int                     dma_lch_in;
 127         dma_addr_t              dma_addr_in;
 128         void                    *buf_out;
 129         int                     dma_out;
 130         int                     dma_lch_out;
 131         dma_addr_t              dma_addr_out;
 132 };
 133
 134 /* keep registered devices data here */
 135 static LIST_HEAD(dev_list);
 136 static DEFINE_SPINLOCK(list_lock);
 137
 138 static inline u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset)
 139 {
 140         return __raw_readl(dd->io_base + offset);
 141 }
 142
 143 static inline void omap_aes_write(struct omap_aes_dev *dd, u32 offset,
 144                                   u32 value)
 145 {
 146         __raw_writel(value, dd->io_base + offset);
 147 }
 148
 149 static inline void omap_aes_write_mask(struct omap_aes_dev *dd, u32 offset,
 150                                         u32 value, u32 mask)
 151 {
 152         u32 val;
 153
 154         val = omap_aes_read(dd, offset);
 155         val &= ~mask;
 156         val |= value;
 157         omap_aes_write(dd, offset, val);
 158 }
 159
 160 static void omap_aes_write_n(struct omap_aes_dev *dd, u32 offset,
 161                                         u32 *value, int count)
 162 {
 163         for (; count--; value++, offset += 4)
 164                 omap_aes_write(dd, offset, *value);
 165 }
 166
 167 static int omap_aes_wait(struct omap_aes_dev *dd, u32 offset, u32 bit)
 168 {
 169         unsigned long timeout = jiffies + DEFAULT_TIMEOUT;
 170
 171         while (!(omap_aes_read(dd, offset) & bit)) {
 172                 if (time_is_before_jiffies(timeout)) {
 173                         dev_err(dd->dev, "omap-aes timeout\n");
 174                         return -ETIMEDOUT;
 175                 }
 176         }
 177         return 0;
 178 }
 179
 180 static int omap_aes_hw_init(struct omap_aes_dev *dd)
 181 {
 182         clk_enable(dd->iclk);
 183
 184         if (!(dd->flags & FLAGS_INIT)) {
 185                 /* is it necessary to reset before every operation? */
 186                 omap_aes_write_mask(dd, AES_REG_MASK, AES_REG_MASK_SOFTRESET,
 187                                         AES_REG_MASK_SOFTRESET);
 188                 /*
 189                  * prevent OCP bus error (SRESP) in case an access to the module
 190                  * is performed while the module is coming out of soft reset
 191                  */
 192                 __asm__ __volatile__("nop");
 193                 __asm__ __volatile__("nop");
 194
 195                 if (omap_aes_wait(dd, AES_REG_SYSSTATUS,
 196                                 AES_REG_SYSSTATUS_RESETDONE)) {
 197                         clk_disable(dd->iclk);
 198                         return -ETIMEDOUT;
 199                 }
 200                 dd->flags |= FLAGS_INIT;
 201         }
 202
 203         return 0;
 204 }
 205
 206 static void omap_aes_write_ctrl(struct omap_aes_dev *dd)
 207 {
 208         unsigned int key32;
 209         int i;
 210         u32 val, mask;
 211
 212         val = FLD_VAL(((dd->ctx->keylen >> 3) - 1), 4, 3);
 213         if (dd->flags & FLAGS_CBC)
 214                 val |= AES_REG_CTRL_CBC;
 215         if (dd->flags & FLAGS_ENCRYPT)
 216                 val |= AES_REG_CTRL_DIRECTION;
 217
 218         if (dd->ctrl == val && !(dd->flags & FLAGS_NEW_IV) &&
 219                    !(dd->ctx->flags & FLAGS_NEW_KEY))
 220                 goto out;
 221
 222         /* only need to write control registers for new settings */
 223
 224         dd->ctrl = val;
 225
 226         val = 0;
 227         if (dd->dma_lch_out >= 0)
 228                 val |= AES_REG_MASK_DMA_OUT_EN;
 229         if (dd->dma_lch_in >= 0)
 230                 val |= AES_REG_MASK_DMA_IN_EN;
 231
 232         mask = AES_REG_MASK_DMA_IN_EN | AES_REG_MASK_DMA_OUT_EN;
 233
 234         omap_aes_write_mask(dd, AES_REG_MASK, val, mask);
 235
 236         pr_debug("Set key\n");
 237         key32 = dd->ctx->keylen / sizeof(u32);
 238         /* set a key */
 239         for (i = 0; i < key32; i++) {
 240                 omap_aes_write(dd, AES_REG_KEY(i),
 241                         __le32_to_cpu(dd->ctx->key[i]));
 242         }
 243         dd->ctx->flags &= ~FLAGS_NEW_KEY;
 244
 245         if (dd->flags & FLAGS_NEW_IV) {
 246                 pr_debug("Set IV\n");
 247                 omap_aes_write_n(dd, AES_REG_IV(0), dd->iv, 4);
 248                 dd->flags &= ~FLAGS_NEW_IV;
 249         }
 250
 251         mask = AES_REG_CTRL_CBC | AES_REG_CTRL_DIRECTION |
 252                         AES_REG_CTRL_KEY_SIZE;
 253
 254         omap_aes_write_mask(dd, AES_REG_CTRL, dd->ctrl, mask);
 255
 256 out:
 257         /* start DMA or disable idle mode */
 258         omap_aes_write_mask(dd, AES_REG_MASK, AES_REG_MASK_START,
 259                             AES_REG_MASK_START);
 260 }
 261
 262 static struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_ctx *ctx)
 263 {
 264         struct omap_aes_dev *dd = NULL, *tmp;
 265
 266         spin_lock_bh(&list_lock);
 267         if (!ctx->dd) {
 268                 list_for_each_entry(tmp, &dev_list, list) {
 269                         /* FIXME: take fist available aes core */
 270                         dd = tmp;
 271                         break;
 272                 }
 273                 ctx->dd = dd;
 274         } else {
 275                 /* already found before */
 276                 dd = ctx->dd;
 277         }
 278         spin_unlock_bh(&list_lock);
 279
 280         return dd;
 281 }
 282
 283 static void omap_aes_dma_callback(int lch, u16 ch_status, void *data)
 284 {
 285         struct omap_aes_dev *dd = data;
 286
 287         if (lch == dd->dma_lch_out)
 288                 tasklet_schedule(&dd->task);
 289 }
 290
 291 static int omap_aes_dma_init(struct omap_aes_dev *dd)
 292 {
 293         int err = -ENOMEM;
 294
 295         dd->dma_lch_out = -1;
 296         dd->dma_lch_in = -1;
 297
 298         dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, OMAP_AES_CACHE_SIZE);
 299         dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, OMAP_AES_CACHE_SIZE);
 300         dd->buflen = PAGE_SIZE << OMAP_AES_CACHE_SIZE;
 301         dd->buflen &= ~(AES_BLOCK_SIZE - 1);
 302
 303         if (!dd->buf_in || !dd->buf_out) {
 304                 dev_err(dd->dev, "unable to alloc pages.\n");
 305                 goto err_alloc;
 306         }
 307
 308         /* MAP here */
 309         dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in, dd->buflen,
 310                                          DMA_TO_DEVICE);
 311         if (dma_mapping_error(dd->dev, dd->dma_addr_in)) {
 312                 dev_err(dd->dev, "dma %d bytes error\n", dd->buflen);
 313                 err = -EINVAL;
 314                 goto err_map_in;
 315         }
 316
 317         dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out, dd->buflen,
 318                                           DMA_FROM_DEVICE);
 319         if (dma_mapping_error(dd->dev, dd->dma_addr_out)) {
 320                 dev_err(dd->dev, "dma %d bytes error\n", dd->buflen);
 321                 err = -EINVAL;
 322                 goto err_map_out;
 323         }
 324
 325         err = omap_request_dma(dd->dma_in, "omap-aes-rx",
 326                                omap_aes_dma_callback, dd, &dd->dma_lch_in);
 327         if (err) {
 328                 dev_err(dd->dev, "Unable to request DMA channel\n");
 329                 goto err_dma_in;
 330         }
 331         err = omap_request_dma(dd->dma_out, "omap-aes-tx",
 332                                omap_aes_dma_callback, dd, &dd->dma_lch_out);
 333         if (err) {
 334                 dev_err(dd->dev, "Unable to request DMA channel\n");
 335                 goto err_dma_out;
 336         }
 337
 338         return 0;
 339
 340 err_dma_out:
 341         omap_free_dma(dd->dma_lch_in);
 342 err_dma_in:
 343         dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen,
 344                          DMA_FROM_DEVICE);
 345 err_map_out:
 346         dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE);
 347 err_map_in:
 348         free_pages((unsigned long)dd->buf_out, OMAP_AES_CACHE_SIZE);
 349         free_pages((unsigned long)dd->buf_in, OMAP_AES_CACHE_SIZE);
 350 err_alloc:
 351         if (err)
 352                 pr_err("error: %d\n", err);
 353         return err;
 354 }
 355
 356 static void omap_aes_dma_cleanup(struct omap_aes_dev *dd)
 357 {
 358         omap_free_dma(dd->dma_lch_out);
 359         omap_free_dma(dd->dma_lch_in);
 360         dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen,
 361                          DMA_FROM_DEVICE);
 362         dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE);
 363         free_pages((unsigned long)dd->buf_out, OMAP_AES_CACHE_SIZE);
 364         free_pages((unsigned long)dd->buf_in, OMAP_AES_CACHE_SIZE);
 365 }
 366
 367 static void sg_copy_buf(void *buf, struct scatterlist *sg,
 368                               unsigned int start, unsigned int nbytes, int out)
 369 {
 370         struct scatter_walk walk;
 371
 372         if (!nbytes)
 373                 return;
 374
 375         scatterwalk_start(&walk, sg);
 376         scatterwalk_advance(&walk, start);
 377         scatterwalk_copychunks(buf, &walk, nbytes, out);
 378         scatterwalk_done(&walk, out, 0);
 379 }
 380
 381 static int sg_copy(struct scatterlist **sg, size_t *offset, void *buf,
 382                    size_t buflen, size_t total, int out)
 383 {
 384         unsigned int count, off = 0;
 385
 386         while (buflen && total) {
 387                 count = min((*sg)->length - *offset, total);
 388                 count = min(count, buflen);
 389
 390                 if (!count)
 391                         return off;
 392
 393                 sg_copy_buf(buf + off, *sg, *offset, count, out);
 394
 395                 off += count;
 396                 buflen -= count;
 397                 *offset += count;
 398                 total -= count;
 399
 400                 if (*offset == (*sg)->length) {
 401                         *sg = sg_next(*sg);
 402                         if (*sg)
 403                                 *offset = 0;
 404                         else
 405                                 total = 0;
 406                 }
 407         }
 408
 409         return off;
 410 }
 411
 412 static int omap_aes_crypt_dma(struct crypto_tfm *tfm, dma_addr_t dma_addr_in,
 413                                dma_addr_t dma_addr_out, int length)
 414 {
 415         struct omap_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 416         struct omap_aes_dev *dd = ctx->dd;
 417         int len32;
 418
 419         pr_debug("len: %d\n", length);
 420
 421         dd->dma_size = length;
 422
 423         if (!(dd->flags & FLAGS_FAST))
 424                 dma_sync_single_for_device(dd->dev, dma_addr_in, length,
 425                                            DMA_TO_DEVICE);
 426
 427         len32 = DIV_ROUND_UP(length, sizeof(u32));
 428
 429         /* IN */
 430         omap_set_dma_dest_params(dd->dma_lch_in, 0, OMAP_DMA_AMODE_CONSTANT,
 431                                  dd->phys_base + AES_REG_DATA, 0, 4);
 432
 433         omap_set_dma_dest_burst_mode(dd->dma_lch_in, OMAP_DMA_DATA_BURST_4);
 434         omap_set_dma_src_burst_mode(dd->dma_lch_in, OMAP_DMA_DATA_BURST_4);
 435
 436         omap_set_dma_transfer_params(dd->dma_lch_in, OMAP_DMA_DATA_TYPE_S32,
 437                                      len32, 1, OMAP_DMA_SYNC_PACKET, dd->dma_in,
 438                                         OMAP_DMA_DST_SYNC);
 439
 440         omap_set_dma_src_params(dd->dma_lch_in, 0, OMAP_DMA_AMODE_POST_INC,
 441                                 dma_addr_in, 0, 0);
 442
 443         /* OUT */
 444         omap_set_dma_src_params(dd->dma_lch_out, 0, OMAP_DMA_AMODE_CONSTANT,
 445                                 dd->phys_base + AES_REG_DATA, 0, 4);
 446
 447         omap_set_dma_src_burst_mode(dd->dma_lch_out, OMAP_DMA_DATA_BURST_4);
 448         omap_set_dma_dest_burst_mode(dd->dma_lch_out, OMAP_DMA_DATA_BURST_4);
 449
 450         omap_set_dma_transfer_params(dd->dma_lch_out, OMAP_DMA_DATA_TYPE_S32,
 451                                      len32, 1, OMAP_DMA_SYNC_PACKET,
 452                                         dd->dma_out, OMAP_DMA_SRC_SYNC);
 453
 454         omap_set_dma_dest_params(dd->dma_lch_out, 0, OMAP_DMA_AMODE_POST_INC,
 455                                  dma_addr_out, 0, 0);
 456
 457         omap_start_dma(dd->dma_lch_in);
 458         omap_start_dma(dd->dma_lch_out);
 459
 460         omap_aes_write_ctrl(dd);
 461
 462         return 0;
 463 }
 464
 465 static int omap_aes_crypt_dma_start(struct omap_aes_dev *dd)
 466 {
 467         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(
 468                                         crypto_ablkcipher_reqtfm(dd->req));
 469         int err, fast = 0, in, out;
 470         size_t count;
 471         dma_addr_t addr_in, addr_out;
 472
 473         pr_debug("total: %d\n", dd->total);
 474
 475         if (sg_is_last(dd->in_sg) && sg_is_last(dd->out_sg)) {
 476                 /* check for alignment */
 477                 in = IS_ALIGNED((u32)dd->in_sg->offset, sizeof(u32));
 478                 out = IS_ALIGNED((u32)dd->out_sg->offset, sizeof(u32));
 479
 480                 fast = in && out;
 481         }
 482
 483         if (fast)  {
 484                 count = min(dd->total, sg_dma_len(dd->in_sg));
 485                 count = min(count, sg_dma_len(dd->out_sg));
 486
 487                 if (count != dd->total)
 488                         return -EINVAL;
 489
 490                 pr_debug("fast\n");
 491
 492                 err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 493                 if (!err) {
 494                         dev_err(dd->dev, "dma_map_sg() error\n");
 495                         return -EINVAL;
 496                 }
 497
 498                 err = dma_map_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
 499                 if (!err) {
 500                         dev_err(dd->dev, "dma_map_sg() error\n");
 501                         dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 502                         return -EINVAL;
 503                 }
 504
 505                 addr_in = sg_dma_address(dd->in_sg);
 506                 addr_out = sg_dma_address(dd->out_sg);
 507
 508                 dd->flags |= FLAGS_FAST;
 509
 510         } else {
 511                 /* use cache buffers */
 512                 count = sg_copy(&dd->in_sg, &dd->in_offset, dd->buf_in,
 513                                  dd->buflen, dd->total, 0);
 514
 515                 addr_in = dd->dma_addr_in;
 516                 addr_out = dd->dma_addr_out;
 517
 518                 dd->flags &= ~FLAGS_FAST;
 519
 520         }
 521
 522         dd->total -= count;
 523
 524         err = omap_aes_hw_init(dd);
 525
 526         err = omap_aes_crypt_dma(tfm, addr_in, addr_out, count);
 527
 528         return err;
 529 }
 530
 531 static void omap_aes_finish_req(struct omap_aes_dev *dd, int err)
 532 {
 533         struct omap_aes_ctx *ctx;
 534
 535         pr_debug("err: %d\n", err);
 536
 537         dd->flags &= ~FLAGS_BUSY;
 538
 539         ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(dd->req));
 540
 541         if (!dd->total)
 542                 dd->req->base.complete(&dd->req->base, err);
 543 }
 544
 545 static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd)
 546 {
 547         int err = 0;
 548         size_t count;
 549
 550         pr_debug("total: %d\n", dd->total);
 551
 552         omap_aes_write_mask(dd, AES_REG_MASK, 0, AES_REG_MASK_START);
 553
 554         clk_disable(dd->iclk);
 555
 556         omap_stop_dma(dd->dma_lch_in);
 557         omap_stop_dma(dd->dma_lch_out);
 558
 559         if (dd->flags & FLAGS_FAST) {
 560                 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
 561                 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 562         } else {
 563                 dma_sync_single_for_device(dd->dev, dd->dma_addr_out,
 564                                            dd->dma_size, DMA_FROM_DEVICE);
 565
 566                 /* copy data */
 567                 count = sg_copy(&dd->out_sg, &dd->out_offset, dd->buf_out,
 568                                  dd->buflen, dd->dma_size, 1);
 569                 if (count != dd->dma_size) {
 570                         err = -EINVAL;
 571                         pr_err("not all data converted: %u\n", count);
 572                 }
 573         }
 574
 575         if (err || !dd->total)
 576                 omap_aes_finish_req(dd, err);
 577
 578         return err;
 579 }
 580
 581 static int omap_aes_handle_req(struct omap_aes_dev *dd,
 582                                struct ablkcipher_request *req)
 583 {
 584         struct crypto_async_request *async_req, *backlog;
 585         struct omap_aes_ctx *ctx;
 586         struct omap_aes_reqctx *rctx;
 587         unsigned long flags;
 588         int err = 0;
 589
 590         spin_lock_irqsave(&dd->lock, flags);
 591         if (req)
 592                 err = ablkcipher_enqueue_request(&dd->queue, req);
 593         if (dd->flags & FLAGS_BUSY) {
 594                 spin_unlock_irqrestore(&dd->lock, flags);
 595                 return err;
 596         }
 597         backlog = crypto_get_backlog(&dd->queue);
 598         async_req = crypto_dequeue_request(&dd->queue);
 599         if (async_req)
 600                 dd->flags |= FLAGS_BUSY;
 601         spin_unlock_irqrestore(&dd->lock, flags);
 602
 603         if (!async_req)
 604                 return 0;
 605
 606         if (backlog)
 607                 backlog->complete(backlog, -EINPROGRESS);
 608
 609         req = ablkcipher_request_cast(async_req);
 610
 611         pr_debug("get new req\n");
 612
 613         /* assign new request to device */
 614         dd->req = req;
 615         dd->total = req->nbytes;
 616         dd->in_offset = 0;
 617         dd->in_sg = req->src;
 618         dd->out_offset = 0;
 619         dd->out_sg = req->dst;
 620
 621         rctx = ablkcipher_request_ctx(req);
 622         ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
 623         rctx->mode &= FLAGS_MODE_MASK;
 624         dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
 625
 626         dd->iv = req->info;
 627         if ((dd->flags & FLAGS_CBC) && dd->iv)
 628                 dd->flags |= FLAGS_NEW_IV;
 629         else
 630                 dd->flags &= ~FLAGS_NEW_IV;
 631
 632         ctx->dd = dd;
 633         if (dd->ctx != ctx) {
 634                 /* assign new context to device */
 635                 dd->ctx = ctx;
 636                 ctx->flags |= FLAGS_NEW_KEY;
 637         }
 638
 639         if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE))
 640                 pr_err("request size is not exact amount of AES blocks\n");
 641
 642         omap_aes_crypt_dma_start(dd);
 643
 644         return err;
 645 }
 646
 647 static void omap_aes_task(unsigned long data)
 648 {
 649         struct omap_aes_dev *dd = (struct omap_aes_dev *)data;
 650
 651         pr_debug("enter\n");
 652
 653         omap_aes_crypt_dma_stop(dd);
 654
 655         if (dd->total)
 656                 omap_aes_crypt_dma_start(dd);
 657         else
 658                 omap_aes_handle_req(dd, NULL);
 659
 660         pr_debug("exit\n");
 661 }
 662
 663 static int omap_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
 664 {
 665         struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx(
 666                         crypto_ablkcipher_reqtfm(req));
 667         struct omap_aes_reqctx *rctx = ablkcipher_request_ctx(req);
 668         struct omap_aes_dev *dd;
 669
 670         pr_debug("nbytes: %d, enc: %d, cbc: %d\n", req->nbytes,
 671                   !!(mode & FLAGS_ENCRYPT),
 672                   !!(mode & FLAGS_CBC));
 673
 674         dd = omap_aes_find_dev(ctx);
 675         if (!dd)
 676                 return -ENODEV;
 677
 678         rctx->mode = mode;
 679
 680         return omap_aes_handle_req(dd, req);
 681 }
 682
 683 /* ********************** ALG API ************************************ */
 684
 685 static int omap_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 686                            unsigned int keylen)
 687 {
 688         struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 689
 690         if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
 691                    keylen != AES_KEYSIZE_256)
 692                 return -EINVAL;
 693
 694         pr_debug("enter, keylen: %d\n", keylen);
 695
 696         memcpy(ctx->key, key, keylen);
 697         ctx->keylen = keylen;
 698         ctx->flags |= FLAGS_NEW_KEY;
 699
 700         return 0;
 701 }
 702
 703 static int omap_aes_ecb_encrypt(struct ablkcipher_request *req)
 704 {
 705         return omap_aes_crypt(req, FLAGS_ENCRYPT);
 706 }
 707
 708 static int omap_aes_ecb_decrypt(struct ablkcipher_request *req)
 709 {
 710         return omap_aes_crypt(req, 0);
 711 }
 712
 713 static int omap_aes_cbc_encrypt(struct ablkcipher_request *req)
 714 {
 715         return omap_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC);
 716 }
 717
 718 static int omap_aes_cbc_decrypt(struct ablkcipher_request *req)
 719 {
 720         return omap_aes_crypt(req, FLAGS_CBC);
 721 }
 722
 723 static int omap_aes_cra_init(struct crypto_tfm *tfm)
 724 {
 725         pr_debug("enter\n");
 726
 727         tfm->crt_ablkcipher.reqsize = sizeof(struct omap_aes_reqctx);
 728
 729         return 0;
 730 }
 731
 732 static void omap_aes_cra_exit(struct crypto_tfm *tfm)
 733 {
 734         pr_debug("enter\n");
 735 }
 736
 737 /* ********************** ALGS ************************************ */
 738
 739 static struct crypto_alg algs[] = {
 740 {
 741         .cra_name               = "ecb(aes)",
 742         .cra_driver_name        = "ecb-aes-omap",
 743         .cra_priority           = 100,
 744         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 745         .cra_blocksize          = AES_BLOCK_SIZE,
 746         .cra_ctxsize            = sizeof(struct omap_aes_ctx),
 747         .cra_alignmask          = 0,
 748         .cra_type               = &crypto_ablkcipher_type,
 749         .cra_module             = THIS_MODULE,
 750         .cra_init               = omap_aes_cra_init,
 751         .cra_exit               = omap_aes_cra_exit,
 752         .cra_u.ablkcipher = {
 753                 .min_keysize    = AES_MIN_KEY_SIZE,
 754                 .max_keysize    = AES_MAX_KEY_SIZE,
 755                 .setkey         = omap_aes_setkey,
 756                 .encrypt        = omap_aes_ecb_encrypt,
 757                 .decrypt        = omap_aes_ecb_decrypt,
 758         }
 759 },
 760 {
 761         .cra_name               = "cbc(aes)",
 762         .cra_driver_name        = "cbc-aes-omap",
 763         .cra_priority           = 100,
 764         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 765         .cra_blocksize          = AES_BLOCK_SIZE,
 766         .cra_ctxsize            = sizeof(struct omap_aes_ctx),
 767         .cra_alignmask          = 0,
 768         .cra_type               = &crypto_ablkcipher_type,
 769         .cra_module             = THIS_MODULE,
 770         .cra_init               = omap_aes_cra_init,
 771         .cra_exit               = omap_aes_cra_exit,
 772         .cra_u.ablkcipher = {
 773                 .min_keysize    = AES_MIN_KEY_SIZE,
 774                 .max_keysize    = AES_MAX_KEY_SIZE,
 775                 .ivsize         = AES_BLOCK_SIZE,
 776                 .setkey         = omap_aes_setkey,
 777                 .encrypt        = omap_aes_cbc_encrypt,
 778                 .decrypt        = omap_aes_cbc_decrypt,
 779         }
 780 }
 781 };
 782
 783 static int omap_aes_probe(struct platform_device *pdev)
 784 {
 785         struct device *dev = &pdev->dev;
 786         struct omap_aes_dev *dd;
 787         struct resource *res;
 788         int err = -ENOMEM, i, j;
 789         u32 reg;
 790
 791         dd = kzalloc(sizeof(struct omap_aes_dev), GFP_KERNEL);
 792         if (dd == NULL) {
 793                 dev_err(dev, "unable to alloc data struct.\n");
 794                 goto err_data;
 795         }
 796         dd->dev = dev;
 797         platform_set_drvdata(pdev, dd);
 798
 799         spin_lock_init(&dd->lock);
 800         crypto_init_queue(&dd->queue, OMAP_AES_QUEUE_LENGTH);
 801
 802         /* Get the base address */
 803         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 804         if (!res) {
 805                 dev_err(dev, "invalid resource type\n");
 806                 err = -ENODEV;
 807                 goto err_res;
 808         }
 809         dd->phys_base = res->start;
 810
 811         /* Get the DMA */
 812         res = platform_get_resource(pdev, IORESOURCE_DMA, 0);
 813         if (!res)
 814                 dev_info(dev, "no DMA info\n");
 815         else
 816                 dd->dma_out = res->start;
 817
 818         /* Get the DMA */
 819         res = platform_get_resource(pdev, IORESOURCE_DMA, 1);
 820         if (!res)
 821                 dev_info(dev, "no DMA info\n");
 822         else
 823                 dd->dma_in = res->start;
 824
 825         /* Initializing the clock */
 826         dd->iclk = clk_get(dev, "ick");
 827         if (!dd->iclk) {
 828                 dev_err(dev, "clock intialization failed.\n");
 829                 err = -ENODEV;
 830                 goto err_res;
 831         }
 832
 833         dd->io_base = ioremap(dd->phys_base, SZ_4K);
 834         if (!dd->io_base) {
 835                 dev_err(dev, "can't ioremap\n");
 836                 err = -ENOMEM;
 837                 goto err_io;
 838         }
 839
 840         clk_enable(dd->iclk);
 841         reg = omap_aes_read(dd, AES_REG_REV);
 842         dev_info(dev, "OMAP AES hw accel rev: %u.%u\n",
 843                  (reg & AES_REG_REV_MAJOR) >> 4, reg & AES_REG_REV_MINOR);
 844         clk_disable(dd->iclk);
 845
 846         tasklet_init(&dd->task, omap_aes_task, (unsigned long)dd);
 847
 848         err = omap_aes_dma_init(dd);
 849         if (err)
 850                 goto err_dma;
 851
 852         INIT_LIST_HEAD(&dd->list);
 853         spin_lock(&list_lock);
 854         list_add_tail(&dd->list, &dev_list);
 855         spin_unlock(&list_lock);
 856
 857         for (i = 0; i < ARRAY_SIZE(algs); i++) {
 858                 pr_debug("i: %d\n", i);
 859                 INIT_LIST_HEAD(&algs[i].cra_list);
 860                 err = crypto_register_alg(&algs[i]);
 861                 if (err)
 862                         goto err_algs;
 863         }
 864
 865         pr_info("probe() done\n");
 866
 867         return 0;
 868 err_algs:
 869         for (j = 0; j < i; j++)
 870                 crypto_unregister_alg(&algs[j]);
 871         omap_aes_dma_cleanup(dd);
 872 err_dma:
 873         tasklet_kill(&dd->task);
 874         iounmap(dd->io_base);
 875 err_io:
 876         clk_put(dd->iclk);
 877 err_res:
 878         kfree(dd);
 879         dd = NULL;
 880 err_data:
 881         dev_err(dev, "initialization failed.\n");
 882         return err;
 883 }
 884
 885 static int omap_aes_remove(struct platform_device *pdev)
 886 {
 887         struct omap_aes_dev *dd = platform_get_drvdata(pdev);
 888         int i;
 889
 890         if (!dd)
 891                 return -ENODEV;
 892
 893         spin_lock(&list_lock);
 894         list_del(&dd->list);
 895         spin_unlock(&list_lock);
 896
 897         for (i = 0; i < ARRAY_SIZE(algs); i++)
 898                 crypto_unregister_alg(&algs[i]);
 899
 900         tasklet_kill(&dd->task);
 901         omap_aes_dma_cleanup(dd);
 902         iounmap(dd->io_base);
 903         clk_put(dd->iclk);
 904         kfree(dd);
 905         dd = NULL;
 906
 907         return 0;
 908 }
 909
 910 static struct platform_driver omap_aes_driver = {
 911         .probe  = omap_aes_probe,
 912         .remove = omap_aes_remove,
 913         .driver = {
 914                 .name   = "omap-aes",
 915                 .owner  = THIS_MODULE,
 916         },
 917 };
 918
 919 static int __init omap_aes_mod_init(void)
 920 {
 921         pr_info("loading %s driver\n", "omap-aes");
 922
 923         if (!cpu_class_is_omap2() || omap_type() != OMAP2_DEVICE_TYPE_SEC) {
 924                 pr_err("Unsupported cpu\n");
 925                 return -ENODEV;
 926         }
 927
 928         return  platform_driver_register(&omap_aes_driver);
 929 }
 930
 931 static void __exit omap_aes_mod_exit(void)
 932 {
 933         platform_driver_unregister(&omap_aes_driver);
 934 }
 935
 936 module_init(omap_aes_mod_init);
 937 module_exit(omap_aes_mod_exit);
 938
 939 MODULE_DESCRIPTION("OMAP AES hw acceleration support.");
 940 MODULE_LICENSE("GPL v2");
 941 MODULE_AUTHOR("Dmitry Kasatkin");
 942