drivers/crypto/caam/caamalg.c

   1 /*
   2  * caam - Freescale FSL CAAM support for crypto API
   3  *
   4  * Copyright 2008-2011 Freescale Semiconductor, Inc.
   5  *
   6  * Based on talitos crypto API driver.
   7  *
   8  * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008):
   9  *
  10  * ---------------                     ---------------
  11  * | JobDesc #1  |-------------------->|  ShareDesc  |
  12  * | *(packet 1) |                     |   (PDB)     |
  13  * ---------------      |------------->|  (hashKey)  |
  14  *       .              |              | (cipherKey) |
  15  *       .              |    |-------->| (operation) |
  16  * ---------------      |    |         ---------------
  17  * | JobDesc #2  |------|    |
  18  * | *(packet 2) |           |
  19  * ---------------           |
  20  *       .                   |
  21  *       .                   |
  22  * ---------------           |
  23  * | JobDesc #3  |------------
  24  * | *(packet 3) |
  25  * ---------------
  26  *
  27  * The SharedDesc never changes for a connection unless rekeyed, but
  28  * each packet will likely be in a different place. So all we need
  29  * to know to process the packet is where the input is, where the
  30  * output goes, and what context we want to process with. Context is
  31  * in the SharedDesc, packet references in the JobDesc.
  32  *
  33  * So, a job desc looks like:
  34  *
  35  * ---------------------
  36  * | Header            |
  37  * | ShareDesc Pointer |
  38  * | SEQ_OUT_PTR       |
  39  * | (output buffer)   |
  40  * | SEQ_IN_PTR        |
  41  * | (input buffer)    |
  42  * | LOAD (to DECO)    |
  43  * ---------------------
  44  */
  45
  46 #include "compat.h"
  47
  48 #include "regs.h"
  49 #include "intern.h"
  50 #include "desc_constr.h"
  51 #include "jr.h"
  52 #include "error.h"
  53
  54 /*
  55  * crypto alg
  56  */
  57 #define CAAM_CRA_PRIORITY               3000
  58 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
  59 #define CAAM_MAX_KEY_SIZE               (AES_MAX_KEY_SIZE + \
  60                                          SHA512_DIGEST_SIZE * 2)
  61 /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
  62 #define CAAM_MAX_IV_LENGTH              16
  63
  64 #ifdef DEBUG
  65 /* for print_hex_dumps with line references */
  66 #define xstr(s) str(s)
  67 #define str(s) #s
  68 #define debug(format, arg...) printk(format, arg)
  69 #else
  70 #define debug(format, arg...)
  71 #endif
  72
  73 /*
  74  * per-session context
  75  */
  76 struct caam_ctx {
  77         struct device *jrdev;
  78         u32 *sh_desc;
  79         dma_addr_t shared_desc_phys;
  80         u32 class1_alg_type;
  81         u32 class2_alg_type;
  82         u32 alg_op;
  83         u8 *key;
  84         dma_addr_t key_phys;
  85         unsigned int keylen;
  86         unsigned int enckeylen;
  87         unsigned int authkeylen;
  88         unsigned int split_key_len;
  89         unsigned int split_key_pad_len;
  90         unsigned int authsize;
  91 };
  92
  93 static int aead_authenc_setauthsize(struct crypto_aead *authenc,
  94                                     unsigned int authsize)
  95 {
  96         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
  97
  98         ctx->authsize = authsize;
  99
 100         return 0;
 101 }
 102
 103 struct split_key_result {
 104         struct completion completion;
 105         int err;
 106 };
 107
 108 static void split_key_done(struct device *dev, u32 *desc, u32 err,
 109                            void *context)
 110 {
 111         struct split_key_result *res = context;
 112
 113 #ifdef DEBUG
 114         dev_err(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 115 #endif
 116         if (err) {
 117                 char tmp[256];
 118
 119                 dev_err(dev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
 120         }
 121
 122         res->err = err;
 123
 124         complete(&res->completion);
 125 }
 126
 127 /*
 128 get a split ipad/opad key
 129
 130 Split key generation-----------------------------------------------
 131
 132 [00] 0xb0810008    jobdesc: stidx=1 share=never len=8
 133 [01] 0x04000014        key: class2->keyreg len=20
 134                         @0xffe01000
 135 [03] 0x84410014  operation: cls2-op sha1 hmac init dec
 136 [04] 0x24940000     fifold: class2 msgdata-last2 len=0 imm
 137 [05] 0xa4000001       jump: class2 local all ->1 [06]
 138 [06] 0x64260028    fifostr: class2 mdsplit-jdk len=40
 139                         @0xffe04000
 140 */
 141 static u32 gen_split_key(struct caam_ctx *ctx, const u8 *key_in, u32 authkeylen)
 142 {
 143         struct device *jrdev = ctx->jrdev;
 144         u32 *desc;
 145         struct split_key_result result;
 146         dma_addr_t dma_addr_in, dma_addr_out;
 147         int ret = 0;
 148
 149         desc = kmalloc(CAAM_CMD_SZ * 6 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
 150
 151         init_job_desc(desc, 0);
 152
 153         dma_addr_in = dma_map_single(jrdev, (void *)key_in, authkeylen,
 154                                      DMA_TO_DEVICE);
 155         if (dma_mapping_error(jrdev, dma_addr_in)) {
 156                 dev_err(jrdev, "unable to map key input memory\n");
 157                 kfree(desc);
 158                 return -ENOMEM;
 159         }
 160         append_key(desc, dma_addr_in, authkeylen, CLASS_2 |
 161                        KEY_DEST_CLASS_REG);
 162
 163         /* Sets MDHA up into an HMAC-INIT */
 164         append_operation(desc, ctx->alg_op | OP_ALG_DECRYPT |
 165                              OP_ALG_AS_INIT);
 166
 167         /*
 168          * do a FIFO_LOAD of zero, this will trigger the internal key expansion
 169            into both pads inside MDHA
 170          */
 171         append_fifo_load_as_imm(desc, NULL, 0, LDST_CLASS_2_CCB |
 172                                 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST2);
 173
 174         /*
 175          * FIFO_STORE with the explicit split-key content store
 176          * (0x26 output type)
 177          */
 178         dma_addr_out = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len,
 179                                       DMA_FROM_DEVICE);
 180         if (dma_mapping_error(jrdev, dma_addr_out)) {
 181                 dev_err(jrdev, "unable to map key output memory\n");
 182                 kfree(desc);
 183                 return -ENOMEM;
 184         }
 185         append_fifo_store(desc, dma_addr_out, ctx->split_key_len,
 186                           LDST_CLASS_2_CCB | FIFOST_TYPE_SPLIT_KEK);
 187
 188 #ifdef DEBUG
 189         print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ",
 190                        DUMP_PREFIX_ADDRESS, 16, 4, key_in, authkeylen, 1);
 191         print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
 192                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 193 #endif
 194
 195         result.err = 0;
 196         init_completion(&result.completion);
 197
 198         ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result);
 199         if (!ret) {
 200                 /* in progress */
 201                 wait_for_completion_interruptible(&result.completion);
 202                 ret = result.err;
 203 #ifdef DEBUG
 204                 print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ",
 205                                DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 206                                ctx->split_key_pad_len, 1);
 207 #endif
 208         }
 209
 210         dma_unmap_single(jrdev, dma_addr_out, ctx->split_key_pad_len,
 211                          DMA_FROM_DEVICE);
 212         dma_unmap_single(jrdev, dma_addr_in, authkeylen, DMA_TO_DEVICE);
 213
 214         kfree(desc);
 215
 216         return ret;
 217 }
 218
 219 static int build_sh_desc_ipsec(struct caam_ctx *ctx)
 220 {
 221         struct device *jrdev = ctx->jrdev;
 222         u32 *sh_desc;
 223         u32 *jump_cmd;
 224
 225         /* build shared descriptor for this session */
 226         sh_desc = kmalloc(CAAM_CMD_SZ * 4 + ctx->split_key_pad_len +
 227                           ctx->enckeylen, GFP_DMA | GFP_KERNEL);
 228         if (!sh_desc) {
 229                 dev_err(jrdev, "could not allocate shared descriptor\n");
 230                 return -ENOMEM;
 231         }
 232
 233         init_sh_desc(sh_desc, HDR_SAVECTX | HDR_SHARE_SERIAL);
 234
 235         jump_cmd = append_jump(sh_desc, CLASS_BOTH | JUMP_TEST_ALL |
 236                                JUMP_COND_SHRD | JUMP_COND_SELF);
 237
 238         /* process keys, starting with class 2/authentication */
 239         append_key_as_imm(sh_desc, ctx->key, ctx->split_key_pad_len,
 240                           ctx->split_key_len,
 241                           CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
 242
 243         append_key_as_imm(sh_desc, (void *)ctx->key + ctx->split_key_pad_len,
 244                           ctx->enckeylen, ctx->enckeylen,
 245                           CLASS_1 | KEY_DEST_CLASS_REG);
 246
 247         /* update jump cmd now that we are at the jump target */
 248         set_jump_tgt_here(sh_desc, jump_cmd);
 249
 250         ctx->shared_desc_phys = dma_map_single(jrdev, sh_desc,
 251                                                desc_bytes(sh_desc),
 252                                                DMA_TO_DEVICE);
 253         if (dma_mapping_error(jrdev, ctx->shared_desc_phys)) {
 254                 dev_err(jrdev, "unable to map shared descriptor\n");
 255                 kfree(sh_desc);
 256                 return -ENOMEM;
 257         }
 258
 259         ctx->sh_desc = sh_desc;
 260
 261         return 0;
 262 }
 263
 264 static int aead_authenc_setkey(struct crypto_aead *aead,
 265                                const u8 *key, unsigned int keylen)
 266 {
 267         /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
 268         static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
 269         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 270         struct device *jrdev = ctx->jrdev;
 271         struct rtattr *rta = (void *)key;
 272         struct crypto_authenc_key_param *param;
 273         unsigned int authkeylen;
 274         unsigned int enckeylen;
 275         int ret = 0;
 276
 277         param = RTA_DATA(rta);
 278         enckeylen = be32_to_cpu(param->enckeylen);
 279
 280         key += RTA_ALIGN(rta->rta_len);
 281         keylen -= RTA_ALIGN(rta->rta_len);
 282
 283         if (keylen < enckeylen)
 284                 goto badkey;
 285
 286         authkeylen = keylen - enckeylen;
 287
 288         if (keylen > CAAM_MAX_KEY_SIZE)
 289                 goto badkey;
 290
 291         /* Pick class 2 key length from algorithm submask */
 292         ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >>
 293                                       OP_ALG_ALGSEL_SHIFT] * 2;
 294         ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16);
 295
 296 #ifdef DEBUG
 297         printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n",
 298                keylen, enckeylen, authkeylen);
 299         printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
 300                ctx->split_key_len, ctx->split_key_pad_len);
 301         print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ",
 302                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 303 #endif
 304         ctx->key = kmalloc(ctx->split_key_pad_len + enckeylen,
 305                            GFP_KERNEL | GFP_DMA);
 306         if (!ctx->key) {
 307                 dev_err(jrdev, "could not allocate key output memory\n");
 308                 return -ENOMEM;
 309         }
 310
 311         ret = gen_split_key(ctx, key, authkeylen);
 312         if (ret) {
 313                 kfree(ctx->key);
 314                 goto badkey;
 315         }
 316
 317         /* postpend encryption key to auth split key */
 318         memcpy(ctx->key + ctx->split_key_pad_len, key + authkeylen, enckeylen);
 319
 320         ctx->key_phys = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len +
 321                                        enckeylen, DMA_TO_DEVICE);
 322         if (dma_mapping_error(jrdev, ctx->key_phys)) {
 323                 dev_err(jrdev, "unable to map key i/o memory\n");
 324                 kfree(ctx->key);
 325                 return -ENOMEM;
 326         }
 327 #ifdef DEBUG
 328         print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ",
 329                        DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 330                        ctx->split_key_pad_len + enckeylen, 1);
 331 #endif
 332
 333         ctx->keylen = keylen;
 334         ctx->enckeylen = enckeylen;
 335         ctx->authkeylen = authkeylen;
 336
 337         ret = build_sh_desc_ipsec(ctx);
 338         if (ret) {
 339                 dma_unmap_single(jrdev, ctx->key_phys, ctx->split_key_pad_len +
 340                                  enckeylen, DMA_TO_DEVICE);
 341                 kfree(ctx->key);
 342         }
 343
 344         return ret;
 345 badkey:
 346         crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
 347         return -EINVAL;
 348 }
 349
 350 struct link_tbl_entry {
 351         u64 ptr;
 352         u32 len;
 353         u8 reserved;
 354         u8 buf_pool_id;
 355         u16 offset;
 356 };
 357
 358 /*
 359  * ipsec_esp_edesc - s/w-extended ipsec_esp descriptor
 360  * @src_nents: number of segments in input scatterlist
 361  * @dst_nents: number of segments in output scatterlist
 362  * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist
 363  * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
 364  * @link_tbl_bytes: length of dma mapped link_tbl space
 365  * @link_tbl_dma: bus physical mapped address of h/w link table
 366  * @hw_desc: the h/w job descriptor followed by any referenced link tables
 367  */
 368 struct ipsec_esp_edesc {
 369         int assoc_nents;
 370         int src_nents;
 371         int dst_nents;
 372         int link_tbl_bytes;
 373         dma_addr_t link_tbl_dma;
 374         struct link_tbl_entry *link_tbl;
 375         u32 hw_desc[0];
 376 };
 377
 378 static void ipsec_esp_unmap(struct device *dev,
 379                             struct ipsec_esp_edesc *edesc,
 380                             struct aead_request *areq)
 381 {
 382         dma_unmap_sg(dev, areq->assoc, edesc->assoc_nents, DMA_TO_DEVICE);
 383
 384         if (unlikely(areq->dst != areq->src)) {
 385                 dma_unmap_sg(dev, areq->src, edesc->src_nents,
 386                              DMA_TO_DEVICE);
 387                 dma_unmap_sg(dev, areq->dst, edesc->dst_nents,
 388                              DMA_FROM_DEVICE);
 389         } else {
 390                 dma_unmap_sg(dev, areq->src, edesc->src_nents,
 391                              DMA_BIDIRECTIONAL);
 392         }
 393
 394         if (edesc->link_tbl_bytes)
 395                 dma_unmap_single(dev, edesc->link_tbl_dma,
 396                                  edesc->link_tbl_bytes,
 397                                  DMA_TO_DEVICE);
 398 }
 399
 400 /*
 401  * ipsec_esp descriptor callbacks
 402  */
 403 static void ipsec_esp_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
 404                                    void *context)
 405 {
 406         struct aead_request *areq = context;
 407         struct ipsec_esp_edesc *edesc;
 408 #ifdef DEBUG
 409         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
 410         int ivsize = crypto_aead_ivsize(aead);
 411         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 412
 413         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 414 #endif
 415         edesc = (struct ipsec_esp_edesc *)((char *)desc -
 416                  offsetof(struct ipsec_esp_edesc, hw_desc));
 417
 418         if (err) {
 419                 char tmp[256];
 420
 421                 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
 422         }
 423
 424         ipsec_esp_unmap(jrdev, edesc, areq);
 425
 426 #ifdef DEBUG
 427         print_hex_dump(KERN_ERR, "assoc  @"xstr(__LINE__)": ",
 428                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->assoc),
 429                        areq->assoclen , 1);
 430         print_hex_dump(KERN_ERR, "dstiv  @"xstr(__LINE__)": ",
 431                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src) - ivsize,
 432                        edesc->src_nents ? 100 : ivsize, 1);
 433         print_hex_dump(KERN_ERR, "dst    @"xstr(__LINE__)": ",
 434                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src),
 435                        edesc->src_nents ? 100 : areq->cryptlen +
 436                        ctx->authsize + 4, 1);
 437 #endif
 438
 439         kfree(edesc);
 440
 441         aead_request_complete(areq, err);
 442 }
 443
 444 static void ipsec_esp_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
 445                                    void *context)
 446 {
 447         struct aead_request *areq = context;
 448         struct ipsec_esp_edesc *edesc;
 449 #ifdef DEBUG
 450         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
 451         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 452
 453         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 454 #endif
 455         edesc = (struct ipsec_esp_edesc *)((char *)desc -
 456                  offsetof(struct ipsec_esp_edesc, hw_desc));
 457
 458         if (err) {
 459                 char tmp[256];
 460
 461                 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
 462         }
 463
 464         ipsec_esp_unmap(jrdev, edesc, areq);
 465
 466         /*
 467          * verify hw auth check passed else return -EBADMSG
 468          */
 469         if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK)
 470                 err = -EBADMSG;
 471
 472 #ifdef DEBUG
 473         print_hex_dump(KERN_ERR, "iphdrout@"xstr(__LINE__)": ",
 474                        DUMP_PREFIX_ADDRESS, 16, 4,
 475                        ((char *)sg_virt(areq->assoc) - sizeof(struct iphdr)),
 476                        sizeof(struct iphdr) + areq->assoclen +
 477                        ((areq->cryptlen > 1500) ? 1500 : areq->cryptlen) +
 478                        ctx->authsize + 36, 1);
 479         if (!err && edesc->link_tbl_bytes) {
 480                 struct scatterlist *sg = sg_last(areq->src, edesc->src_nents);
 481                 print_hex_dump(KERN_ERR, "sglastout@"xstr(__LINE__)": ",
 482                                DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(sg),
 483                         sg->length + ctx->authsize + 16, 1);
 484         }
 485 #endif
 486         kfree(edesc);
 487
 488         aead_request_complete(areq, err);
 489 }
 490
 491 /*
 492  * convert scatterlist to h/w link table format
 493  * scatterlist must have been previously dma mapped
 494  */
 495 static void sg_to_link_tbl(struct scatterlist *sg, int sg_count,
 496                            struct link_tbl_entry *link_tbl_ptr, u32 offset)
 497 {
 498         while (sg_count) {
 499                 link_tbl_ptr->ptr = sg_dma_address(sg);
 500                 link_tbl_ptr->len = sg_dma_len(sg);
 501                 link_tbl_ptr->reserved = 0;
 502                 link_tbl_ptr->buf_pool_id = 0;
 503                 link_tbl_ptr->offset = offset;
 504                 link_tbl_ptr++;
 505                 sg = sg_next(sg);
 506                 sg_count--;
 507         }
 508
 509         /* set Final bit (marks end of link table) */
 510         link_tbl_ptr--;
 511         link_tbl_ptr->len |= 0x40000000;
 512 }
 513
 514 /*
 515  * fill in and submit ipsec_esp job descriptor
 516  */
 517 static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq,
 518                      u32 encrypt,
 519                      void (*callback) (struct device *dev, u32 *desc,
 520                                        u32 err, void *context))
 521 {
 522         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
 523         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 524         struct device *jrdev = ctx->jrdev;
 525         u32 *desc = edesc->hw_desc, options;
 526         int ret, sg_count, assoc_sg_count;
 527         int ivsize = crypto_aead_ivsize(aead);
 528         int authsize = ctx->authsize;
 529         dma_addr_t ptr, dst_dma, src_dma;
 530 #ifdef DEBUG
 531         u32 *sh_desc = ctx->sh_desc;
 532
 533         debug("assoclen %d cryptlen %d authsize %d\n",
 534               areq->assoclen, areq->cryptlen, authsize);
 535         print_hex_dump(KERN_ERR, "assoc  @"xstr(__LINE__)": ",
 536                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->assoc),
 537                        areq->assoclen , 1);
 538         print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ",
 539                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src) - ivsize,
 540                        edesc->src_nents ? 100 : ivsize, 1);
 541         print_hex_dump(KERN_ERR, "src    @"xstr(__LINE__)": ",
 542                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src),
 543                         edesc->src_nents ? 100 : areq->cryptlen + authsize, 1);
 544         print_hex_dump(KERN_ERR, "shrdesc@"xstr(__LINE__)": ",
 545                        DUMP_PREFIX_ADDRESS, 16, 4, sh_desc,
 546                        desc_bytes(sh_desc), 1);
 547 #endif
 548         assoc_sg_count = dma_map_sg(jrdev, areq->assoc, edesc->assoc_nents ?: 1,
 549                                     DMA_TO_DEVICE);
 550         if (areq->src == areq->dst)
 551                 sg_count = dma_map_sg(jrdev, areq->src, edesc->src_nents ? : 1,
 552                                       DMA_BIDIRECTIONAL);
 553         else
 554                 sg_count = dma_map_sg(jrdev, areq->src, edesc->src_nents ? : 1,
 555                                       DMA_TO_DEVICE);
 556
 557         /* start auth operation */
 558         append_operation(desc, ctx->class2_alg_type | OP_ALG_AS_INITFINAL |
 559                          (encrypt ? : OP_ALG_ICV_ON));
 560
 561         /* Load FIFO with data for Class 2 CHA */
 562         options = FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG;
 563         if (!edesc->assoc_nents) {
 564                 ptr = sg_dma_address(areq->assoc);
 565         } else {
 566                 sg_to_link_tbl(areq->assoc, edesc->assoc_nents,
 567                                edesc->link_tbl, 0);
 568                 ptr = edesc->link_tbl_dma;
 569                 options |= LDST_SGF;
 570         }
 571         append_fifo_load(desc, ptr, areq->assoclen, options);
 572
 573         /* copy iv from cipher/class1 input context to class2 infifo */
 574         append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | ivsize);
 575
 576         /* start class 1 (cipher) operation */
 577         append_operation(desc, ctx->class1_alg_type | OP_ALG_AS_INITFINAL |
 578                          encrypt);
 579
 580         /* load payload & instruct to class2 to snoop class 1 if encrypting */
 581         options = 0;
 582         if (!edesc->src_nents) {
 583                 src_dma = sg_dma_address(areq->src);
 584         } else {
 585                 sg_to_link_tbl(areq->src, edesc->src_nents, edesc->link_tbl +
 586                                edesc->assoc_nents, 0);
 587                 src_dma = edesc->link_tbl_dma + edesc->assoc_nents *
 588                           sizeof(struct link_tbl_entry);
 589                 options |= LDST_SGF;
 590         }
 591         append_seq_in_ptr(desc, src_dma, areq->cryptlen + authsize, options);
 592         append_seq_fifo_load(desc, areq->cryptlen, FIFOLD_CLASS_BOTH |
 593                              FIFOLD_TYPE_LASTBOTH |
 594                              (encrypt ? FIFOLD_TYPE_MSG1OUT2
 595                                       : FIFOLD_TYPE_MSG));
 596
 597         /* specify destination */
 598         if (areq->src == areq->dst) {
 599                 dst_dma = src_dma;
 600         } else {
 601                 sg_count = dma_map_sg(jrdev, areq->dst, edesc->dst_nents ? : 1,
 602                                       DMA_FROM_DEVICE);
 603                 if (!edesc->dst_nents) {
 604                         dst_dma = sg_dma_address(areq->dst);
 605                         options = 0;
 606                 } else {
 607                         sg_to_link_tbl(areq->dst, edesc->dst_nents,
 608                                        edesc->link_tbl + edesc->assoc_nents +
 609                                        edesc->src_nents, 0);
 610                         dst_dma = edesc->link_tbl_dma + (edesc->assoc_nents +
 611                                   edesc->src_nents) *
 612                                   sizeof(struct link_tbl_entry);
 613                         options = LDST_SGF;
 614                 }
 615         }
 616         append_seq_out_ptr(desc, dst_dma, areq->cryptlen + authsize, options);
 617         append_seq_fifo_store(desc, areq->cryptlen, FIFOST_TYPE_MESSAGE_DATA);
 618
 619         /* ICV */
 620         if (encrypt)
 621                 append_seq_store(desc, authsize, LDST_CLASS_2_CCB |
 622                                  LDST_SRCDST_BYTE_CONTEXT);
 623         else
 624                 append_seq_fifo_load(desc, authsize, FIFOLD_CLASS_CLASS2 |
 625                                      FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
 626
 627 #ifdef DEBUG
 628         debug("job_desc_len %d\n", desc_len(desc));
 629         print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
 630                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc) , 1);
 631         print_hex_dump(KERN_ERR, "jdlinkt@"xstr(__LINE__)": ",
 632                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->link_tbl,
 633                         edesc->link_tbl_bytes, 1);
 634 #endif
 635
 636         ret = caam_jr_enqueue(jrdev, desc, callback, areq);
 637         if (!ret)
 638                 ret = -EINPROGRESS;
 639         else {
 640                 ipsec_esp_unmap(jrdev, edesc, areq);
 641                 kfree(edesc);
 642         }
 643
 644         return ret;
 645 }
 646
 647 /*
 648  * derive number of elements in scatterlist
 649  */
 650 static int sg_count(struct scatterlist *sg_list, int nbytes, int *chained)
 651 {
 652         struct scatterlist *sg = sg_list;
 653         int sg_nents = 0;
 654
 655         *chained = 0;
 656         while (nbytes > 0) {
 657                 sg_nents++;
 658                 nbytes -= sg->length;
 659                 if (!sg_is_last(sg) && (sg + 1)->length == 0)
 660                         *chained = 1;
 661                 sg = scatterwalk_sg_next(sg);
 662         }
 663
 664         return sg_nents;
 665 }
 666
 667 /*
 668  * allocate and map the ipsec_esp extended descriptor
 669  */
 670 static struct ipsec_esp_edesc *ipsec_esp_edesc_alloc(struct aead_request *areq,
 671                                                      int desc_bytes)
 672 {
 673         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
 674         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 675         struct device *jrdev = ctx->jrdev;
 676         gfp_t flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
 677                       GFP_ATOMIC;
 678         int assoc_nents, src_nents, dst_nents = 0, chained, link_tbl_bytes;
 679         struct ipsec_esp_edesc *edesc;
 680
 681         assoc_nents = sg_count(areq->assoc, areq->assoclen, &chained);
 682         BUG_ON(chained);
 683         if (likely(assoc_nents == 1))
 684                 assoc_nents = 0;
 685
 686         src_nents = sg_count(areq->src, areq->cryptlen + ctx->authsize,
 687                              &chained);
 688         BUG_ON(chained);
 689         if (src_nents == 1)
 690                 src_nents = 0;
 691
 692         if (unlikely(areq->dst != areq->src)) {
 693                 dst_nents = sg_count(areq->dst, areq->cryptlen + ctx->authsize,
 694                                      &chained);
 695                 BUG_ON(chained);
 696                 if (dst_nents == 1)
 697                         dst_nents = 0;
 698         }
 699
 700         link_tbl_bytes = (assoc_nents + src_nents + dst_nents) *
 701                          sizeof(struct link_tbl_entry);
 702         debug("link_tbl_bytes %d\n", link_tbl_bytes);
 703
 704         /* allocate space for base edesc and hw desc commands, link tables */
 705         edesc = kmalloc(sizeof(struct ipsec_esp_edesc) + desc_bytes +
 706                         link_tbl_bytes, GFP_DMA | flags);
 707         if (!edesc) {
 708                 dev_err(jrdev, "could not allocate extended descriptor\n");
 709                 return ERR_PTR(-ENOMEM);
 710         }
 711
 712         edesc->assoc_nents = assoc_nents;
 713         edesc->src_nents = src_nents;
 714         edesc->dst_nents = dst_nents;
 715         edesc->link_tbl = (void *)edesc + sizeof(struct ipsec_esp_edesc) +
 716                           desc_bytes;
 717         edesc->link_tbl_dma = dma_map_single(jrdev, edesc->link_tbl,
 718                                              link_tbl_bytes, DMA_TO_DEVICE);
 719         edesc->link_tbl_bytes = link_tbl_bytes;
 720
 721         return edesc;
 722 }
 723
 724 static int aead_authenc_encrypt(struct aead_request *areq)
 725 {
 726         struct ipsec_esp_edesc *edesc;
 727         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
 728         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 729         struct device *jrdev = ctx->jrdev;
 730         int ivsize = crypto_aead_ivsize(aead);
 731         u32 *desc;
 732         dma_addr_t iv_dma;
 733
 734         /* allocate extended descriptor */
 735         edesc = ipsec_esp_edesc_alloc(areq, 21 * sizeof(u32));
 736         if (IS_ERR(edesc))
 737                 return PTR_ERR(edesc);
 738
 739         desc = edesc->hw_desc;
 740
 741         /* insert shared descriptor pointer */
 742         init_job_desc_shared(desc, ctx->shared_desc_phys,
 743                              desc_len(ctx->sh_desc), HDR_SHARE_DEFER);
 744
 745         iv_dma = dma_map_single(jrdev, areq->iv, ivsize, DMA_TO_DEVICE);
 746         /* check dma error */
 747
 748         append_load(desc, iv_dma, ivsize,
 749                     LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT);
 750
 751         return ipsec_esp(edesc, areq, OP_ALG_ENCRYPT, ipsec_esp_encrypt_done);
 752 }
 753
 754 static int aead_authenc_decrypt(struct aead_request *req)
 755 {
 756         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 757         int ivsize = crypto_aead_ivsize(aead);
 758         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 759         struct device *jrdev = ctx->jrdev;
 760         struct ipsec_esp_edesc *edesc;
 761         u32 *desc;
 762         dma_addr_t iv_dma;
 763
 764         req->cryptlen -= ctx->authsize;
 765
 766         /* allocate extended descriptor */
 767         edesc = ipsec_esp_edesc_alloc(req, 21 * sizeof(u32));
 768         if (IS_ERR(edesc))
 769                 return PTR_ERR(edesc);
 770
 771         desc = edesc->hw_desc;
 772
 773         /* insert shared descriptor pointer */
 774         init_job_desc_shared(desc, ctx->shared_desc_phys,
 775                              desc_len(ctx->sh_desc), HDR_SHARE_DEFER);
 776
 777         iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE);
 778         /* check dma error */
 779
 780         append_load(desc, iv_dma, ivsize,
 781                     LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT);
 782
 783         return ipsec_esp(edesc, req, !OP_ALG_ENCRYPT, ipsec_esp_decrypt_done);
 784 }
 785
 786 static int aead_authenc_givencrypt(struct aead_givcrypt_request *req)
 787 {
 788         struct aead_request *areq = &req->areq;
 789         struct ipsec_esp_edesc *edesc;
 790         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
 791         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 792         struct device *jrdev = ctx->jrdev;
 793         int ivsize = crypto_aead_ivsize(aead);
 794         dma_addr_t iv_dma;
 795         u32 *desc;
 796
 797         iv_dma = dma_map_single(jrdev, req->giv, ivsize, DMA_FROM_DEVICE);
 798
 799         debug("%s: giv %p\n", __func__, req->giv);
 800
 801         /* allocate extended descriptor */
 802         edesc = ipsec_esp_edesc_alloc(areq, 27 * sizeof(u32));
 803         if (IS_ERR(edesc))
 804                 return PTR_ERR(edesc);
 805
 806         desc = edesc->hw_desc;
 807
 808         /* insert shared descriptor pointer */
 809         init_job_desc_shared(desc, ctx->shared_desc_phys,
 810                              desc_len(ctx->sh_desc), HDR_SHARE_DEFER);
 811
 812         /*
 813          * LOAD IMM Info FIFO
 814          * to DECO, Last, Padding, Random, Message, 16 bytes
 815          */
 816         append_load_imm_u32(desc, NFIFOENTRY_DEST_DECO | NFIFOENTRY_LC1 |
 817                             NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DTYPE_MSG |
 818                             NFIFOENTRY_PTYPE_RND | ivsize,
 819                             LDST_SRCDST_WORD_INFO_FIFO);
 820
 821         /*
 822          * disable info fifo entries since the above serves as the entry
 823          * this way, the MOVE command won't generate an entry.
 824          * Note that this isn't required in more recent versions of
 825          * SEC as a MOVE that doesn't do info FIFO entries is available.
 826          */
 827         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
 828
 829         /* MOVE DECO Alignment -> C1 Context 16 bytes */
 830         append_move(desc, MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | ivsize);
 831
 832         /* re-enable info fifo entries */
 833         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
 834
 835         /* MOVE C1 Context -> OFIFO 16 bytes */
 836         append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | ivsize);
 837
 838         append_fifo_store(desc, iv_dma, ivsize, FIFOST_TYPE_MESSAGE_DATA);
 839
 840         return ipsec_esp(edesc, areq, OP_ALG_ENCRYPT, ipsec_esp_encrypt_done);
 841 }
 842
 843 struct caam_alg_template {
 844         char name[CRYPTO_MAX_ALG_NAME];
 845         char driver_name[CRYPTO_MAX_ALG_NAME];
 846         unsigned int blocksize;
 847         struct aead_alg aead;
 848         u32 class1_alg_type;
 849         u32 class2_alg_type;
 850         u32 alg_op;
 851 };
 852
 853 static struct caam_alg_template driver_algs[] = {
 854         /* single-pass ipsec_esp descriptor */
 855         {
 856                 .name = "authenc(hmac(sha1),cbc(aes))",
 857                 .driver_name = "authenc-hmac-sha1-cbc-aes-caam",
 858                 .blocksize = AES_BLOCK_SIZE,
 859                 .aead = {
 860                         .setkey = aead_authenc_setkey,
 861                         .setauthsize = aead_authenc_setauthsize,
 862                         .encrypt = aead_authenc_encrypt,
 863                         .decrypt = aead_authenc_decrypt,
 864                         .givencrypt = aead_authenc_givencrypt,
 865                         .geniv = "<built-in>",
 866                         .ivsize = AES_BLOCK_SIZE,
 867                         .maxauthsize = SHA1_DIGEST_SIZE,
 868                         },
 869                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
 870                 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
 871                 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
 872         },
 873         {
 874                 .name = "authenc(hmac(sha256),cbc(aes))",
 875                 .driver_name = "authenc-hmac-sha256-cbc-aes-caam",
 876                 .blocksize = AES_BLOCK_SIZE,
 877                 .aead = {
 878                         .setkey = aead_authenc_setkey,
 879                         .setauthsize = aead_authenc_setauthsize,
 880                         .encrypt = aead_authenc_encrypt,
 881                         .decrypt = aead_authenc_decrypt,
 882                         .givencrypt = aead_authenc_givencrypt,
 883                         .geniv = "<built-in>",
 884                         .ivsize = AES_BLOCK_SIZE,
 885                         .maxauthsize = SHA256_DIGEST_SIZE,
 886                         },
 887                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
 888                 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
 889                                    OP_ALG_AAI_HMAC_PRECOMP,
 890                 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
 891         },
 892         {
 893                 .name = "authenc(hmac(sha1),cbc(des3_ede))",
 894                 .driver_name = "authenc-hmac-sha1-cbc-des3_ede-caam",
 895                 .blocksize = DES3_EDE_BLOCK_SIZE,
 896                 .aead = {
 897                         .setkey = aead_authenc_setkey,
 898                         .setauthsize = aead_authenc_setauthsize,
 899                         .encrypt = aead_authenc_encrypt,
 900                         .decrypt = aead_authenc_decrypt,
 901                         .givencrypt = aead_authenc_givencrypt,
 902                         .geniv = "<built-in>",
 903                         .ivsize = DES3_EDE_BLOCK_SIZE,
 904                         .maxauthsize = SHA1_DIGEST_SIZE,
 905                         },
 906                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
 907                 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
 908                 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
 909         },
 910         {
 911                 .name = "authenc(hmac(sha256),cbc(des3_ede))",
 912                 .driver_name = "authenc-hmac-sha256-cbc-des3_ede-caam",
 913                 .blocksize = DES3_EDE_BLOCK_SIZE,
 914                 .aead = {
 915                         .setkey = aead_authenc_setkey,
 916                         .setauthsize = aead_authenc_setauthsize,
 917                         .encrypt = aead_authenc_encrypt,
 918                         .decrypt = aead_authenc_decrypt,
 919                         .givencrypt = aead_authenc_givencrypt,
 920                         .geniv = "<built-in>",
 921                         .ivsize = DES3_EDE_BLOCK_SIZE,
 922                         .maxauthsize = SHA256_DIGEST_SIZE,
 923                         },
 924                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
 925                 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
 926                                    OP_ALG_AAI_HMAC_PRECOMP,
 927                 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
 928         },
 929         {
 930                 .name = "authenc(hmac(sha1),cbc(des))",
 931                 .driver_name = "authenc-hmac-sha1-cbc-des-caam",
 932                 .blocksize = DES_BLOCK_SIZE,
 933                 .aead = {
 934                         .setkey = aead_authenc_setkey,
 935                         .setauthsize = aead_authenc_setauthsize,
 936                         .encrypt = aead_authenc_encrypt,
 937                         .decrypt = aead_authenc_decrypt,
 938                         .givencrypt = aead_authenc_givencrypt,
 939                         .geniv = "<built-in>",
 940                         .ivsize = DES_BLOCK_SIZE,
 941                         .maxauthsize = SHA1_DIGEST_SIZE,
 942                         },
 943                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
 944                 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
 945                 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
 946         },
 947         {
 948                 .name = "authenc(hmac(sha256),cbc(des))",
 949                 .driver_name = "authenc-hmac-sha256-cbc-des-caam",
 950                 .blocksize = DES_BLOCK_SIZE,
 951                 .aead = {
 952                         .setkey = aead_authenc_setkey,
 953                         .setauthsize = aead_authenc_setauthsize,
 954                         .encrypt = aead_authenc_encrypt,
 955                         .decrypt = aead_authenc_decrypt,
 956                         .givencrypt = aead_authenc_givencrypt,
 957                         .geniv = "<built-in>",
 958                         .ivsize = DES_BLOCK_SIZE,
 959                         .maxauthsize = SHA256_DIGEST_SIZE,
 960                         },
 961                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
 962                 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
 963                                    OP_ALG_AAI_HMAC_PRECOMP,
 964                 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
 965         },
 966 };
 967
 968 struct caam_crypto_alg {
 969         struct list_head entry;
 970         struct device *ctrldev;
 971         int class1_alg_type;
 972         int class2_alg_type;
 973         int alg_op;
 974         struct crypto_alg crypto_alg;
 975 };
 976
 977 static int caam_cra_init(struct crypto_tfm *tfm)
 978 {
 979         struct crypto_alg *alg = tfm->__crt_alg;
 980         struct caam_crypto_alg *caam_alg =
 981                  container_of(alg, struct caam_crypto_alg, crypto_alg);
 982         struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
 983         struct caam_drv_private *priv = dev_get_drvdata(caam_alg->ctrldev);
 984         int tgt_jr = atomic_inc_return(&priv->tfm_count);
 985
 986         /*
 987          * distribute tfms across job rings to ensure in-order
 988          * crypto request processing per tfm
 989          */
 990         ctx->jrdev = priv->algapi_jr[(tgt_jr / 2) % priv->num_jrs_for_algapi];
 991
 992         /* copy descriptor header template value */
 993         ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam_alg->class1_alg_type;
 994         ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam_alg->class2_alg_type;
 995         ctx->alg_op = OP_TYPE_CLASS2_ALG | caam_alg->alg_op;
 996
 997         return 0;
 998 }
 999
1000 static void caam_cra_exit(struct crypto_tfm *tfm)
1001 {
1002         struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
1003
1004         if (!dma_mapping_error(ctx->jrdev, ctx->shared_desc_phys))
1005                 dma_unmap_single(ctx->jrdev, ctx->shared_desc_phys,
1006                                  desc_bytes(ctx->sh_desc), DMA_TO_DEVICE);
1007         kfree(ctx->sh_desc);
1008 }
1009
1010 static void __exit caam_algapi_exit(void)
1011 {
1012
1013         struct device_node *dev_node;
1014         struct platform_device *pdev;
1015         struct device *ctrldev;
1016         struct caam_drv_private *priv;
1017         struct caam_crypto_alg *t_alg, *n;
1018         int i, err;
1019
1020         dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
1021         if (!dev_node)
1022                 return;
1023
1024         pdev = of_find_device_by_node(dev_node);
1025         if (!pdev)
1026                 return;
1027
1028         ctrldev = &pdev->dev;
1029         of_node_put(dev_node);
1030         priv = dev_get_drvdata(ctrldev);
1031
1032         if (!priv->alg_list.next)
1033                 return;
1034
1035         list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
1036                 crypto_unregister_alg(&t_alg->crypto_alg);
1037                 list_del(&t_alg->entry);
1038                 kfree(t_alg);
1039         }
1040
1041         for (i = 0; i < priv->total_jobrs; i++) {
1042                 err = caam_jr_deregister(priv->algapi_jr[i]);
1043                 if (err < 0)
1044                         break;
1045         }
1046         kfree(priv->algapi_jr);
1047 }
1048
1049 static struct caam_crypto_alg *caam_alg_alloc(struct device *ctrldev,
1050                                               struct caam_alg_template
1051                                               *template)
1052 {
1053         struct caam_crypto_alg *t_alg;
1054         struct crypto_alg *alg;
1055
1056         t_alg = kzalloc(sizeof(struct caam_crypto_alg), GFP_KERNEL);
1057         if (!t_alg) {
1058                 dev_err(ctrldev, "failed to allocate t_alg\n");
1059                 return ERR_PTR(-ENOMEM);
1060         }
1061
1062         alg = &t_alg->crypto_alg;
1063
1064         snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name);
1065         snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1066                  template->driver_name);
1067         alg->cra_module = THIS_MODULE;
1068         alg->cra_init = caam_cra_init;
1069         alg->cra_exit = caam_cra_exit;
1070         alg->cra_priority = CAAM_CRA_PRIORITY;
1071         alg->cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC;
1072         alg->cra_blocksize = template->blocksize;
1073         alg->cra_alignmask = 0;
1074         alg->cra_type = &crypto_aead_type;
1075         alg->cra_ctxsize = sizeof(struct caam_ctx);
1076         alg->cra_u.aead = template->aead;
1077
1078         t_alg->class1_alg_type = template->class1_alg_type;
1079         t_alg->class2_alg_type = template->class2_alg_type;
1080         t_alg->alg_op = template->alg_op;
1081         t_alg->ctrldev = ctrldev;
1082
1083         return t_alg;
1084 }
1085
1086 static int __init caam_algapi_init(void)
1087 {
1088         struct device_node *dev_node;
1089         struct platform_device *pdev;
1090         struct device *ctrldev, **jrdev;
1091         struct caam_drv_private *priv;
1092         int i = 0, err = 0;
1093
1094         dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
1095         if (!dev_node)
1096                 return -ENODEV;
1097
1098         pdev = of_find_device_by_node(dev_node);
1099         if (!pdev)
1100                 return -ENODEV;
1101
1102         ctrldev = &pdev->dev;
1103         priv = dev_get_drvdata(ctrldev);
1104         of_node_put(dev_node);
1105
1106         INIT_LIST_HEAD(&priv->alg_list);
1107
1108         jrdev = kmalloc(sizeof(*jrdev) * priv->total_jobrs, GFP_KERNEL);
1109         if (!jrdev)
1110                 return -ENOMEM;
1111
1112         for (i = 0; i < priv->total_jobrs; i++) {
1113                 err = caam_jr_register(ctrldev, &jrdev[i]);
1114                 if (err < 0)
1115                         break;
1116         }
1117         if (err < 0 && i == 0) {
1118                 dev_err(ctrldev, "algapi error in job ring registration: %d\n",
1119                         err);
1120                 kfree(jrdev);
1121                 return err;
1122         }
1123
1124         priv->num_jrs_for_algapi = i;
1125         priv->algapi_jr = jrdev;
1126         atomic_set(&priv->tfm_count, -1);
1127
1128         /* register crypto algorithms the device supports */
1129         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
1130                 /* TODO: check if h/w supports alg */
1131                 struct caam_crypto_alg *t_alg;
1132
1133                 t_alg = caam_alg_alloc(ctrldev, &driver_algs[i]);
1134                 if (IS_ERR(t_alg)) {
1135                         err = PTR_ERR(t_alg);
1136                         dev_warn(ctrldev, "%s alg allocation failed\n",
1137                                  driver_algs[i].driver_name);
1138                         continue;
1139                 }
1140
1141                 err = crypto_register_alg(&t_alg->crypto_alg);
1142                 if (err) {
1143                         dev_warn(ctrldev, "%s alg registration failed\n",
1144                                 t_alg->crypto_alg.cra_driver_name);
1145                         kfree(t_alg);
1146                 } else {
1147                         list_add_tail(&t_alg->entry, &priv->alg_list);
1148                         dev_info(ctrldev, "%s\n",
1149                                  t_alg->crypto_alg.cra_driver_name);
1150                 }
1151         }
1152
1153         return err;
1154 }
1155
1156 module_init(caam_algapi_init);
1157 module_exit(caam_algapi_exit);
1158
1159 MODULE_LICENSE("GPL");
1160 MODULE_DESCRIPTION("FSL CAAM support for crypto API");
1161 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");