drivers/crypto/caam/caamalg.c

   1 /*
   2  * caam - Freescale FSL CAAM support for crypto API
   3  *
   4  * Copyright 2008-2011 Freescale Semiconductor, Inc.
   5  *
   6  * Based on talitos crypto API driver.
   7  *
   8  * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008):
   9  *
  10  * ---------------                     ---------------
  11  * | JobDesc #1  |-------------------->|  ShareDesc  |
  12  * | *(packet 1) |                     |   (PDB)     |
  13  * ---------------      |------------->|  (hashKey)  |
  14  *       .              |              | (cipherKey) |
  15  *       .              |    |-------->| (operation) |
  16  * ---------------      |    |         ---------------
  17  * | JobDesc #2  |------|    |
  18  * | *(packet 2) |           |
  19  * ---------------           |
  20  *       .                   |
  21  *       .                   |
  22  * ---------------           |
  23  * | JobDesc #3  |------------
  24  * | *(packet 3) |
  25  * ---------------
  26  *
  27  * The SharedDesc never changes for a connection unless rekeyed, but
  28  * each packet will likely be in a different place. So all we need
  29  * to know to process the packet is where the input is, where the
  30  * output goes, and what context we want to process with. Context is
  31  * in the SharedDesc, packet references in the JobDesc.
  32  *
  33  * So, a job desc looks like:
  34  *
  35  * ---------------------
  36  * | Header            |
  37  * | ShareDesc Pointer |
  38  * | SEQ_OUT_PTR       |
  39  * | (output buffer)   |
  40  * | SEQ_IN_PTR        |
  41  * | (input buffer)    |
  42  * | LOAD (to DECO)    |
  43  * ---------------------
  44  */
  45
  46 #include "compat.h"
  47
  48 #include "regs.h"
  49 #include "intern.h"
  50 #include "desc_constr.h"
  51 #include "jr.h"
  52 #include "error.h"
  53
  54 /*
  55  * crypto alg
  56  */
  57 #define CAAM_CRA_PRIORITY               3000
  58 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
  59 #define CAAM_MAX_KEY_SIZE               (AES_MAX_KEY_SIZE + \
  60                                          SHA512_DIGEST_SIZE * 2)
  61 /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
  62 #define CAAM_MAX_IV_LENGTH              16
  63
  64 /* length of descriptors text */
  65 #define DESC_AEAD_SHARED_TEXT_LEN       4
  66 #define DESC_AEAD_ENCRYPT_TEXT_LEN      21
  67 #define DESC_AEAD_DECRYPT_TEXT_LEN      24
  68 #define DESC_AEAD_GIVENCRYPT_TEXT_LEN   27
  69
  70 #ifdef DEBUG
  71 /* for print_hex_dumps with line references */
  72 #define xstr(s) str(s)
  73 #define str(s) #s
  74 #define debug(format, arg...) printk(format, arg)
  75 #else
  76 #define debug(format, arg...)
  77 #endif
  78
  79 /*
  80  * per-session context
  81  */
  82 struct caam_ctx {
  83         struct device *jrdev;
  84         u32 *sh_desc;
  85         dma_addr_t shared_desc_phys;
  86         u32 class1_alg_type;
  87         u32 class2_alg_type;
  88         u32 alg_op;
  89         u8 *key;
  90         dma_addr_t key_phys;
  91         unsigned int enckeylen;
  92         unsigned int split_key_len;
  93         unsigned int split_key_pad_len;
  94         unsigned int authsize;
  95 };
  96
  97 static int aead_authenc_setauthsize(struct crypto_aead *authenc,
  98                                     unsigned int authsize)
  99 {
 100         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 101
 102         ctx->authsize = authsize;
 103
 104         return 0;
 105 }
 106
 107 struct split_key_result {
 108         struct completion completion;
 109         int err;
 110 };
 111
 112 static void split_key_done(struct device *dev, u32 *desc, u32 err,
 113                            void *context)
 114 {
 115         struct split_key_result *res = context;
 116
 117 #ifdef DEBUG
 118         dev_err(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 119 #endif
 120         if (err) {
 121                 char tmp[CAAM_ERROR_STR_MAX];
 122
 123                 dev_err(dev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
 124         }
 125
 126         res->err = err;
 127
 128         complete(&res->completion);
 129 }
 130
 131 /*
 132 get a split ipad/opad key
 133
 134 Split key generation-----------------------------------------------
 135
 136 [00] 0xb0810008    jobdesc: stidx=1 share=never len=8
 137 [01] 0x04000014        key: class2->keyreg len=20
 138                         @0xffe01000
 139 [03] 0x84410014  operation: cls2-op sha1 hmac init dec
 140 [04] 0x24940000     fifold: class2 msgdata-last2 len=0 imm
 141 [05] 0xa4000001       jump: class2 local all ->1 [06]
 142 [06] 0x64260028    fifostr: class2 mdsplit-jdk len=40
 143                         @0xffe04000
 144 */
 145 static u32 gen_split_key(struct caam_ctx *ctx, const u8 *key_in, u32 authkeylen)
 146 {
 147         struct device *jrdev = ctx->jrdev;
 148         u32 *desc;
 149         struct split_key_result result;
 150         dma_addr_t dma_addr_in, dma_addr_out;
 151         int ret = 0;
 152
 153         desc = kmalloc(CAAM_CMD_SZ * 6 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
 154
 155         init_job_desc(desc, 0);
 156
 157         dma_addr_in = dma_map_single(jrdev, (void *)key_in, authkeylen,
 158                                      DMA_TO_DEVICE);
 159         if (dma_mapping_error(jrdev, dma_addr_in)) {
 160                 dev_err(jrdev, "unable to map key input memory\n");
 161                 kfree(desc);
 162                 return -ENOMEM;
 163         }
 164         append_key(desc, dma_addr_in, authkeylen, CLASS_2 |
 165                        KEY_DEST_CLASS_REG);
 166
 167         /* Sets MDHA up into an HMAC-INIT */
 168         append_operation(desc, ctx->alg_op | OP_ALG_DECRYPT |
 169                              OP_ALG_AS_INIT);
 170
 171         /*
 172          * do a FIFO_LOAD of zero, this will trigger the internal key expansion
 173            into both pads inside MDHA
 174          */
 175         append_fifo_load_as_imm(desc, NULL, 0, LDST_CLASS_2_CCB |
 176                                 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST2);
 177
 178         /*
 179          * FIFO_STORE with the explicit split-key content store
 180          * (0x26 output type)
 181          */
 182         dma_addr_out = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len,
 183                                       DMA_FROM_DEVICE);
 184         if (dma_mapping_error(jrdev, dma_addr_out)) {
 185                 dev_err(jrdev, "unable to map key output memory\n");
 186                 kfree(desc);
 187                 return -ENOMEM;
 188         }
 189         append_fifo_store(desc, dma_addr_out, ctx->split_key_len,
 190                           LDST_CLASS_2_CCB | FIFOST_TYPE_SPLIT_KEK);
 191
 192 #ifdef DEBUG
 193         print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ",
 194                        DUMP_PREFIX_ADDRESS, 16, 4, key_in, authkeylen, 1);
 195         print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
 196                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 197 #endif
 198
 199         result.err = 0;
 200         init_completion(&result.completion);
 201
 202         ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result);
 203         if (!ret) {
 204                 /* in progress */
 205                 wait_for_completion_interruptible(&result.completion);
 206                 ret = result.err;
 207 #ifdef DEBUG
 208                 print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ",
 209                                DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 210                                ctx->split_key_pad_len, 1);
 211 #endif
 212         }
 213
 214         dma_unmap_single(jrdev, dma_addr_out, ctx->split_key_pad_len,
 215                          DMA_FROM_DEVICE);
 216         dma_unmap_single(jrdev, dma_addr_in, authkeylen, DMA_TO_DEVICE);
 217
 218         kfree(desc);
 219
 220         return ret;
 221 }
 222
 223 static int build_sh_desc_ipsec(struct caam_ctx *ctx)
 224 {
 225         struct device *jrdev = ctx->jrdev;
 226         u32 *sh_desc;
 227         u32 *jump_cmd;
 228         bool keys_fit_inline = 0;
 229
 230         /*
 231          * largest Job Descriptor and its Shared Descriptor
 232          * must both fit into the 64-word Descriptor h/w Buffer
 233          */
 234         if ((DESC_AEAD_GIVENCRYPT_TEXT_LEN +
 235              DESC_AEAD_SHARED_TEXT_LEN) * CAAM_CMD_SZ +
 236             ctx->split_key_pad_len + ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
 237                 keys_fit_inline = 1;
 238
 239         /* build shared descriptor for this session */
 240         sh_desc = kmalloc(CAAM_CMD_SZ * DESC_AEAD_SHARED_TEXT_LEN +
 241                           keys_fit_inline ?
 242                           ctx->split_key_pad_len + ctx->enckeylen :
 243                           CAAM_PTR_SZ * 2, GFP_DMA | GFP_KERNEL);
 244         if (!sh_desc) {
 245                 dev_err(jrdev, "could not allocate shared descriptor\n");
 246                 return -ENOMEM;
 247         }
 248
 249         init_sh_desc(sh_desc, HDR_SAVECTX | HDR_SHARE_SERIAL);
 250
 251         jump_cmd = append_jump(sh_desc, CLASS_BOTH | JUMP_TEST_ALL |
 252                                JUMP_COND_SHRD | JUMP_COND_SELF);
 253
 254         /*
 255          * process keys, starting with class 2/authentication.
 256          */
 257         if (keys_fit_inline) {
 258                 append_key_as_imm(sh_desc, ctx->key, ctx->split_key_pad_len,
 259                                   ctx->split_key_len,
 260                                   CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
 261
 262                 append_key_as_imm(sh_desc, (void *)ctx->key +
 263                                   ctx->split_key_pad_len, ctx->enckeylen,
 264                                   ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
 265         } else {
 266                 append_key(sh_desc, ctx->key_phys, ctx->split_key_len, CLASS_2 |
 267                            KEY_DEST_MDHA_SPLIT | KEY_ENC);
 268                 append_key(sh_desc, ctx->key_phys + ctx->split_key_pad_len,
 269                            ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
 270         }
 271
 272         /* update jump cmd now that we are at the jump target */
 273         set_jump_tgt_here(sh_desc, jump_cmd);
 274
 275         ctx->shared_desc_phys = dma_map_single(jrdev, sh_desc,
 276                                                desc_bytes(sh_desc),
 277                                                DMA_TO_DEVICE);
 278         if (dma_mapping_error(jrdev, ctx->shared_desc_phys)) {
 279                 dev_err(jrdev, "unable to map shared descriptor\n");
 280                 kfree(sh_desc);
 281                 return -ENOMEM;
 282         }
 283
 284         ctx->sh_desc = sh_desc;
 285
 286         return 0;
 287 }
 288
 289 static int aead_authenc_setkey(struct crypto_aead *aead,
 290                                const u8 *key, unsigned int keylen)
 291 {
 292         /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
 293         static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
 294         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 295         struct device *jrdev = ctx->jrdev;
 296         struct rtattr *rta = (void *)key;
 297         struct crypto_authenc_key_param *param;
 298         unsigned int authkeylen;
 299         unsigned int enckeylen;
 300         int ret = 0;
 301
 302         param = RTA_DATA(rta);
 303         enckeylen = be32_to_cpu(param->enckeylen);
 304
 305         key += RTA_ALIGN(rta->rta_len);
 306         keylen -= RTA_ALIGN(rta->rta_len);
 307
 308         if (keylen < enckeylen)
 309                 goto badkey;
 310
 311         authkeylen = keylen - enckeylen;
 312
 313         if (keylen > CAAM_MAX_KEY_SIZE)
 314                 goto badkey;
 315
 316         /* Pick class 2 key length from algorithm submask */
 317         ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >>
 318                                       OP_ALG_ALGSEL_SHIFT] * 2;
 319         ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16);
 320
 321 #ifdef DEBUG
 322         printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n",
 323                keylen, enckeylen, authkeylen);
 324         printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
 325                ctx->split_key_len, ctx->split_key_pad_len);
 326         print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ",
 327                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 328 #endif
 329         ctx->key = kmalloc(ctx->split_key_pad_len + enckeylen,
 330                            GFP_KERNEL | GFP_DMA);
 331         if (!ctx->key) {
 332                 dev_err(jrdev, "could not allocate key output memory\n");
 333                 return -ENOMEM;
 334         }
 335
 336         ret = gen_split_key(ctx, key, authkeylen);
 337         if (ret) {
 338                 kfree(ctx->key);
 339                 goto badkey;
 340         }
 341
 342         /* postpend encryption key to auth split key */
 343         memcpy(ctx->key + ctx->split_key_pad_len, key + authkeylen, enckeylen);
 344
 345         ctx->key_phys = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len +
 346                                        enckeylen, DMA_TO_DEVICE);
 347         if (dma_mapping_error(jrdev, ctx->key_phys)) {
 348                 dev_err(jrdev, "unable to map key i/o memory\n");
 349                 kfree(ctx->key);
 350                 return -ENOMEM;
 351         }
 352 #ifdef DEBUG
 353         print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ",
 354                        DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 355                        ctx->split_key_pad_len + enckeylen, 1);
 356 #endif
 357
 358         ctx->enckeylen = enckeylen;
 359
 360         ret = build_sh_desc_ipsec(ctx);
 361         if (ret) {
 362                 dma_unmap_single(jrdev, ctx->key_phys, ctx->split_key_pad_len +
 363                                  enckeylen, DMA_TO_DEVICE);
 364                 kfree(ctx->key);
 365         }
 366
 367         return ret;
 368 badkey:
 369         crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
 370         return -EINVAL;
 371 }
 372
 373 struct link_tbl_entry {
 374         u64 ptr;
 375         u32 len;
 376         u8 reserved;
 377         u8 buf_pool_id;
 378         u16 offset;
 379 };
 380
 381 /*
 382  * ipsec_esp_edesc - s/w-extended ipsec_esp descriptor
 383  * @src_nents: number of segments in input scatterlist
 384  * @dst_nents: number of segments in output scatterlist
 385  * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist
 386  * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
 387  * @link_tbl_bytes: length of dma mapped link_tbl space
 388  * @link_tbl_dma: bus physical mapped address of h/w link table
 389  * @hw_desc: the h/w job descriptor followed by any referenced link tables
 390  */
 391 struct ipsec_esp_edesc {
 392         int assoc_nents;
 393         int src_nents;
 394         int dst_nents;
 395         int link_tbl_bytes;
 396         dma_addr_t link_tbl_dma;
 397         struct link_tbl_entry *link_tbl;
 398         u32 hw_desc[0];
 399 };
 400
 401 static void ipsec_esp_unmap(struct device *dev,
 402                             struct ipsec_esp_edesc *edesc,
 403                             struct aead_request *areq)
 404 {
 405         dma_unmap_sg(dev, areq->assoc, edesc->assoc_nents, DMA_TO_DEVICE);
 406
 407         if (unlikely(areq->dst != areq->src)) {
 408                 dma_unmap_sg(dev, areq->src, edesc->src_nents,
 409                              DMA_TO_DEVICE);
 410                 dma_unmap_sg(dev, areq->dst, edesc->dst_nents,
 411                              DMA_FROM_DEVICE);
 412         } else {
 413                 dma_unmap_sg(dev, areq->src, edesc->src_nents,
 414                              DMA_BIDIRECTIONAL);
 415         }
 416
 417         if (edesc->link_tbl_bytes)
 418                 dma_unmap_single(dev, edesc->link_tbl_dma,
 419                                  edesc->link_tbl_bytes,
 420                                  DMA_TO_DEVICE);
 421 }
 422
 423 /*
 424  * ipsec_esp descriptor callbacks
 425  */
 426 static void ipsec_esp_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
 427                                    void *context)
 428 {
 429         struct aead_request *areq = context;
 430         struct ipsec_esp_edesc *edesc;
 431 #ifdef DEBUG
 432         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
 433         int ivsize = crypto_aead_ivsize(aead);
 434         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 435
 436         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 437 #endif
 438         edesc = (struct ipsec_esp_edesc *)((char *)desc -
 439                  offsetof(struct ipsec_esp_edesc, hw_desc));
 440
 441         if (err) {
 442                 char tmp[CAAM_ERROR_STR_MAX];
 443
 444                 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
 445         }
 446
 447         ipsec_esp_unmap(jrdev, edesc, areq);
 448
 449 #ifdef DEBUG
 450         print_hex_dump(KERN_ERR, "assoc  @"xstr(__LINE__)": ",
 451                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->assoc),
 452                        areq->assoclen , 1);
 453         print_hex_dump(KERN_ERR, "dstiv  @"xstr(__LINE__)": ",
 454                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src) - ivsize,
 455                        edesc->src_nents ? 100 : ivsize, 1);
 456         print_hex_dump(KERN_ERR, "dst    @"xstr(__LINE__)": ",
 457                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src),
 458                        edesc->src_nents ? 100 : areq->cryptlen +
 459                        ctx->authsize + 4, 1);
 460 #endif
 461
 462         kfree(edesc);
 463
 464         aead_request_complete(areq, err);
 465 }
 466
 467 static void ipsec_esp_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
 468                                    void *context)
 469 {
 470         struct aead_request *areq = context;
 471         struct ipsec_esp_edesc *edesc;
 472 #ifdef DEBUG
 473         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
 474         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 475
 476         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 477 #endif
 478         edesc = (struct ipsec_esp_edesc *)((char *)desc -
 479                  offsetof(struct ipsec_esp_edesc, hw_desc));
 480
 481         if (err) {
 482                 char tmp[CAAM_ERROR_STR_MAX];
 483
 484                 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
 485         }
 486
 487         ipsec_esp_unmap(jrdev, edesc, areq);
 488
 489         /*
 490          * verify hw auth check passed else return -EBADMSG
 491          */
 492         if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK)
 493                 err = -EBADMSG;
 494
 495 #ifdef DEBUG
 496         print_hex_dump(KERN_ERR, "iphdrout@"xstr(__LINE__)": ",
 497                        DUMP_PREFIX_ADDRESS, 16, 4,
 498                        ((char *)sg_virt(areq->assoc) - sizeof(struct iphdr)),
 499                        sizeof(struct iphdr) + areq->assoclen +
 500                        ((areq->cryptlen > 1500) ? 1500 : areq->cryptlen) +
 501                        ctx->authsize + 36, 1);
 502         if (!err && edesc->link_tbl_bytes) {
 503                 struct scatterlist *sg = sg_last(areq->src, edesc->src_nents);
 504                 print_hex_dump(KERN_ERR, "sglastout@"xstr(__LINE__)": ",
 505                                DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(sg),
 506                         sg->length + ctx->authsize + 16, 1);
 507         }
 508 #endif
 509         kfree(edesc);
 510
 511         aead_request_complete(areq, err);
 512 }
 513
 514 /*
 515  * convert scatterlist to h/w link table format
 516  * scatterlist must have been previously dma mapped
 517  */
 518 static void sg_to_link_tbl(struct scatterlist *sg, int sg_count,
 519                            struct link_tbl_entry *link_tbl_ptr, u32 offset)
 520 {
 521         while (sg_count) {
 522                 link_tbl_ptr->ptr = sg_dma_address(sg);
 523                 link_tbl_ptr->len = sg_dma_len(sg);
 524                 link_tbl_ptr->reserved = 0;
 525                 link_tbl_ptr->buf_pool_id = 0;
 526                 link_tbl_ptr->offset = offset;
 527                 link_tbl_ptr++;
 528                 sg = sg_next(sg);
 529                 sg_count--;
 530         }
 531
 532         /* set Final bit (marks end of link table) */
 533         link_tbl_ptr--;
 534         link_tbl_ptr->len |= 0x40000000;
 535 }
 536
 537 /*
 538  * fill in and submit ipsec_esp job descriptor
 539  */
 540 static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq,
 541                      u32 encrypt,
 542                      void (*callback) (struct device *dev, u32 *desc,
 543                                        u32 err, void *context))
 544 {
 545         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
 546         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 547         struct device *jrdev = ctx->jrdev;
 548         u32 *desc = edesc->hw_desc, options;
 549         int ret, sg_count, assoc_sg_count;
 550         int ivsize = crypto_aead_ivsize(aead);
 551         int authsize = ctx->authsize;
 552         dma_addr_t ptr, dst_dma, src_dma;
 553 #ifdef DEBUG
 554         u32 *sh_desc = ctx->sh_desc;
 555
 556         debug("assoclen %d cryptlen %d authsize %d\n",
 557               areq->assoclen, areq->cryptlen, authsize);
 558         print_hex_dump(KERN_ERR, "assoc  @"xstr(__LINE__)": ",
 559                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->assoc),
 560                        areq->assoclen , 1);
 561         print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ",
 562                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src) - ivsize,
 563                        edesc->src_nents ? 100 : ivsize, 1);
 564         print_hex_dump(KERN_ERR, "src    @"xstr(__LINE__)": ",
 565                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src),
 566                         edesc->src_nents ? 100 : areq->cryptlen + authsize, 1);
 567         print_hex_dump(KERN_ERR, "shrdesc@"xstr(__LINE__)": ",
 568                        DUMP_PREFIX_ADDRESS, 16, 4, sh_desc,
 569                        desc_bytes(sh_desc), 1);
 570 #endif
 571         assoc_sg_count = dma_map_sg(jrdev, areq->assoc, edesc->assoc_nents ?: 1,
 572                                     DMA_TO_DEVICE);
 573         if (areq->src == areq->dst)
 574                 sg_count = dma_map_sg(jrdev, areq->src, edesc->src_nents ? : 1,
 575                                       DMA_BIDIRECTIONAL);
 576         else
 577                 sg_count = dma_map_sg(jrdev, areq->src, edesc->src_nents ? : 1,
 578                                       DMA_TO_DEVICE);
 579
 580         /* start auth operation */
 581         append_operation(desc, ctx->class2_alg_type | OP_ALG_AS_INITFINAL |
 582                          (encrypt ? : OP_ALG_ICV_ON));
 583
 584         /* Load FIFO with data for Class 2 CHA */
 585         options = FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG;
 586         if (!edesc->assoc_nents) {
 587                 ptr = sg_dma_address(areq->assoc);
 588         } else {
 589                 sg_to_link_tbl(areq->assoc, edesc->assoc_nents,
 590                                edesc->link_tbl, 0);
 591                 ptr = edesc->link_tbl_dma;
 592                 options |= LDST_SGF;
 593         }
 594         append_fifo_load(desc, ptr, areq->assoclen, options);
 595
 596         /* copy iv from cipher/class1 input context to class2 infifo */
 597         append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | ivsize);
 598
 599         if (!encrypt) {
 600                 u32 *jump_cmd, *uncond_jump_cmd;
 601
 602                 /* JUMP if shared */
 603                 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
 604
 605                 /* start class 1 (cipher) operation, non-shared version */
 606                 append_operation(desc, ctx->class1_alg_type |
 607                                  OP_ALG_AS_INITFINAL);
 608
 609                 uncond_jump_cmd = append_jump(desc, 0);
 610
 611                 set_jump_tgt_here(desc, jump_cmd);
 612
 613                 /* start class 1 (cipher) operation, shared version */
 614                 append_operation(desc, ctx->class1_alg_type |
 615                                  OP_ALG_AS_INITFINAL | OP_ALG_AAI_DK);
 616                 set_jump_tgt_here(desc, uncond_jump_cmd);
 617         } else
 618                 append_operation(desc, ctx->class1_alg_type |
 619                                  OP_ALG_AS_INITFINAL | encrypt);
 620
 621         /* load payload & instruct to class2 to snoop class 1 if encrypting */
 622         options = 0;
 623         if (!edesc->src_nents) {
 624                 src_dma = sg_dma_address(areq->src);
 625         } else {
 626                 sg_to_link_tbl(areq->src, edesc->src_nents, edesc->link_tbl +
 627                                edesc->assoc_nents, 0);
 628                 src_dma = edesc->link_tbl_dma + edesc->assoc_nents *
 629                           sizeof(struct link_tbl_entry);
 630                 options |= LDST_SGF;
 631         }
 632         append_seq_in_ptr(desc, src_dma, areq->cryptlen + authsize, options);
 633         append_seq_fifo_load(desc, areq->cryptlen, FIFOLD_CLASS_BOTH |
 634                              FIFOLD_TYPE_LASTBOTH |
 635                              (encrypt ? FIFOLD_TYPE_MSG1OUT2
 636                                       : FIFOLD_TYPE_MSG));
 637
 638         /* specify destination */
 639         if (areq->src == areq->dst) {
 640                 dst_dma = src_dma;
 641         } else {
 642                 sg_count = dma_map_sg(jrdev, areq->dst, edesc->dst_nents ? : 1,
 643                                       DMA_FROM_DEVICE);
 644                 if (!edesc->dst_nents) {
 645                         dst_dma = sg_dma_address(areq->dst);
 646                         options = 0;
 647                 } else {
 648                         sg_to_link_tbl(areq->dst, edesc->dst_nents,
 649                                        edesc->link_tbl + edesc->assoc_nents +
 650                                        edesc->src_nents, 0);
 651                         dst_dma = edesc->link_tbl_dma + (edesc->assoc_nents +
 652                                   edesc->src_nents) *
 653                                   sizeof(struct link_tbl_entry);
 654                         options = LDST_SGF;
 655                 }
 656         }
 657         append_seq_out_ptr(desc, dst_dma, areq->cryptlen + authsize, options);
 658         append_seq_fifo_store(desc, areq->cryptlen, FIFOST_TYPE_MESSAGE_DATA);
 659
 660         /* ICV */
 661         if (encrypt)
 662                 append_seq_store(desc, authsize, LDST_CLASS_2_CCB |
 663                                  LDST_SRCDST_BYTE_CONTEXT);
 664         else
 665                 append_seq_fifo_load(desc, authsize, FIFOLD_CLASS_CLASS2 |
 666                                      FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
 667
 668 #ifdef DEBUG
 669         debug("job_desc_len %d\n", desc_len(desc));
 670         print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
 671                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc) , 1);
 672         print_hex_dump(KERN_ERR, "jdlinkt@"xstr(__LINE__)": ",
 673                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->link_tbl,
 674                         edesc->link_tbl_bytes, 1);
 675 #endif
 676
 677         ret = caam_jr_enqueue(jrdev, desc, callback, areq);
 678         if (!ret)
 679                 ret = -EINPROGRESS;
 680         else {
 681                 ipsec_esp_unmap(jrdev, edesc, areq);
 682                 kfree(edesc);
 683         }
 684
 685         return ret;
 686 }
 687
 688 /*
 689  * derive number of elements in scatterlist
 690  */
 691 static int sg_count(struct scatterlist *sg_list, int nbytes, int *chained)
 692 {
 693         struct scatterlist *sg = sg_list;
 694         int sg_nents = 0;
 695
 696         *chained = 0;
 697         while (nbytes > 0) {
 698                 sg_nents++;
 699                 nbytes -= sg->length;
 700                 if (!sg_is_last(sg) && (sg + 1)->length == 0)
 701                         *chained = 1;
 702                 sg = scatterwalk_sg_next(sg);
 703         }
 704
 705         return sg_nents;
 706 }
 707
 708 /*
 709  * allocate and map the ipsec_esp extended descriptor
 710  */
 711 static struct ipsec_esp_edesc *ipsec_esp_edesc_alloc(struct aead_request *areq,
 712                                                      int desc_bytes)
 713 {
 714         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
 715         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 716         struct device *jrdev = ctx->jrdev;
 717         gfp_t flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
 718                       GFP_ATOMIC;
 719         int assoc_nents, src_nents, dst_nents = 0, chained, link_tbl_bytes;
 720         struct ipsec_esp_edesc *edesc;
 721
 722         assoc_nents = sg_count(areq->assoc, areq->assoclen, &chained);
 723         BUG_ON(chained);
 724         if (likely(assoc_nents == 1))
 725                 assoc_nents = 0;
 726
 727         src_nents = sg_count(areq->src, areq->cryptlen + ctx->authsize,
 728                              &chained);
 729         BUG_ON(chained);
 730         if (src_nents == 1)
 731                 src_nents = 0;
 732
 733         if (unlikely(areq->dst != areq->src)) {
 734                 dst_nents = sg_count(areq->dst, areq->cryptlen + ctx->authsize,
 735                                      &chained);
 736                 BUG_ON(chained);
 737                 if (dst_nents == 1)
 738                         dst_nents = 0;
 739         }
 740
 741         link_tbl_bytes = (assoc_nents + src_nents + dst_nents) *
 742                          sizeof(struct link_tbl_entry);
 743         debug("link_tbl_bytes %d\n", link_tbl_bytes);
 744
 745         /* allocate space for base edesc and hw desc commands, link tables */
 746         edesc = kmalloc(sizeof(struct ipsec_esp_edesc) + desc_bytes +
 747                         link_tbl_bytes, GFP_DMA | flags);
 748         if (!edesc) {
 749                 dev_err(jrdev, "could not allocate extended descriptor\n");
 750                 return ERR_PTR(-ENOMEM);
 751         }
 752
 753         edesc->assoc_nents = assoc_nents;
 754         edesc->src_nents = src_nents;
 755         edesc->dst_nents = dst_nents;
 756         edesc->link_tbl = (void *)edesc + sizeof(struct ipsec_esp_edesc) +
 757                           desc_bytes;
 758         edesc->link_tbl_dma = dma_map_single(jrdev, edesc->link_tbl,
 759                                              link_tbl_bytes, DMA_TO_DEVICE);
 760         edesc->link_tbl_bytes = link_tbl_bytes;
 761
 762         return edesc;
 763 }
 764
 765 static int aead_authenc_encrypt(struct aead_request *areq)
 766 {
 767         struct ipsec_esp_edesc *edesc;
 768         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
 769         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 770         struct device *jrdev = ctx->jrdev;
 771         int ivsize = crypto_aead_ivsize(aead);
 772         u32 *desc;
 773         dma_addr_t iv_dma;
 774
 775         /* allocate extended descriptor */
 776         edesc = ipsec_esp_edesc_alloc(areq, DESC_AEAD_ENCRYPT_TEXT_LEN *
 777                                       CAAM_CMD_SZ);
 778         if (IS_ERR(edesc))
 779                 return PTR_ERR(edesc);
 780
 781         desc = edesc->hw_desc;
 782
 783         /* insert shared descriptor pointer */
 784         init_job_desc_shared(desc, ctx->shared_desc_phys,
 785                              desc_len(ctx->sh_desc), HDR_SHARE_DEFER);
 786
 787         iv_dma = dma_map_single(jrdev, areq->iv, ivsize, DMA_TO_DEVICE);
 788         /* check dma error */
 789
 790         append_load(desc, iv_dma, ivsize,
 791                     LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT);
 792
 793         return ipsec_esp(edesc, areq, OP_ALG_ENCRYPT, ipsec_esp_encrypt_done);
 794 }
 795
 796 static int aead_authenc_decrypt(struct aead_request *req)
 797 {
 798         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 799         int ivsize = crypto_aead_ivsize(aead);
 800         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 801         struct device *jrdev = ctx->jrdev;
 802         struct ipsec_esp_edesc *edesc;
 803         u32 *desc;
 804         dma_addr_t iv_dma;
 805
 806         req->cryptlen -= ctx->authsize;
 807
 808         /* allocate extended descriptor */
 809         edesc = ipsec_esp_edesc_alloc(req, DESC_AEAD_DECRYPT_TEXT_LEN *
 810                                       CAAM_CMD_SZ);
 811         if (IS_ERR(edesc))
 812                 return PTR_ERR(edesc);
 813
 814         desc = edesc->hw_desc;
 815
 816         /* insert shared descriptor pointer */
 817         init_job_desc_shared(desc, ctx->shared_desc_phys,
 818                              desc_len(ctx->sh_desc), HDR_SHARE_DEFER);
 819
 820         iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE);
 821         /* check dma error */
 822
 823         append_load(desc, iv_dma, ivsize,
 824                     LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT);
 825
 826         return ipsec_esp(edesc, req, !OP_ALG_ENCRYPT, ipsec_esp_decrypt_done);
 827 }
 828
 829 static int aead_authenc_givencrypt(struct aead_givcrypt_request *req)
 830 {
 831         struct aead_request *areq = &req->areq;
 832         struct ipsec_esp_edesc *edesc;
 833         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
 834         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 835         struct device *jrdev = ctx->jrdev;
 836         int ivsize = crypto_aead_ivsize(aead);
 837         dma_addr_t iv_dma;
 838         u32 *desc;
 839
 840         iv_dma = dma_map_single(jrdev, req->giv, ivsize, DMA_FROM_DEVICE);
 841
 842         debug("%s: giv %p\n", __func__, req->giv);
 843
 844         /* allocate extended descriptor */
 845         edesc = ipsec_esp_edesc_alloc(areq, DESC_AEAD_GIVENCRYPT_TEXT_LEN *
 846                                       CAAM_CMD_SZ);
 847         if (IS_ERR(edesc))
 848                 return PTR_ERR(edesc);
 849
 850         desc = edesc->hw_desc;
 851
 852         /* insert shared descriptor pointer */
 853         init_job_desc_shared(desc, ctx->shared_desc_phys,
 854                              desc_len(ctx->sh_desc), HDR_SHARE_DEFER);
 855
 856         /*
 857          * LOAD IMM Info FIFO
 858          * to DECO, Last, Padding, Random, Message, 16 bytes
 859          */
 860         append_load_imm_u32(desc, NFIFOENTRY_DEST_DECO | NFIFOENTRY_LC1 |
 861                             NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DTYPE_MSG |
 862                             NFIFOENTRY_PTYPE_RND | ivsize,
 863                             LDST_SRCDST_WORD_INFO_FIFO);
 864
 865         /*
 866          * disable info fifo entries since the above serves as the entry
 867          * this way, the MOVE command won't generate an entry.
 868          * Note that this isn't required in more recent versions of
 869          * SEC as a MOVE that doesn't do info FIFO entries is available.
 870          */
 871         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
 872
 873         /* MOVE DECO Alignment -> C1 Context 16 bytes */
 874         append_move(desc, MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | ivsize);
 875
 876         /* re-enable info fifo entries */
 877         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
 878
 879         /* MOVE C1 Context -> OFIFO 16 bytes */
 880         append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | ivsize);
 881
 882         append_fifo_store(desc, iv_dma, ivsize, FIFOST_TYPE_MESSAGE_DATA);
 883
 884         return ipsec_esp(edesc, areq, OP_ALG_ENCRYPT, ipsec_esp_encrypt_done);
 885 }
 886
 887 struct caam_alg_template {
 888         char name[CRYPTO_MAX_ALG_NAME];
 889         char driver_name[CRYPTO_MAX_ALG_NAME];
 890         unsigned int blocksize;
 891         struct aead_alg aead;
 892         u32 class1_alg_type;
 893         u32 class2_alg_type;
 894         u32 alg_op;
 895 };
 896
 897 static struct caam_alg_template driver_algs[] = {
 898         /* single-pass ipsec_esp descriptor */
 899         {
 900                 .name = "authenc(hmac(sha1),cbc(aes))",
 901                 .driver_name = "authenc-hmac-sha1-cbc-aes-caam",
 902                 .blocksize = AES_BLOCK_SIZE,
 903                 .aead = {
 904                         .setkey = aead_authenc_setkey,
 905                         .setauthsize = aead_authenc_setauthsize,
 906                         .encrypt = aead_authenc_encrypt,
 907                         .decrypt = aead_authenc_decrypt,
 908                         .givencrypt = aead_authenc_givencrypt,
 909                         .geniv = "<built-in>",
 910                         .ivsize = AES_BLOCK_SIZE,
 911                         .maxauthsize = SHA1_DIGEST_SIZE,
 912                         },
 913                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
 914                 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
 915                 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
 916         },
 917         {
 918                 .name = "authenc(hmac(sha256),cbc(aes))",
 919                 .driver_name = "authenc-hmac-sha256-cbc-aes-caam",
 920                 .blocksize = AES_BLOCK_SIZE,
 921                 .aead = {
 922                         .setkey = aead_authenc_setkey,
 923                         .setauthsize = aead_authenc_setauthsize,
 924                         .encrypt = aead_authenc_encrypt,
 925                         .decrypt = aead_authenc_decrypt,
 926                         .givencrypt = aead_authenc_givencrypt,
 927                         .geniv = "<built-in>",
 928                         .ivsize = AES_BLOCK_SIZE,
 929                         .maxauthsize = SHA256_DIGEST_SIZE,
 930                         },
 931                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
 932                 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
 933                                    OP_ALG_AAI_HMAC_PRECOMP,
 934                 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
 935         },
 936         {
 937                 .name = "authenc(hmac(sha512),cbc(aes))",
 938                 .driver_name = "authenc-hmac-sha512-cbc-aes-caam",
 939                 .blocksize = AES_BLOCK_SIZE,
 940                 .aead = {
 941                         .setkey = aead_authenc_setkey,
 942                         .setauthsize = aead_authenc_setauthsize,
 943                         .encrypt = aead_authenc_encrypt,
 944                         .decrypt = aead_authenc_decrypt,
 945                         .givencrypt = aead_authenc_givencrypt,
 946                         .geniv = "<built-in>",
 947                         .ivsize = AES_BLOCK_SIZE,
 948                         .maxauthsize = SHA512_DIGEST_SIZE,
 949                         },
 950                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
 951                 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
 952                                    OP_ALG_AAI_HMAC_PRECOMP,
 953                 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
 954         },
 955         {
 956                 .name = "authenc(hmac(sha1),cbc(des3_ede))",
 957                 .driver_name = "authenc-hmac-sha1-cbc-des3_ede-caam",
 958                 .blocksize = DES3_EDE_BLOCK_SIZE,
 959                 .aead = {
 960                         .setkey = aead_authenc_setkey,
 961                         .setauthsize = aead_authenc_setauthsize,
 962                         .encrypt = aead_authenc_encrypt,
 963                         .decrypt = aead_authenc_decrypt,
 964                         .givencrypt = aead_authenc_givencrypt,
 965                         .geniv = "<built-in>",
 966                         .ivsize = DES3_EDE_BLOCK_SIZE,
 967                         .maxauthsize = SHA1_DIGEST_SIZE,
 968                         },
 969                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
 970                 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
 971                 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
 972         },
 973         {
 974                 .name = "authenc(hmac(sha256),cbc(des3_ede))",
 975                 .driver_name = "authenc-hmac-sha256-cbc-des3_ede-caam",
 976                 .blocksize = DES3_EDE_BLOCK_SIZE,
 977                 .aead = {
 978                         .setkey = aead_authenc_setkey,
 979                         .setauthsize = aead_authenc_setauthsize,
 980                         .encrypt = aead_authenc_encrypt,
 981                         .decrypt = aead_authenc_decrypt,
 982                         .givencrypt = aead_authenc_givencrypt,
 983                         .geniv = "<built-in>",
 984                         .ivsize = DES3_EDE_BLOCK_SIZE,
 985                         .maxauthsize = SHA256_DIGEST_SIZE,
 986                         },
 987                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
 988                 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
 989                                    OP_ALG_AAI_HMAC_PRECOMP,
 990                 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
 991         },
 992         {
 993                 .name = "authenc(hmac(sha512),cbc(des3_ede))",
 994                 .driver_name = "authenc-hmac-sha512-cbc-des3_ede-caam",
 995                 .blocksize = DES3_EDE_BLOCK_SIZE,
 996                 .aead = {
 997                         .setkey = aead_authenc_setkey,
 998                         .setauthsize = aead_authenc_setauthsize,
 999                         .encrypt = aead_authenc_encrypt,
1000                         .decrypt = aead_authenc_decrypt,
1001                         .givencrypt = aead_authenc_givencrypt,
1002                         .geniv = "<built-in>",
1003                         .ivsize = DES3_EDE_BLOCK_SIZE,
1004                         .maxauthsize = SHA512_DIGEST_SIZE,
1005                         },
1006                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1007                 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
1008                                    OP_ALG_AAI_HMAC_PRECOMP,
1009                 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
1010         },
1011         {
1012                 .name = "authenc(hmac(sha1),cbc(des))",
1013                 .driver_name = "authenc-hmac-sha1-cbc-des-caam",
1014                 .blocksize = DES_BLOCK_SIZE,
1015                 .aead = {
1016                         .setkey = aead_authenc_setkey,
1017                         .setauthsize = aead_authenc_setauthsize,
1018                         .encrypt = aead_authenc_encrypt,
1019                         .decrypt = aead_authenc_decrypt,
1020                         .givencrypt = aead_authenc_givencrypt,
1021                         .geniv = "<built-in>",
1022                         .ivsize = DES_BLOCK_SIZE,
1023                         .maxauthsize = SHA1_DIGEST_SIZE,
1024                         },
1025                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1026                 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
1027                 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
1028         },
1029         {
1030                 .name = "authenc(hmac(sha256),cbc(des))",
1031                 .driver_name = "authenc-hmac-sha256-cbc-des-caam",
1032                 .blocksize = DES_BLOCK_SIZE,
1033                 .aead = {
1034                         .setkey = aead_authenc_setkey,
1035                         .setauthsize = aead_authenc_setauthsize,
1036                         .encrypt = aead_authenc_encrypt,
1037                         .decrypt = aead_authenc_decrypt,
1038                         .givencrypt = aead_authenc_givencrypt,
1039                         .geniv = "<built-in>",
1040                         .ivsize = DES_BLOCK_SIZE,
1041                         .maxauthsize = SHA256_DIGEST_SIZE,
1042                         },
1043                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1044                 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1045                                    OP_ALG_AAI_HMAC_PRECOMP,
1046                 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
1047         },
1048         {
1049                 .name = "authenc(hmac(sha512),cbc(des))",
1050                 .driver_name = "authenc-hmac-sha512-cbc-des-caam",
1051                 .blocksize = DES_BLOCK_SIZE,
1052                 .aead = {
1053                         .setkey = aead_authenc_setkey,
1054                         .setauthsize = aead_authenc_setauthsize,
1055                         .encrypt = aead_authenc_encrypt,
1056                         .decrypt = aead_authenc_decrypt,
1057                         .givencrypt = aead_authenc_givencrypt,
1058                         .geniv = "<built-in>",
1059                         .ivsize = DES_BLOCK_SIZE,
1060                         .maxauthsize = SHA512_DIGEST_SIZE,
1061                         },
1062                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1063                 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
1064                                    OP_ALG_AAI_HMAC_PRECOMP,
1065                 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
1066         },
1067 };
1068
1069 struct caam_crypto_alg {
1070         struct list_head entry;
1071         struct device *ctrldev;
1072         int class1_alg_type;
1073         int class2_alg_type;
1074         int alg_op;
1075         struct crypto_alg crypto_alg;
1076 };
1077
1078 static int caam_cra_init(struct crypto_tfm *tfm)
1079 {
1080         struct crypto_alg *alg = tfm->__crt_alg;
1081         struct caam_crypto_alg *caam_alg =
1082                  container_of(alg, struct caam_crypto_alg, crypto_alg);
1083         struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
1084         struct caam_drv_private *priv = dev_get_drvdata(caam_alg->ctrldev);
1085         int tgt_jr = atomic_inc_return(&priv->tfm_count);
1086
1087         /*
1088          * distribute tfms across job rings to ensure in-order
1089          * crypto request processing per tfm
1090          */
1091         ctx->jrdev = priv->algapi_jr[(tgt_jr / 2) % priv->num_jrs_for_algapi];
1092
1093         /* copy descriptor header template value */
1094         ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam_alg->class1_alg_type;
1095         ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam_alg->class2_alg_type;
1096         ctx->alg_op = OP_TYPE_CLASS2_ALG | caam_alg->alg_op;
1097
1098         return 0;
1099 }
1100
1101 static void caam_cra_exit(struct crypto_tfm *tfm)
1102 {
1103         struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
1104
1105         if (!dma_mapping_error(ctx->jrdev, ctx->shared_desc_phys))
1106                 dma_unmap_single(ctx->jrdev, ctx->shared_desc_phys,
1107                                  desc_bytes(ctx->sh_desc), DMA_TO_DEVICE);
1108         kfree(ctx->sh_desc);
1109
1110         if (!dma_mapping_error(ctx->jrdev, ctx->key_phys))
1111                 dma_unmap_single(ctx->jrdev, ctx->key_phys,
1112                                  ctx->split_key_pad_len + ctx->enckeylen,
1113                                  DMA_TO_DEVICE);
1114         kfree(ctx->key);
1115 }
1116
1117 static void __exit caam_algapi_exit(void)
1118 {
1119
1120         struct device_node *dev_node;
1121         struct platform_device *pdev;
1122         struct device *ctrldev;
1123         struct caam_drv_private *priv;
1124         struct caam_crypto_alg *t_alg, *n;
1125         int i, err;
1126
1127         dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
1128         if (!dev_node)
1129                 return;
1130
1131         pdev = of_find_device_by_node(dev_node);
1132         if (!pdev)
1133                 return;
1134
1135         ctrldev = &pdev->dev;
1136         of_node_put(dev_node);
1137         priv = dev_get_drvdata(ctrldev);
1138
1139         if (!priv->alg_list.next)
1140                 return;
1141
1142         list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
1143                 crypto_unregister_alg(&t_alg->crypto_alg);
1144                 list_del(&t_alg->entry);
1145                 kfree(t_alg);
1146         }
1147
1148         for (i = 0; i < priv->total_jobrs; i++) {
1149                 err = caam_jr_deregister(priv->algapi_jr[i]);
1150                 if (err < 0)
1151                         break;
1152         }
1153         kfree(priv->algapi_jr);
1154 }
1155
1156 static struct caam_crypto_alg *caam_alg_alloc(struct device *ctrldev,
1157                                               struct caam_alg_template
1158                                               *template)
1159 {
1160         struct caam_crypto_alg *t_alg;
1161         struct crypto_alg *alg;
1162
1163         t_alg = kzalloc(sizeof(struct caam_crypto_alg), GFP_KERNEL);
1164         if (!t_alg) {
1165                 dev_err(ctrldev, "failed to allocate t_alg\n");
1166                 return ERR_PTR(-ENOMEM);
1167         }
1168
1169         alg = &t_alg->crypto_alg;
1170
1171         snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name);
1172         snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1173                  template->driver_name);
1174         alg->cra_module = THIS_MODULE;
1175         alg->cra_init = caam_cra_init;
1176         alg->cra_exit = caam_cra_exit;
1177         alg->cra_priority = CAAM_CRA_PRIORITY;
1178         alg->cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC;
1179         alg->cra_blocksize = template->blocksize;
1180         alg->cra_alignmask = 0;
1181         alg->cra_type = &crypto_aead_type;
1182         alg->cra_ctxsize = sizeof(struct caam_ctx);
1183         alg->cra_u.aead = template->aead;
1184
1185         t_alg->class1_alg_type = template->class1_alg_type;
1186         t_alg->class2_alg_type = template->class2_alg_type;
1187         t_alg->alg_op = template->alg_op;
1188         t_alg->ctrldev = ctrldev;
1189
1190         return t_alg;
1191 }
1192
1193 static int __init caam_algapi_init(void)
1194 {
1195         struct device_node *dev_node;
1196         struct platform_device *pdev;
1197         struct device *ctrldev, **jrdev;
1198         struct caam_drv_private *priv;
1199         int i = 0, err = 0;
1200
1201         dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
1202         if (!dev_node)
1203                 return -ENODEV;
1204
1205         pdev = of_find_device_by_node(dev_node);
1206         if (!pdev)
1207                 return -ENODEV;
1208
1209         ctrldev = &pdev->dev;
1210         priv = dev_get_drvdata(ctrldev);
1211         of_node_put(dev_node);
1212
1213         INIT_LIST_HEAD(&priv->alg_list);
1214
1215         jrdev = kmalloc(sizeof(*jrdev) * priv->total_jobrs, GFP_KERNEL);
1216         if (!jrdev)
1217                 return -ENOMEM;
1218
1219         for (i = 0; i < priv->total_jobrs; i++) {
1220                 err = caam_jr_register(ctrldev, &jrdev[i]);
1221                 if (err < 0)
1222                         break;
1223         }
1224         if (err < 0 && i == 0) {
1225                 dev_err(ctrldev, "algapi error in job ring registration: %d\n",
1226                         err);
1227                 kfree(jrdev);
1228                 return err;
1229         }
1230
1231         priv->num_jrs_for_algapi = i;
1232         priv->algapi_jr = jrdev;
1233         atomic_set(&priv->tfm_count, -1);
1234
1235         /* register crypto algorithms the device supports */
1236         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
1237                 /* TODO: check if h/w supports alg */
1238                 struct caam_crypto_alg *t_alg;
1239
1240                 t_alg = caam_alg_alloc(ctrldev, &driver_algs[i]);
1241                 if (IS_ERR(t_alg)) {
1242                         err = PTR_ERR(t_alg);
1243                         dev_warn(ctrldev, "%s alg allocation failed\n",
1244                                  driver_algs[i].driver_name);
1245                         continue;
1246                 }
1247
1248                 err = crypto_register_alg(&t_alg->crypto_alg);
1249                 if (err) {
1250                         dev_warn(ctrldev, "%s alg registration failed\n",
1251                                 t_alg->crypto_alg.cra_driver_name);
1252                         kfree(t_alg);
1253                 } else {
1254                         list_add_tail(&t_alg->entry, &priv->alg_list);
1255                         dev_info(ctrldev, "%s\n",
1256                                  t_alg->crypto_alg.cra_driver_name);
1257                 }
1258         }
1259
1260         return err;
1261 }
1262
1263 module_init(caam_algapi_init);
1264 module_exit(caam_algapi_exit);
1265
1266 MODULE_LICENSE("GPL");
1267 MODULE_DESCRIPTION("FSL CAAM support for crypto API");
1268 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");