drivers/crypto/caam/caamalg.c

   1 /*
   2  * caam - Freescale FSL CAAM support for crypto API
   3  *
   4  * Copyright 2008-2011 Freescale Semiconductor, Inc.
   5  *
   6  * Based on talitos crypto API driver.
   7  *
   8  * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008):
   9  *
  10  * ---------------                     ---------------
  11  * | JobDesc #1  |-------------------->|  ShareDesc  |
  12  * | *(packet 1) |                     |   (PDB)     |
  13  * ---------------      |------------->|  (hashKey)  |
  14  *       .              |              | (cipherKey) |
  15  *       .              |    |-------->| (operation) |
  16  * ---------------      |    |         ---------------
  17  * | JobDesc #2  |------|    |
  18  * | *(packet 2) |           |
  19  * ---------------           |
  20  *       .                   |
  21  *       .                   |
  22  * ---------------           |
  23  * | JobDesc #3  |------------
  24  * | *(packet 3) |
  25  * ---------------
  26  *
  27  * The SharedDesc never changes for a connection unless rekeyed, but
  28  * each packet will likely be in a different place. So all we need
  29  * to know to process the packet is where the input is, where the
  30  * output goes, and what context we want to process with. Context is
  31  * in the SharedDesc, packet references in the JobDesc.
  32  *
  33  * So, a job desc looks like:
  34  *
  35  * ---------------------
  36  * | Header            |
  37  * | ShareDesc Pointer |
  38  * | SEQ_OUT_PTR       |
  39  * | (output buffer)   |
  40  * | SEQ_IN_PTR        |
  41  * | (input buffer)    |
  42  * | LOAD (to DECO)    |
  43  * ---------------------
  44  */
  45
  46 #include "compat.h"
  47
  48 #include "regs.h"
  49 #include "intern.h"
  50 #include "desc_constr.h"
  51 #include "jr.h"
  52 #include "error.h"
  53
  54 /*
  55  * crypto alg
  56  */
  57 #define CAAM_CRA_PRIORITY               3000
  58 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
  59 #define CAAM_MAX_KEY_SIZE               (AES_MAX_KEY_SIZE + \
  60                                          SHA512_DIGEST_SIZE * 2)
  61 /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
  62 #define CAAM_MAX_IV_LENGTH              16
  63
  64 /* length of descriptors text */
  65 #define DESC_JOB_IO_LEN                 (CAAM_CMD_SZ * 3 + CAAM_PTR_SZ * 3)
  66
  67 #define DESC_AEAD_BASE                  (4 * CAAM_CMD_SZ)
  68 #define DESC_AEAD_ENC_LEN               (DESC_AEAD_BASE + 16 * CAAM_CMD_SZ)
  69 #define DESC_AEAD_DEC_LEN               (DESC_AEAD_BASE + 21 * CAAM_CMD_SZ)
  70 #define DESC_AEAD_GIVENC_LEN            (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ)
  71
  72 #define DESC_ABLKCIPHER_BASE            (3 * CAAM_CMD_SZ)
  73 #define DESC_ABLKCIPHER_ENC_LEN         (DESC_ABLKCIPHER_BASE + \
  74                                          20 * CAAM_CMD_SZ)
  75 #define DESC_ABLKCIPHER_DEC_LEN         (DESC_ABLKCIPHER_BASE + \
  76                                          15 * CAAM_CMD_SZ)
  77
  78 #define DESC_MAX_USED_BYTES             (DESC_AEAD_GIVENC_LEN + \
  79                                          CAAM_MAX_KEY_SIZE)
  80 #define DESC_MAX_USED_LEN               (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
  81
  82 #ifdef DEBUG
  83 /* for print_hex_dumps with line references */
  84 #define xstr(s) str(s)
  85 #define str(s) #s
  86 #define debug(format, arg...) printk(format, arg)
  87 #else
  88 #define debug(format, arg...)
  89 #endif
  90
  91 /* Set DK bit in class 1 operation if shared */
  92 static inline void append_dec_op1(u32 *desc, u32 type)
  93 {
  94         u32 *jump_cmd, *uncond_jump_cmd;
  95
  96         jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
  97         append_operation(desc, type | OP_ALG_AS_INITFINAL |
  98                          OP_ALG_DECRYPT);
  99         uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
 100         set_jump_tgt_here(desc, jump_cmd);
 101         append_operation(desc, type | OP_ALG_AS_INITFINAL |
 102                          OP_ALG_DECRYPT | OP_ALG_AAI_DK);
 103         set_jump_tgt_here(desc, uncond_jump_cmd);
 104 }
 105
 106 /*
 107  * Wait for completion of class 1 key loading before allowing
 108  * error propagation
 109  */
 110 static inline void append_dec_shr_done(u32 *desc)
 111 {
 112         u32 *jump_cmd;
 113
 114         jump_cmd = append_jump(desc, JUMP_CLASS_CLASS1 | JUMP_TEST_ALL);
 115         set_jump_tgt_here(desc, jump_cmd);
 116         append_cmd(desc, SET_OK_PROP_ERRORS | CMD_LOAD);
 117 }
 118
 119 /*
 120  * For aead functions, read payload and write payload,
 121  * both of which are specified in req->src and req->dst
 122  */
 123 static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
 124 {
 125         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
 126                              KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
 127         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
 128 }
 129
 130 /*
 131  * For aead encrypt and decrypt, read iv for both classes
 132  */
 133 static inline void aead_append_ld_iv(u32 *desc, int ivsize)
 134 {
 135         append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
 136                    LDST_CLASS_1_CCB | ivsize);
 137         append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | ivsize);
 138 }
 139
 140 /*
 141  * For ablkcipher encrypt and decrypt, read from req->src and
 142  * write to req->dst
 143  */
 144 static inline void ablkcipher_append_src_dst(u32 *desc)
 145 {
 146         append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); \
 147         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); \
 148         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | \
 149                              KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); \
 150         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); \
 151 }
 152
 153 /*
 154  * If all data, including src (with assoc and iv) or dst (with iv only) are
 155  * contiguous
 156  */
 157 #define GIV_SRC_CONTIG          1
 158 #define GIV_DST_CONTIG          (1 << 1)
 159
 160 /*
 161  * per-session context
 162  */
 163 struct caam_ctx {
 164         struct device *jrdev;
 165         u32 sh_desc_enc[DESC_MAX_USED_LEN];
 166         u32 sh_desc_dec[DESC_MAX_USED_LEN];
 167         u32 sh_desc_givenc[DESC_MAX_USED_LEN];
 168         dma_addr_t sh_desc_enc_dma;
 169         dma_addr_t sh_desc_dec_dma;
 170         dma_addr_t sh_desc_givenc_dma;
 171         u32 class1_alg_type;
 172         u32 class2_alg_type;
 173         u32 alg_op;
 174         u8 key[CAAM_MAX_KEY_SIZE];
 175         dma_addr_t key_dma;
 176         unsigned int enckeylen;
 177         unsigned int split_key_len;
 178         unsigned int split_key_pad_len;
 179         unsigned int authsize;
 180 };
 181
 182 static void append_key_aead(u32 *desc, struct caam_ctx *ctx,
 183                             int keys_fit_inline)
 184 {
 185         if (keys_fit_inline) {
 186                 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
 187                                   ctx->split_key_len, CLASS_2 |
 188                                   KEY_DEST_MDHA_SPLIT | KEY_ENC);
 189                 append_key_as_imm(desc, (void *)ctx->key +
 190                                   ctx->split_key_pad_len, ctx->enckeylen,
 191                                   ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
 192         } else {
 193                 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
 194                            KEY_DEST_MDHA_SPLIT | KEY_ENC);
 195                 append_key(desc, ctx->key_dma + ctx->split_key_pad_len,
 196                            ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
 197         }
 198 }
 199
 200 static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx,
 201                                   int keys_fit_inline)
 202 {
 203         u32 *key_jump_cmd;
 204
 205         init_sh_desc(desc, HDR_SHARE_WAIT);
 206
 207         /* Skip if already shared */
 208         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 209                                    JUMP_COND_SHRD);
 210
 211         append_key_aead(desc, ctx, keys_fit_inline);
 212
 213         set_jump_tgt_here(desc, key_jump_cmd);
 214
 215         /* Propagate errors from shared to job descriptor */
 216         append_cmd(desc, SET_OK_PROP_ERRORS | CMD_LOAD);
 217 }
 218
 219 static int aead_set_sh_desc(struct crypto_aead *aead)
 220 {
 221         struct aead_tfm *tfm = &aead->base.crt_aead;
 222         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 223         struct device *jrdev = ctx->jrdev;
 224         bool keys_fit_inline = 0;
 225         u32 *key_jump_cmd, *jump_cmd;
 226         u32 geniv, moveiv;
 227         u32 *desc;
 228
 229         if (!ctx->enckeylen || !ctx->authsize)
 230                 return 0;
 231
 232         /*
 233          * Job Descriptor and Shared Descriptors
 234          * must all fit into the 64-word Descriptor h/w Buffer
 235          */
 236         if (DESC_AEAD_ENC_LEN + DESC_JOB_IO_LEN +
 237             ctx->split_key_pad_len + ctx->enckeylen <=
 238             CAAM_DESC_BYTES_MAX)
 239                 keys_fit_inline = 1;
 240
 241         /* aead_encrypt shared descriptor */
 242         desc = ctx->sh_desc_enc;
 243
 244         init_sh_desc_key_aead(desc, ctx, keys_fit_inline);
 245
 246         /* Class 2 operation */
 247         append_operation(desc, ctx->class2_alg_type |
 248                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 249
 250         /* cryptlen = seqoutlen - authsize */
 251         append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
 252
 253         /* assoclen + cryptlen = seqinlen - ivsize */
 254         append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize);
 255
 256         /* assoclen + cryptlen = (assoclen + cryptlen) - cryptlen */
 257         append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ);
 258
 259         /* read assoc before reading payload */
 260         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 261                              KEY_VLF);
 262         aead_append_ld_iv(desc, tfm->ivsize);
 263
 264         /* Class 1 operation */
 265         append_operation(desc, ctx->class1_alg_type |
 266                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 267
 268         /* Read and write cryptlen bytes */
 269         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 270         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 271         aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
 272
 273         /* Write ICV */
 274         append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
 275                          LDST_SRCDST_BYTE_CONTEXT);
 276
 277         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 278                                               desc_bytes(desc),
 279                                               DMA_TO_DEVICE);
 280         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 281                 dev_err(jrdev, "unable to map shared descriptor\n");
 282                 return -ENOMEM;
 283         }
 284 #ifdef DEBUG
 285         print_hex_dump(KERN_ERR, "aead enc shdesc@"xstr(__LINE__)": ",
 286                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 287                        desc_bytes(desc), 1);
 288 #endif
 289
 290         /*
 291          * Job Descriptor and Shared Descriptors
 292          * must all fit into the 64-word Descriptor h/w Buffer
 293          */
 294         if (DESC_AEAD_DEC_LEN + DESC_JOB_IO_LEN +
 295             ctx->split_key_pad_len + ctx->enckeylen <=
 296             CAAM_DESC_BYTES_MAX)
 297                 keys_fit_inline = 1;
 298
 299         desc = ctx->sh_desc_dec;
 300
 301         /* aead_decrypt shared descriptor */
 302         init_sh_desc(desc, HDR_SHARE_WAIT);
 303
 304         /* Skip if already shared */
 305         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 306                                    JUMP_COND_SHRD);
 307
 308         append_key_aead(desc, ctx, keys_fit_inline);
 309
 310         /* Only propagate error immediately if shared */
 311         jump_cmd = append_jump(desc, JUMP_TEST_ALL);
 312         set_jump_tgt_here(desc, key_jump_cmd);
 313         append_cmd(desc, SET_OK_PROP_ERRORS | CMD_LOAD);
 314         set_jump_tgt_here(desc, jump_cmd);
 315
 316         /* Class 2 operation */
 317         append_operation(desc, ctx->class2_alg_type |
 318                          OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
 319
 320         /* assoclen + cryptlen = seqinlen - ivsize */
 321         append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM,
 322                                 ctx->authsize + tfm->ivsize)
 323         /* assoclen = (assoclen + cryptlen) - cryptlen */
 324         append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 325         append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ);
 326
 327         /* read assoc before reading payload */
 328         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 329                              KEY_VLF);
 330
 331         aead_append_ld_iv(desc, tfm->ivsize);
 332
 333         append_dec_op1(desc, ctx->class1_alg_type);
 334
 335         /* Read and write cryptlen bytes */
 336         append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
 337         append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
 338         aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
 339
 340         /* Load ICV */
 341         append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 |
 342                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
 343         append_dec_shr_done(desc);
 344
 345         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 346                                               desc_bytes(desc),
 347                                               DMA_TO_DEVICE);
 348         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
 349                 dev_err(jrdev, "unable to map shared descriptor\n");
 350                 return -ENOMEM;
 351         }
 352 #ifdef DEBUG
 353         print_hex_dump(KERN_ERR, "aead dec shdesc@"xstr(__LINE__)": ",
 354                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 355                        desc_bytes(desc), 1);
 356 #endif
 357
 358         /*
 359          * Job Descriptor and Shared Descriptors
 360          * must all fit into the 64-word Descriptor h/w Buffer
 361          */
 362         if (DESC_AEAD_GIVENC_LEN + DESC_JOB_IO_LEN +
 363             ctx->split_key_pad_len + ctx->enckeylen <=
 364             CAAM_DESC_BYTES_MAX)
 365                 keys_fit_inline = 1;
 366
 367         /* aead_givencrypt shared descriptor */
 368         desc = ctx->sh_desc_givenc;
 369
 370         init_sh_desc_key_aead(desc, ctx, keys_fit_inline);
 371
 372         /* Generate IV */
 373         geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
 374                 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
 375                 NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
 376         append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
 377                             LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
 378         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
 379         append_move(desc, MOVE_SRC_INFIFO |
 380                     MOVE_DEST_CLASS1CTX | (tfm->ivsize << MOVE_LEN_SHIFT));
 381         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
 382
 383         /* Copy IV to class 1 context */
 384         append_move(desc, MOVE_SRC_CLASS1CTX |
 385                     MOVE_DEST_OUTFIFO | (tfm->ivsize << MOVE_LEN_SHIFT));
 386
 387         /* Return to encryption */
 388         append_operation(desc, ctx->class2_alg_type |
 389                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 390
 391         /* ivsize + cryptlen = seqoutlen - authsize */
 392         append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
 393
 394         /* assoclen = seqinlen - (ivsize + cryptlen) */
 395         append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
 396
 397         /* read assoc before reading payload */
 398         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 399                              KEY_VLF);
 400
 401         /* Copy iv from class 1 ctx to class 2 fifo*/
 402         moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
 403                  NFIFOENTRY_DTYPE_MSG | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
 404         append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
 405                             LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
 406         append_load_imm_u32(desc, tfm->ivsize, LDST_CLASS_2_CCB |
 407                             LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
 408
 409         /* Class 1 operation */
 410         append_operation(desc, ctx->class1_alg_type |
 411                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 412
 413         /* Will write ivsize + cryptlen */
 414         append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 415
 416         /* Not need to reload iv */
 417         append_seq_fifo_load(desc, tfm->ivsize,
 418                              FIFOLD_CLASS_SKIP);
 419
 420         /* Will read cryptlen */
 421         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 422         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF |
 423                              FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH);
 424         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
 425
 426         /* Write ICV */
 427         append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
 428                          LDST_SRCDST_BYTE_CONTEXT);
 429
 430         ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc,
 431                                                  desc_bytes(desc),
 432                                                  DMA_TO_DEVICE);
 433         if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
 434                 dev_err(jrdev, "unable to map shared descriptor\n");
 435                 return -ENOMEM;
 436         }
 437 #ifdef DEBUG
 438         print_hex_dump(KERN_ERR, "aead givenc shdesc@"xstr(__LINE__)": ",
 439                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 440                        desc_bytes(desc), 1);
 441 #endif
 442
 443         return 0;
 444 }
 445
 446 static int aead_setauthsize(struct crypto_aead *authenc,
 447                                     unsigned int authsize)
 448 {
 449         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 450
 451         ctx->authsize = authsize;
 452         aead_set_sh_desc(authenc);
 453
 454         return 0;
 455 }
 456
 457 struct split_key_result {
 458         struct completion completion;
 459         int err;
 460 };
 461
 462 static void split_key_done(struct device *dev, u32 *desc, u32 err,
 463                            void *context)
 464 {
 465         struct split_key_result *res = context;
 466
 467 #ifdef DEBUG
 468         dev_err(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 469 #endif
 470
 471         if (err) {
 472                 char tmp[CAAM_ERROR_STR_MAX];
 473
 474                 dev_err(dev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
 475         }
 476
 477         res->err = err;
 478
 479         complete(&res->completion);
 480 }
 481
 482 /*
 483 get a split ipad/opad key
 484
 485 Split key generation-----------------------------------------------
 486
 487 [00] 0xb0810008    jobdesc: stidx=1 share=never len=8
 488 [01] 0x04000014        key: class2->keyreg len=20
 489                         @0xffe01000
 490 [03] 0x84410014  operation: cls2-op sha1 hmac init dec
 491 [04] 0x24940000     fifold: class2 msgdata-last2 len=0 imm
 492 [05] 0xa4000001       jump: class2 local all ->1 [06]
 493 [06] 0x64260028    fifostr: class2 mdsplit-jdk len=40
 494                         @0xffe04000
 495 */
 496 static u32 gen_split_key(struct caam_ctx *ctx, const u8 *key_in, u32 authkeylen)
 497 {
 498         struct device *jrdev = ctx->jrdev;
 499         u32 *desc;
 500         struct split_key_result result;
 501         dma_addr_t dma_addr_in, dma_addr_out;
 502         int ret = 0;
 503
 504         desc = kmalloc(CAAM_CMD_SZ * 6 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
 505
 506         init_job_desc(desc, 0);
 507
 508         dma_addr_in = dma_map_single(jrdev, (void *)key_in, authkeylen,
 509                                      DMA_TO_DEVICE);
 510         if (dma_mapping_error(jrdev, dma_addr_in)) {
 511                 dev_err(jrdev, "unable to map key input memory\n");
 512                 kfree(desc);
 513                 return -ENOMEM;
 514         }
 515         append_key(desc, dma_addr_in, authkeylen, CLASS_2 |
 516                        KEY_DEST_CLASS_REG);
 517
 518         /* Sets MDHA up into an HMAC-INIT */
 519         append_operation(desc, ctx->alg_op | OP_ALG_DECRYPT |
 520                              OP_ALG_AS_INIT);
 521
 522         /*
 523          * do a FIFO_LOAD of zero, this will trigger the internal key expansion
 524            into both pads inside MDHA
 525          */
 526         append_fifo_load_as_imm(desc, NULL, 0, LDST_CLASS_2_CCB |
 527                                 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST2);
 528
 529         /*
 530          * FIFO_STORE with the explicit split-key content store
 531          * (0x26 output type)
 532          */
 533         dma_addr_out = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len,
 534                                       DMA_FROM_DEVICE);
 535         if (dma_mapping_error(jrdev, dma_addr_out)) {
 536                 dev_err(jrdev, "unable to map key output memory\n");
 537                 kfree(desc);
 538                 return -ENOMEM;
 539         }
 540         append_fifo_store(desc, dma_addr_out, ctx->split_key_len,
 541                           LDST_CLASS_2_CCB | FIFOST_TYPE_SPLIT_KEK);
 542
 543 #ifdef DEBUG
 544         print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ",
 545                        DUMP_PREFIX_ADDRESS, 16, 4, key_in, authkeylen, 1);
 546         print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
 547                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 548 #endif
 549
 550         result.err = 0;
 551         init_completion(&result.completion);
 552
 553         ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result);
 554         if (!ret) {
 555                 /* in progress */
 556                 wait_for_completion_interruptible(&result.completion);
 557                 ret = result.err;
 558 #ifdef DEBUG
 559                 print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ",
 560                                DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 561                                ctx->split_key_pad_len, 1);
 562 #endif
 563         }
 564
 565         dma_unmap_single(jrdev, dma_addr_out, ctx->split_key_pad_len,
 566                          DMA_FROM_DEVICE);
 567         dma_unmap_single(jrdev, dma_addr_in, authkeylen, DMA_TO_DEVICE);
 568
 569         kfree(desc);
 570
 571         return ret;
 572 }
 573
 574 static int aead_setkey(struct crypto_aead *aead,
 575                                const u8 *key, unsigned int keylen)
 576 {
 577         /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
 578         static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
 579         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 580         struct device *jrdev = ctx->jrdev;
 581         struct rtattr *rta = (void *)key;
 582         struct crypto_authenc_key_param *param;
 583         unsigned int authkeylen;
 584         unsigned int enckeylen;
 585         int ret = 0;
 586
 587         param = RTA_DATA(rta);
 588         enckeylen = be32_to_cpu(param->enckeylen);
 589
 590         key += RTA_ALIGN(rta->rta_len);
 591         keylen -= RTA_ALIGN(rta->rta_len);
 592
 593         if (keylen < enckeylen)
 594                 goto badkey;
 595
 596         authkeylen = keylen - enckeylen;
 597
 598         if (keylen > CAAM_MAX_KEY_SIZE)
 599                 goto badkey;
 600
 601         /* Pick class 2 key length from algorithm submask */
 602         ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >>
 603                                       OP_ALG_ALGSEL_SHIFT] * 2;
 604         ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16);
 605
 606 #ifdef DEBUG
 607         printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n",
 608                keylen, enckeylen, authkeylen);
 609         printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
 610                ctx->split_key_len, ctx->split_key_pad_len);
 611         print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ",
 612                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 613 #endif
 614
 615         ret = gen_split_key(ctx, key, authkeylen);
 616         if (ret) {
 617                 goto badkey;
 618         }
 619
 620         /* postpend encryption key to auth split key */
 621         memcpy(ctx->key + ctx->split_key_pad_len, key + authkeylen, enckeylen);
 622
 623         ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len +
 624                                        enckeylen, DMA_TO_DEVICE);
 625         if (dma_mapping_error(jrdev, ctx->key_dma)) {
 626                 dev_err(jrdev, "unable to map key i/o memory\n");
 627                 return -ENOMEM;
 628         }
 629 #ifdef DEBUG
 630         print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ",
 631                        DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 632                        ctx->split_key_pad_len + enckeylen, 1);
 633 #endif
 634
 635         ctx->enckeylen = enckeylen;
 636
 637         ret = aead_set_sh_desc(aead);
 638         if (ret) {
 639                 dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len +
 640                                  enckeylen, DMA_TO_DEVICE);
 641         }
 642
 643         return ret;
 644 badkey:
 645         crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
 646         return -EINVAL;
 647 }
 648
 649 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 650                              const u8 *key, unsigned int keylen)
 651 {
 652         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
 653         struct ablkcipher_tfm *tfm = &ablkcipher->base.crt_ablkcipher;
 654         struct device *jrdev = ctx->jrdev;
 655         int ret = 0;
 656         u32 *key_jump_cmd, *jump_cmd;
 657         u32 *desc;
 658
 659 #ifdef DEBUG
 660         print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ",
 661                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 662 #endif
 663
 664         memcpy(ctx->key, key, keylen);
 665         ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen,
 666                                       DMA_TO_DEVICE);
 667         if (dma_mapping_error(jrdev, ctx->key_dma)) {
 668                 dev_err(jrdev, "unable to map key i/o memory\n");
 669                 return -ENOMEM;
 670         }
 671         ctx->enckeylen = keylen;
 672
 673         /* ablkcipher_encrypt shared descriptor */
 674         desc = ctx->sh_desc_enc;
 675         init_sh_desc(desc, HDR_SHARE_WAIT);
 676         /* Skip if already shared */
 677         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 678                                    JUMP_COND_SHRD);
 679
 680         /* Load class1 key only */
 681         append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
 682                           ctx->enckeylen, CLASS_1 |
 683                           KEY_DEST_CLASS_REG);
 684
 685         set_jump_tgt_here(desc, key_jump_cmd);
 686
 687         /* Propagate errors from shared to job descriptor */
 688         append_cmd(desc, SET_OK_PROP_ERRORS | CMD_LOAD);
 689
 690         /* Load iv */
 691         append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
 692                    LDST_CLASS_1_CCB | tfm->ivsize);
 693
 694         /* Load operation */
 695         append_operation(desc, ctx->class1_alg_type |
 696                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 697
 698         /* Perform operation */
 699         ablkcipher_append_src_dst(desc);
 700
 701         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 702                                               desc_bytes(desc),
 703                                               DMA_TO_DEVICE);
 704         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 705                 dev_err(jrdev, "unable to map shared descriptor\n");
 706                 return -ENOMEM;
 707         }
 708 #ifdef DEBUG
 709         print_hex_dump(KERN_ERR, "ablkcipher enc shdesc@"xstr(__LINE__)": ",
 710                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 711                        desc_bytes(desc), 1);
 712 #endif
 713         /* ablkcipher_decrypt shared descriptor */
 714         desc = ctx->sh_desc_dec;
 715
 716         init_sh_desc(desc, HDR_SHARE_WAIT);
 717         /* Skip if already shared */
 718         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 719                                    JUMP_COND_SHRD);
 720
 721         /* Load class1 key only */
 722         append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
 723                           ctx->enckeylen, CLASS_1 |
 724                           KEY_DEST_CLASS_REG);
 725
 726         /* For aead, only propagate error immediately if shared */
 727         jump_cmd = append_jump(desc, JUMP_TEST_ALL);
 728         set_jump_tgt_here(desc, key_jump_cmd);
 729         append_cmd(desc, SET_OK_PROP_ERRORS | CMD_LOAD);
 730         set_jump_tgt_here(desc, jump_cmd);
 731
 732         /* load IV */
 733         append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
 734                    LDST_CLASS_1_CCB | tfm->ivsize);
 735
 736         /* Choose operation */
 737         append_dec_op1(desc, ctx->class1_alg_type);
 738
 739         /* Perform operation */
 740         ablkcipher_append_src_dst(desc);
 741
 742         /* Wait for key to load before allowing propagating error */
 743         append_dec_shr_done(desc);
 744
 745         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 746                                               desc_bytes(desc),
 747                                               DMA_TO_DEVICE);
 748         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 749                 dev_err(jrdev, "unable to map shared descriptor\n");
 750                 return -ENOMEM;
 751         }
 752
 753 #ifdef DEBUG
 754         print_hex_dump(KERN_ERR, "ablkcipher dec shdesc@"xstr(__LINE__)": ",
 755                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 756                        desc_bytes(desc), 1);
 757 #endif
 758
 759         return ret;
 760 }
 761
 762 struct link_tbl_entry {
 763         u64 ptr;
 764         u32 len;
 765         u8 reserved;
 766         u8 buf_pool_id;
 767         u16 offset;
 768 };
 769
 770 /*
 771  * aead_edesc - s/w-extended aead descriptor
 772  * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist
 773  * @src_nents: number of segments in input scatterlist
 774  * @dst_nents: number of segments in output scatterlist
 775  * @iv_dma: dma address of iv for checking continuity and link table
 776  * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
 777  * @link_tbl_bytes: length of dma mapped link_tbl space
 778  * @link_tbl_dma: bus physical mapped address of h/w link table
 779  * @hw_desc: the h/w job descriptor followed by any referenced link tables
 780  */
 781 struct aead_edesc {
 782         int assoc_nents;
 783         int src_nents;
 784         int dst_nents;
 785         dma_addr_t iv_dma;
 786         int link_tbl_bytes;
 787         dma_addr_t link_tbl_dma;
 788         struct link_tbl_entry *link_tbl;
 789         u32 hw_desc[0];
 790 };
 791
 792 /*
 793  * ablkcipher_edesc - s/w-extended ablkcipher descriptor
 794  * @src_nents: number of segments in input scatterlist
 795  * @dst_nents: number of segments in output scatterlist
 796  * @iv_dma: dma address of iv for checking continuity and link table
 797  * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
 798  * @link_tbl_bytes: length of dma mapped link_tbl space
 799  * @link_tbl_dma: bus physical mapped address of h/w link table
 800  * @hw_desc: the h/w job descriptor followed by any referenced link tables
 801  */
 802 struct ablkcipher_edesc {
 803         int src_nents;
 804         int dst_nents;
 805         dma_addr_t iv_dma;
 806         int link_tbl_bytes;
 807         dma_addr_t link_tbl_dma;
 808         struct link_tbl_entry *link_tbl;
 809         u32 hw_desc[0];
 810 };
 811
 812 static void caam_unmap(struct device *dev, struct scatterlist *src,
 813                        struct scatterlist *dst, int src_nents, int dst_nents,
 814                        dma_addr_t iv_dma, int ivsize, dma_addr_t link_tbl_dma,
 815                        int link_tbl_bytes)
 816 {
 817         if (unlikely(dst != src)) {
 818                 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
 819                 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
 820         } else {
 821                 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
 822         }
 823
 824         if (iv_dma)
 825                 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
 826         if (link_tbl_bytes)
 827                 dma_unmap_single(dev, link_tbl_dma, link_tbl_bytes,
 828                                  DMA_TO_DEVICE);
 829 }
 830
 831 static void aead_unmap(struct device *dev,
 832                        struct aead_edesc *edesc,
 833                        struct aead_request *req)
 834 {
 835         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 836         int ivsize = crypto_aead_ivsize(aead);
 837
 838         dma_unmap_sg(dev, req->assoc, edesc->assoc_nents, DMA_TO_DEVICE);
 839
 840         caam_unmap(dev, req->src, req->dst,
 841                    edesc->src_nents, edesc->dst_nents,
 842                    edesc->iv_dma, ivsize, edesc->link_tbl_dma,
 843                    edesc->link_tbl_bytes);
 844 }
 845
 846 static void ablkcipher_unmap(struct device *dev,
 847                              struct ablkcipher_edesc *edesc,
 848                              struct ablkcipher_request *req)
 849 {
 850         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
 851         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
 852
 853         caam_unmap(dev, req->src, req->dst,
 854                    edesc->src_nents, edesc->dst_nents,
 855                    edesc->iv_dma, ivsize, edesc->link_tbl_dma,
 856                    edesc->link_tbl_bytes);
 857 }
 858
 859 static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
 860                                    void *context)
 861 {
 862         struct aead_request *req = context;
 863         struct aead_edesc *edesc;
 864 #ifdef DEBUG
 865         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 866         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 867         int ivsize = crypto_aead_ivsize(aead);
 868
 869         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 870 #endif
 871
 872         edesc = (struct aead_edesc *)((char *)desc -
 873                  offsetof(struct aead_edesc, hw_desc));
 874
 875         if (err) {
 876                 char tmp[CAAM_ERROR_STR_MAX];
 877
 878                 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
 879         }
 880
 881         aead_unmap(jrdev, edesc, req);
 882
 883 #ifdef DEBUG
 884         print_hex_dump(KERN_ERR, "assoc  @"xstr(__LINE__)": ",
 885                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
 886                        req->assoclen , 1);
 887         print_hex_dump(KERN_ERR, "dstiv  @"xstr(__LINE__)": ",
 888                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src) - ivsize,
 889                        edesc->src_nents ? 100 : ivsize, 1);
 890         print_hex_dump(KERN_ERR, "dst    @"xstr(__LINE__)": ",
 891                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
 892                        edesc->src_nents ? 100 : req->cryptlen +
 893                        ctx->authsize + 4, 1);
 894 #endif
 895
 896         kfree(edesc);
 897
 898         aead_request_complete(req, err);
 899 }
 900
 901 static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
 902                                    void *context)
 903 {
 904         struct aead_request *req = context;
 905         struct aead_edesc *edesc;
 906 #ifdef DEBUG
 907         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 908         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 909         int ivsize = crypto_aead_ivsize(aead);
 910
 911         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 912 #endif
 913
 914         edesc = (struct aead_edesc *)((char *)desc -
 915                  offsetof(struct aead_edesc, hw_desc));
 916
 917 #ifdef DEBUG
 918         print_hex_dump(KERN_ERR, "dstiv  @"xstr(__LINE__)": ",
 919                        DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
 920                        ivsize, 1);
 921         print_hex_dump(KERN_ERR, "dst    @"xstr(__LINE__)": ",
 922                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->dst),
 923                        req->cryptlen, 1);
 924 #endif
 925
 926         if (err) {
 927                 char tmp[CAAM_ERROR_STR_MAX];
 928
 929                 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
 930         }
 931
 932         aead_unmap(jrdev, edesc, req);
 933
 934         /*
 935          * verify hw auth check passed else return -EBADMSG
 936          */
 937         if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK)
 938                 err = -EBADMSG;
 939
 940 #ifdef DEBUG
 941         print_hex_dump(KERN_ERR, "iphdrout@"xstr(__LINE__)": ",
 942                        DUMP_PREFIX_ADDRESS, 16, 4,
 943                        ((char *)sg_virt(req->assoc) - sizeof(struct iphdr)),
 944                        sizeof(struct iphdr) + req->assoclen +
 945                        ((req->cryptlen > 1500) ? 1500 : req->cryptlen) +
 946                        ctx->authsize + 36, 1);
 947         if (!err && edesc->link_tbl_bytes) {
 948                 struct scatterlist *sg = sg_last(req->src, edesc->src_nents);
 949                 print_hex_dump(KERN_ERR, "sglastout@"xstr(__LINE__)": ",
 950                                DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(sg),
 951                         sg->length + ctx->authsize + 16, 1);
 952         }
 953 #endif
 954
 955         kfree(edesc);
 956
 957         aead_request_complete(req, err);
 958 }
 959
 960 static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
 961                                    void *context)
 962 {
 963         struct ablkcipher_request *req = context;
 964         struct ablkcipher_edesc *edesc;
 965 #ifdef DEBUG
 966         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
 967         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
 968
 969         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 970 #endif
 971
 972         edesc = (struct ablkcipher_edesc *)((char *)desc -
 973                  offsetof(struct ablkcipher_edesc, hw_desc));
 974
 975         if (err) {
 976                 char tmp[CAAM_ERROR_STR_MAX];
 977
 978                 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
 979         }
 980
 981 #ifdef DEBUG
 982         print_hex_dump(KERN_ERR, "dstiv  @"xstr(__LINE__)": ",
 983                        DUMP_PREFIX_ADDRESS, 16, 4, req->info,
 984                        edesc->src_nents > 1 ? 100 : ivsize, 1);
 985         print_hex_dump(KERN_ERR, "dst    @"xstr(__LINE__)": ",
 986                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
 987                        edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
 988 #endif
 989
 990         ablkcipher_unmap(jrdev, edesc, req);
 991         kfree(edesc);
 992
 993         ablkcipher_request_complete(req, err);
 994 }
 995
 996 static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
 997                                     void *context)
 998 {
 999         struct ablkcipher_request *req = context;
1000         struct ablkcipher_edesc *edesc;
1001 #ifdef DEBUG
1002         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1003         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1004
1005         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
1006 #endif
1007
1008         edesc = (struct ablkcipher_edesc *)((char *)desc -
1009                  offsetof(struct ablkcipher_edesc, hw_desc));
1010         if (err) {
1011                 char tmp[CAAM_ERROR_STR_MAX];
1012
1013                 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
1014         }
1015
1016 #ifdef DEBUG
1017         print_hex_dump(KERN_ERR, "dstiv  @"xstr(__LINE__)": ",
1018                        DUMP_PREFIX_ADDRESS, 16, 4, req->info,
1019                        ivsize, 1);
1020         print_hex_dump(KERN_ERR, "dst    @"xstr(__LINE__)": ",
1021                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1022                        edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
1023 #endif
1024
1025         ablkcipher_unmap(jrdev, edesc, req);
1026         kfree(edesc);
1027
1028         ablkcipher_request_complete(req, err);
1029 }
1030
1031 static void sg_to_link_tbl_one(struct link_tbl_entry *link_tbl_ptr,
1032                                dma_addr_t dma, u32 len, u32 offset)
1033 {
1034         link_tbl_ptr->ptr = dma;
1035         link_tbl_ptr->len = len;
1036         link_tbl_ptr->reserved = 0;
1037         link_tbl_ptr->buf_pool_id = 0;
1038         link_tbl_ptr->offset = offset;
1039 #ifdef DEBUG
1040         print_hex_dump(KERN_ERR, "link_tbl_ptr@"xstr(__LINE__)": ",
1041                        DUMP_PREFIX_ADDRESS, 16, 4, link_tbl_ptr,
1042                        sizeof(struct link_tbl_entry), 1);
1043 #endif
1044 }
1045
1046 /*
1047  * convert scatterlist to h/w link table format
1048  * but does not have final bit; instead, returns last entry
1049  */
1050 static struct link_tbl_entry *sg_to_link_tbl(struct scatterlist *sg,
1051                                              int sg_count, struct link_tbl_entry
1052                                              *link_tbl_ptr, u32 offset)
1053 {
1054         while (sg_count) {
1055                 sg_to_link_tbl_one(link_tbl_ptr, sg_dma_address(sg),
1056                                    sg_dma_len(sg), offset);
1057                 link_tbl_ptr++;
1058                 sg = sg_next(sg);
1059                 sg_count--;
1060         }
1061         return link_tbl_ptr - 1;
1062 }
1063
1064 /*
1065  * convert scatterlist to h/w link table format
1066  * scatterlist must have been previously dma mapped
1067  */
1068 static void sg_to_link_tbl_last(struct scatterlist *sg, int sg_count,
1069                                 struct link_tbl_entry *link_tbl_ptr, u32 offset)
1070 {
1071         link_tbl_ptr = sg_to_link_tbl(sg, sg_count, link_tbl_ptr, offset);
1072         link_tbl_ptr->len |= 0x40000000;
1073 }
1074
1075 /*
1076  * Fill in aead job descriptor
1077  */
1078 static void init_aead_job(u32 *sh_desc, dma_addr_t ptr,
1079                           struct aead_edesc *edesc,
1080                           struct aead_request *req,
1081                           bool all_contig, bool encrypt)
1082 {
1083         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1084         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1085         int ivsize = crypto_aead_ivsize(aead);
1086         int authsize = ctx->authsize;
1087         u32 *desc = edesc->hw_desc;
1088         u32 out_options = 0, in_options;
1089         dma_addr_t dst_dma, src_dma;
1090         int len, link_tbl_index = 0;
1091
1092 #ifdef DEBUG
1093         debug("assoclen %d cryptlen %d authsize %d\n",
1094               req->assoclen, req->cryptlen, authsize);
1095         print_hex_dump(KERN_ERR, "assoc  @"xstr(__LINE__)": ",
1096                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
1097                        req->assoclen , 1);
1098         print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ",
1099                        DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1100                        edesc->src_nents ? 100 : ivsize, 1);
1101         print_hex_dump(KERN_ERR, "src    @"xstr(__LINE__)": ",
1102                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1103                         edesc->src_nents ? 100 : req->cryptlen, 1);
1104         print_hex_dump(KERN_ERR, "shrdesc@"xstr(__LINE__)": ",
1105                        DUMP_PREFIX_ADDRESS, 16, 4, sh_desc,
1106                        desc_bytes(sh_desc), 1);
1107 #endif
1108
1109         len = desc_len(sh_desc);
1110         init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1111
1112         if (all_contig) {
1113                 src_dma = sg_dma_address(req->assoc);
1114                 in_options = 0;
1115         } else {
1116                 src_dma = edesc->link_tbl_dma;
1117                 link_tbl_index += (edesc->assoc_nents ? : 1) + 1 +
1118                                   (edesc->src_nents ? : 1);
1119                 in_options = LDST_SGF;
1120         }
1121         if (encrypt)
1122                 append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize +
1123                                   req->cryptlen - authsize, in_options);
1124         else
1125                 append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize +
1126                                   req->cryptlen, in_options);
1127
1128         if (likely(req->src == req->dst)) {
1129                 if (all_contig) {
1130                         dst_dma = sg_dma_address(req->src);
1131                 } else {
1132                         dst_dma = src_dma + sizeof(struct link_tbl_entry) *
1133                                   ((edesc->assoc_nents ? : 1) + 1);
1134                         out_options = LDST_SGF;
1135                 }
1136         } else {
1137                 if (!edesc->dst_nents) {
1138                         dst_dma = sg_dma_address(req->dst);
1139                 } else {
1140                         dst_dma = edesc->link_tbl_dma +
1141                                   link_tbl_index *
1142                                   sizeof(struct link_tbl_entry);
1143                         out_options = LDST_SGF;
1144                 }
1145         }
1146         if (encrypt)
1147                 append_seq_out_ptr(desc, dst_dma, req->cryptlen, out_options);
1148         else
1149                 append_seq_out_ptr(desc, dst_dma, req->cryptlen - authsize,
1150                                    out_options);
1151 }
1152
1153 /*
1154  * Fill in aead givencrypt job descriptor
1155  */
1156 static void init_aead_giv_job(u32 *sh_desc, dma_addr_t ptr,
1157                               struct aead_edesc *edesc,
1158                               struct aead_request *req,
1159                               int contig)
1160 {
1161         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1162         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1163         int ivsize = crypto_aead_ivsize(aead);
1164         int authsize = ctx->authsize;
1165         u32 *desc = edesc->hw_desc;
1166         u32 out_options = 0, in_options;
1167         dma_addr_t dst_dma, src_dma;
1168         int len, link_tbl_index = 0;
1169
1170 #ifdef DEBUG
1171         debug("assoclen %d cryptlen %d authsize %d\n",
1172               req->assoclen, req->cryptlen, authsize);
1173         print_hex_dump(KERN_ERR, "assoc  @"xstr(__LINE__)": ",
1174                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
1175                        req->assoclen , 1);
1176         print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ",
1177                        DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
1178         print_hex_dump(KERN_ERR, "src    @"xstr(__LINE__)": ",
1179                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1180                         edesc->src_nents > 1 ? 100 : req->cryptlen, 1);
1181         print_hex_dump(KERN_ERR, "shrdesc@"xstr(__LINE__)": ",
1182                        DUMP_PREFIX_ADDRESS, 16, 4, sh_desc,
1183                        desc_bytes(sh_desc), 1);
1184 #endif
1185
1186         len = desc_len(sh_desc);
1187         init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1188
1189         if (contig & GIV_SRC_CONTIG) {
1190                 src_dma = sg_dma_address(req->assoc);
1191                 in_options = 0;
1192         } else {
1193                 src_dma = edesc->link_tbl_dma;
1194                 link_tbl_index += edesc->assoc_nents + 1 + edesc->src_nents;
1195                 in_options = LDST_SGF;
1196         }
1197         append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize +
1198                           req->cryptlen - authsize, in_options);
1199
1200         if (contig & GIV_DST_CONTIG) {
1201                 dst_dma = edesc->iv_dma;
1202         } else {
1203                 if (likely(req->src == req->dst)) {
1204                         dst_dma = src_dma + sizeof(struct link_tbl_entry) *
1205                                   edesc->assoc_nents;
1206                         out_options = LDST_SGF;
1207                 } else {
1208                         dst_dma = edesc->link_tbl_dma +
1209                                   link_tbl_index *
1210                                   sizeof(struct link_tbl_entry);
1211                         out_options = LDST_SGF;
1212                 }
1213         }
1214
1215         append_seq_out_ptr(desc, dst_dma, ivsize + req->cryptlen, out_options);
1216 }
1217
1218 /*
1219  * Fill in ablkcipher job descriptor
1220  */
1221 static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
1222                                 struct ablkcipher_edesc *edesc,
1223                                 struct ablkcipher_request *req,
1224                                 bool iv_contig)
1225 {
1226         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1227         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1228         u32 *desc = edesc->hw_desc;
1229         u32 out_options = 0, in_options;
1230         dma_addr_t dst_dma, src_dma;
1231         int len, link_tbl_index = 0;
1232
1233 #ifdef DEBUG
1234         print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ",
1235                        DUMP_PREFIX_ADDRESS, 16, 4, req->info,
1236                        ivsize, 1);
1237         print_hex_dump(KERN_ERR, "src    @"xstr(__LINE__)": ",
1238                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1239                        edesc->src_nents ? 100 : req->nbytes, 1);
1240 #endif
1241
1242         len = desc_len(sh_desc);
1243         init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1244
1245         if (iv_contig) {
1246                 src_dma = edesc->iv_dma;
1247                 in_options = 0;
1248         } else {
1249                 src_dma = edesc->link_tbl_dma;
1250                 link_tbl_index += (iv_contig ? 0 : 1) + edesc->src_nents;
1251                 in_options = LDST_SGF;
1252         }
1253         append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options);
1254
1255         if (likely(req->src == req->dst)) {
1256                 if (!edesc->src_nents && iv_contig) {
1257                         dst_dma = sg_dma_address(req->src);
1258                 } else {
1259                         dst_dma = edesc->link_tbl_dma +
1260                                 sizeof(struct link_tbl_entry);
1261                         out_options = LDST_SGF;
1262                 }
1263         } else {
1264                 if (!edesc->dst_nents) {
1265                         dst_dma = sg_dma_address(req->dst);
1266                 } else {
1267                         dst_dma = edesc->link_tbl_dma +
1268                                 link_tbl_index * sizeof(struct link_tbl_entry);
1269                         out_options = LDST_SGF;
1270                 }
1271         }
1272         append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options);
1273 }
1274
1275 /*
1276  * derive number of elements in scatterlist
1277  */
1278 static int sg_count(struct scatterlist *sg_list, int nbytes)
1279 {
1280         struct scatterlist *sg = sg_list;
1281         int sg_nents = 0;
1282
1283         while (nbytes > 0) {
1284                 sg_nents++;
1285                 nbytes -= sg->length;
1286                 if (!sg_is_last(sg) && (sg + 1)->length == 0)
1287                         BUG(); /* Not support chaining */
1288                 sg = scatterwalk_sg_next(sg);
1289         }
1290
1291         if (likely(sg_nents == 1))
1292                 return 0;
1293
1294         return sg_nents;
1295 }
1296
1297 /*
1298  * allocate and map the aead extended descriptor
1299  */
1300 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
1301                                            int desc_bytes, bool *all_contig_ptr)
1302 {
1303         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1304         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1305         struct device *jrdev = ctx->jrdev;
1306         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1307                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1308         int assoc_nents, src_nents, dst_nents = 0;
1309         struct aead_edesc *edesc;
1310         dma_addr_t iv_dma = 0;
1311         int sgc;
1312         bool all_contig = true;
1313         int ivsize = crypto_aead_ivsize(aead);
1314         int link_tbl_index, link_tbl_len = 0, link_tbl_bytes;
1315
1316         assoc_nents = sg_count(req->assoc, req->assoclen);
1317         src_nents = sg_count(req->src, req->cryptlen);
1318
1319         if (unlikely(req->dst != req->src))
1320                 dst_nents = sg_count(req->dst, req->cryptlen);
1321
1322         sgc = dma_map_sg(jrdev, req->assoc, assoc_nents ? : 1,
1323                          DMA_BIDIRECTIONAL);
1324         if (likely(req->src == req->dst)) {
1325                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
1326                                  DMA_BIDIRECTIONAL);
1327         } else {
1328                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
1329                                  DMA_TO_DEVICE);
1330                 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
1331                                  DMA_FROM_DEVICE);
1332         }
1333
1334         /* Check if data are contiguous */
1335         iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE);
1336         if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen !=
1337             iv_dma || src_nents || iv_dma + ivsize !=
1338             sg_dma_address(req->src)) {
1339                 all_contig = false;
1340                 assoc_nents = assoc_nents ? : 1;
1341                 src_nents = src_nents ? : 1;
1342                 link_tbl_len = assoc_nents + 1 + src_nents;
1343         }
1344         link_tbl_len += dst_nents;
1345
1346         link_tbl_bytes = link_tbl_len * sizeof(struct link_tbl_entry);
1347
1348         /* allocate space for base edesc and hw desc commands, link tables */
1349         edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes +
1350                         link_tbl_bytes, GFP_DMA | flags);
1351         if (!edesc) {
1352                 dev_err(jrdev, "could not allocate extended descriptor\n");
1353                 return ERR_PTR(-ENOMEM);
1354         }
1355
1356         edesc->assoc_nents = assoc_nents;
1357         edesc->src_nents = src_nents;
1358         edesc->dst_nents = dst_nents;
1359         edesc->iv_dma = iv_dma;
1360         edesc->link_tbl_bytes = link_tbl_bytes;
1361         edesc->link_tbl = (void *)edesc + sizeof(struct aead_edesc) +
1362                           desc_bytes;
1363         edesc->link_tbl_dma = dma_map_single(jrdev, edesc->link_tbl,
1364                                              link_tbl_bytes, DMA_TO_DEVICE);
1365         *all_contig_ptr = all_contig;
1366
1367         link_tbl_index = 0;
1368         if (!all_contig) {
1369                 sg_to_link_tbl(req->assoc,
1370                                (assoc_nents ? : 1),
1371                                edesc->link_tbl +
1372                                link_tbl_index, 0);
1373                 link_tbl_index += assoc_nents ? : 1;
1374                 sg_to_link_tbl_one(edesc->link_tbl + link_tbl_index,
1375                                    iv_dma, ivsize, 0);
1376                 link_tbl_index += 1;
1377                 sg_to_link_tbl_last(req->src,
1378                                     (src_nents ? : 1),
1379                                     edesc->link_tbl +
1380                                     link_tbl_index, 0);
1381                 link_tbl_index += src_nents ? : 1;
1382         }
1383         if (dst_nents) {
1384                 sg_to_link_tbl_last(req->dst, dst_nents,
1385                                     edesc->link_tbl + link_tbl_index, 0);
1386         }
1387
1388         return edesc;
1389 }
1390
1391 static int aead_encrypt(struct aead_request *req)
1392 {
1393         struct aead_edesc *edesc;
1394         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1395         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1396         struct device *jrdev = ctx->jrdev;
1397         bool all_contig;
1398         u32 *desc;
1399         int ret = 0;
1400
1401         req->cryptlen += ctx->authsize;
1402
1403         /* allocate extended descriptor */
1404         edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN *
1405                                  CAAM_CMD_SZ, &all_contig);
1406         if (IS_ERR(edesc))
1407                 return PTR_ERR(edesc);
1408
1409         /* Create and submit job descriptor */
1410         init_aead_job(ctx->sh_desc_enc, ctx->sh_desc_enc_dma, edesc, req,
1411                       all_contig, true);
1412 #ifdef DEBUG
1413         print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ",
1414                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1415                        desc_bytes(edesc->hw_desc), 1);
1416 #endif
1417
1418         desc = edesc->hw_desc;
1419         ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
1420         if (!ret) {
1421                 ret = -EINPROGRESS;
1422         } else {
1423                 aead_unmap(jrdev, edesc, req);
1424                 kfree(edesc);
1425         }
1426
1427         return ret;
1428 }
1429
1430 static int aead_decrypt(struct aead_request *req)
1431 {
1432         struct aead_edesc *edesc;
1433         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1434         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1435         struct device *jrdev = ctx->jrdev;
1436         bool all_contig;
1437         u32 *desc;
1438         int ret = 0;
1439
1440         /* allocate extended descriptor */
1441         edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN *
1442                                  CAAM_CMD_SZ, &all_contig);
1443         if (IS_ERR(edesc))
1444                 return PTR_ERR(edesc);
1445
1446 #ifdef DEBUG
1447         print_hex_dump(KERN_ERR, "dec src@"xstr(__LINE__)": ",
1448                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1449                        req->cryptlen, 1);
1450 #endif
1451
1452         /* Create and submit job descriptor*/
1453         init_aead_job(ctx->sh_desc_dec,
1454                       ctx->sh_desc_dec_dma, edesc, req, all_contig, false);
1455 #ifdef DEBUG
1456         print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ",
1457                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1458                        desc_bytes(edesc->hw_desc), 1);
1459 #endif
1460
1461         desc = edesc->hw_desc;
1462         ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
1463         if (!ret) {
1464                 ret = -EINPROGRESS;
1465         } else {
1466                 aead_unmap(jrdev, edesc, req);
1467                 kfree(edesc);
1468         }
1469
1470         return ret;
1471 }
1472
1473 /*
1474  * allocate and map the aead extended descriptor for aead givencrypt
1475  */
1476 static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request
1477                                                *greq, int desc_bytes,
1478                                                u32 *contig_ptr)
1479 {
1480         struct aead_request *req = &greq->areq;
1481         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1482         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1483         struct device *jrdev = ctx->jrdev;
1484         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1485                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1486         int assoc_nents, src_nents, dst_nents = 0;
1487         struct aead_edesc *edesc;
1488         dma_addr_t iv_dma = 0;
1489         int sgc;
1490         u32 contig = GIV_SRC_CONTIG | GIV_DST_CONTIG;
1491         int ivsize = crypto_aead_ivsize(aead);
1492         int link_tbl_index, link_tbl_len = 0, link_tbl_bytes;
1493
1494         assoc_nents = sg_count(req->assoc, req->assoclen);
1495         src_nents = sg_count(req->src, req->cryptlen);
1496
1497         if (unlikely(req->dst != req->src))
1498                 dst_nents = sg_count(req->dst, req->cryptlen);
1499
1500         sgc = dma_map_sg(jrdev, req->assoc, assoc_nents ? : 1,
1501                          DMA_BIDIRECTIONAL);
1502         if (likely(req->src == req->dst)) {
1503                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
1504                                  DMA_BIDIRECTIONAL);
1505         } else {
1506                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
1507                                  DMA_TO_DEVICE);
1508                 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
1509                                  DMA_FROM_DEVICE);
1510         }
1511
1512         /* Check if data are contiguous */
1513         iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE);
1514         if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen !=
1515             iv_dma || src_nents || iv_dma + ivsize != sg_dma_address(req->src))
1516                 contig &= ~GIV_SRC_CONTIG;
1517         if (dst_nents || iv_dma + ivsize != sg_dma_address(req->dst))
1518                 contig &= ~GIV_DST_CONTIG;
1519                 if (unlikely(req->src != req->dst)) {
1520                         dst_nents = dst_nents ? : 1;
1521                         link_tbl_len += 1;
1522                 }
1523         if (!(contig & GIV_SRC_CONTIG)) {
1524                 assoc_nents = assoc_nents ? : 1;
1525                 src_nents = src_nents ? : 1;
1526                 link_tbl_len += assoc_nents + 1 + src_nents;
1527                 if (likely(req->src == req->dst))
1528                         contig &= ~GIV_DST_CONTIG;
1529         }
1530         link_tbl_len += dst_nents;
1531
1532         link_tbl_bytes = link_tbl_len * sizeof(struct link_tbl_entry);
1533
1534         /* allocate space for base edesc and hw desc commands, link tables */
1535         edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes +
1536                         link_tbl_bytes, GFP_DMA | flags);
1537         if (!edesc) {
1538                 dev_err(jrdev, "could not allocate extended descriptor\n");
1539                 return ERR_PTR(-ENOMEM);
1540         }
1541
1542         edesc->assoc_nents = assoc_nents;
1543         edesc->src_nents = src_nents;
1544         edesc->dst_nents = dst_nents;
1545         edesc->iv_dma = iv_dma;
1546         edesc->link_tbl_bytes = link_tbl_bytes;
1547         edesc->link_tbl = (void *)edesc + sizeof(struct aead_edesc) +
1548                           desc_bytes;
1549         edesc->link_tbl_dma = dma_map_single(jrdev, edesc->link_tbl,
1550                                              link_tbl_bytes, DMA_TO_DEVICE);
1551         *contig_ptr = contig;
1552
1553         link_tbl_index = 0;
1554         if (!(contig & GIV_SRC_CONTIG)) {
1555                 sg_to_link_tbl(req->assoc, assoc_nents,
1556                                edesc->link_tbl +
1557                                link_tbl_index, 0);
1558                 link_tbl_index += assoc_nents;
1559                 sg_to_link_tbl_one(edesc->link_tbl + link_tbl_index,
1560                                    iv_dma, ivsize, 0);
1561                 link_tbl_index += 1;
1562                 sg_to_link_tbl_last(req->src, src_nents,
1563                                     edesc->link_tbl +
1564                                     link_tbl_index, 0);
1565                 link_tbl_index += src_nents;
1566         }
1567         if (unlikely(req->src != req->dst && !(contig & GIV_DST_CONTIG))) {
1568                 sg_to_link_tbl_one(edesc->link_tbl + link_tbl_index,
1569                                    iv_dma, ivsize, 0);
1570                 link_tbl_index += 1;
1571                 sg_to_link_tbl_last(req->dst, dst_nents,
1572                                     edesc->link_tbl + link_tbl_index, 0);
1573         }
1574
1575         return edesc;
1576 }
1577
1578 static int aead_givencrypt(struct aead_givcrypt_request *areq)
1579 {
1580         struct aead_request *req = &areq->areq;
1581         struct aead_edesc *edesc;
1582         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1583         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1584         struct device *jrdev = ctx->jrdev;
1585         u32 contig;
1586         u32 *desc;
1587         int ret = 0;
1588
1589         req->cryptlen += ctx->authsize;
1590
1591         /* allocate extended descriptor */
1592         edesc = aead_giv_edesc_alloc(areq, DESC_JOB_IO_LEN *
1593                                      CAAM_CMD_SZ, &contig);
1594
1595         if (IS_ERR(edesc))
1596                 return PTR_ERR(edesc);
1597
1598 #ifdef DEBUG
1599         print_hex_dump(KERN_ERR, "giv src@"xstr(__LINE__)": ",
1600                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1601                        req->cryptlen, 1);
1602 #endif
1603
1604         /* Create and submit job descriptor*/
1605         init_aead_giv_job(ctx->sh_desc_givenc,
1606                           ctx->sh_desc_givenc_dma, edesc, req, contig);
1607 #ifdef DEBUG
1608         print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ",
1609                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1610                        desc_bytes(edesc->hw_desc), 1);
1611 #endif
1612
1613         desc = edesc->hw_desc;
1614         ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
1615         if (!ret) {
1616                 ret = -EINPROGRESS;
1617         } else {
1618                 aead_unmap(jrdev, edesc, req);
1619                 kfree(edesc);
1620         }
1621
1622         return ret;
1623 }
1624
1625 /*
1626  * allocate and map the ablkcipher extended descriptor for ablkcipher
1627  */
1628 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
1629                                                        *req, int desc_bytes,
1630                                                        bool *iv_contig_out)
1631 {
1632         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1633         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1634         struct device *jrdev = ctx->jrdev;
1635         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1636                                           CRYPTO_TFM_REQ_MAY_SLEEP)) ?
1637                        GFP_KERNEL : GFP_ATOMIC;
1638         int src_nents, dst_nents = 0, link_tbl_bytes;
1639         struct ablkcipher_edesc *edesc;
1640         dma_addr_t iv_dma = 0;
1641         bool iv_contig = false;
1642         int sgc;
1643         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1644         int link_tbl_index;
1645
1646         src_nents = sg_count(req->src, req->nbytes);
1647
1648         if (unlikely(req->dst != req->src))
1649                 dst_nents = sg_count(req->dst, req->nbytes);
1650
1651         if (likely(req->src == req->dst)) {
1652                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
1653                                  DMA_BIDIRECTIONAL);
1654         } else {
1655                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
1656                                  DMA_TO_DEVICE);
1657                 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
1658                                  DMA_FROM_DEVICE);
1659         }
1660
1661         /*
1662          * Check if iv can be contiguous with source and destination.
1663          * If so, include it. If not, create scatterlist.
1664          */
1665         iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE);
1666         if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src))
1667                 iv_contig = true;
1668         else
1669                 src_nents = src_nents ? : 1;
1670         link_tbl_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) *
1671                          sizeof(struct link_tbl_entry);
1672
1673         /* allocate space for base edesc and hw desc commands, link tables */
1674         edesc = kmalloc(sizeof(struct ablkcipher_edesc) + desc_bytes +
1675                         link_tbl_bytes, GFP_DMA | flags);
1676         if (!edesc) {
1677                 dev_err(jrdev, "could not allocate extended descriptor\n");
1678                 return ERR_PTR(-ENOMEM);
1679         }
1680
1681         edesc->src_nents = src_nents;
1682         edesc->dst_nents = dst_nents;
1683         edesc->link_tbl_bytes = link_tbl_bytes;
1684         edesc->link_tbl = (void *)edesc + sizeof(struct ablkcipher_edesc) +
1685                           desc_bytes;
1686
1687         link_tbl_index = 0;
1688         if (!iv_contig) {
1689                 sg_to_link_tbl_one(edesc->link_tbl, iv_dma, ivsize, 0);
1690                 sg_to_link_tbl_last(req->src, src_nents,
1691                                     edesc->link_tbl + 1, 0);
1692                 link_tbl_index += 1 + src_nents;
1693         }
1694
1695         if (unlikely(dst_nents)) {
1696                 sg_to_link_tbl_last(req->dst, dst_nents,
1697                         edesc->link_tbl + link_tbl_index, 0);
1698         }
1699
1700         edesc->link_tbl_dma = dma_map_single(jrdev, edesc->link_tbl,
1701                                              link_tbl_bytes, DMA_TO_DEVICE);
1702         edesc->iv_dma = iv_dma;
1703
1704 #ifdef DEBUG
1705         print_hex_dump(KERN_ERR, "ablkcipher link_tbl@"xstr(__LINE__)": ",
1706                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->link_tbl,
1707                        link_tbl_bytes, 1);
1708 #endif
1709
1710         *iv_contig_out = iv_contig;
1711         return edesc;
1712 }
1713
1714 static int ablkcipher_encrypt(struct ablkcipher_request *req)
1715 {
1716         struct ablkcipher_edesc *edesc;
1717         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1718         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1719         struct device *jrdev = ctx->jrdev;
1720         bool iv_contig;
1721         u32 *desc;
1722         int ret = 0;
1723
1724         /* allocate extended descriptor */
1725         edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
1726                                        CAAM_CMD_SZ, &iv_contig);
1727         if (IS_ERR(edesc))
1728                 return PTR_ERR(edesc);
1729
1730         /* Create and submit job descriptor*/
1731         init_ablkcipher_job(ctx->sh_desc_enc,
1732                 ctx->sh_desc_enc_dma, edesc, req, iv_contig);
1733 #ifdef DEBUG
1734         print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"xstr(__LINE__)": ",
1735                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1736                        desc_bytes(edesc->hw_desc), 1);
1737 #endif
1738         desc = edesc->hw_desc;
1739         ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
1740
1741         if (!ret) {
1742                 ret = -EINPROGRESS;
1743         } else {
1744                 ablkcipher_unmap(jrdev, edesc, req);
1745                 kfree(edesc);
1746         }
1747
1748         return ret;
1749 }
1750
1751 static int ablkcipher_decrypt(struct ablkcipher_request *req)
1752 {
1753         struct ablkcipher_edesc *edesc;
1754         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1755         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1756         struct device *jrdev = ctx->jrdev;
1757         bool iv_contig;
1758         u32 *desc;
1759         int ret = 0;
1760
1761         /* allocate extended descriptor */
1762         edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
1763                                        CAAM_CMD_SZ, &iv_contig);
1764         if (IS_ERR(edesc))
1765                 return PTR_ERR(edesc);
1766
1767         /* Create and submit job descriptor*/
1768         init_ablkcipher_job(ctx->sh_desc_dec,
1769                 ctx->sh_desc_dec_dma, edesc, req, iv_contig);
1770         desc = edesc->hw_desc;
1771 #ifdef DEBUG
1772         print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"xstr(__LINE__)": ",
1773                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1774                        desc_bytes(edesc->hw_desc), 1);
1775 #endif
1776
1777         ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req);
1778         if (!ret) {
1779                 ret = -EINPROGRESS;
1780         } else {
1781                 ablkcipher_unmap(jrdev, edesc, req);
1782                 kfree(edesc);
1783         }
1784
1785         return ret;
1786 }
1787
1788 #define template_aead           template_u.aead
1789 #define template_ablkcipher     template_u.ablkcipher
1790 struct caam_alg_template {
1791         char name[CRYPTO_MAX_ALG_NAME];
1792         char driver_name[CRYPTO_MAX_ALG_NAME];
1793         unsigned int blocksize;
1794         u32 type;
1795         union {
1796                 struct ablkcipher_alg ablkcipher;
1797                 struct aead_alg aead;
1798                 struct blkcipher_alg blkcipher;
1799                 struct cipher_alg cipher;
1800                 struct compress_alg compress;
1801                 struct rng_alg rng;
1802         } template_u;
1803         u32 class1_alg_type;
1804         u32 class2_alg_type;
1805         u32 alg_op;
1806 };
1807
1808 static struct caam_alg_template driver_algs[] = {
1809         /* single-pass ipsec_esp descriptor */
1810         {
1811                 .name = "authenc(hmac(sha1),cbc(aes))",
1812                 .driver_name = "authenc-hmac-sha1-cbc-aes-caam",
1813                 .blocksize = AES_BLOCK_SIZE,
1814                 .type = CRYPTO_ALG_TYPE_AEAD,
1815                 .template_aead = {
1816                         .setkey = aead_setkey,
1817                         .setauthsize = aead_setauthsize,
1818                         .encrypt = aead_encrypt,
1819                         .decrypt = aead_decrypt,
1820                         .givencrypt = aead_givencrypt,
1821                         .geniv = "<built-in>",
1822                         .ivsize = AES_BLOCK_SIZE,
1823                         .maxauthsize = SHA1_DIGEST_SIZE,
1824                         },
1825                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1826                 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
1827                 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
1828         },
1829         {
1830                 .name = "authenc(hmac(sha256),cbc(aes))",
1831                 .driver_name = "authenc-hmac-sha256-cbc-aes-caam",
1832                 .blocksize = AES_BLOCK_SIZE,
1833                 .type = CRYPTO_ALG_TYPE_AEAD,
1834                 .template_aead = {
1835                         .setkey = aead_setkey,
1836                         .setauthsize = aead_setauthsize,
1837                         .encrypt = aead_encrypt,
1838                         .decrypt = aead_decrypt,
1839                         .givencrypt = aead_givencrypt,
1840                         .geniv = "<built-in>",
1841                         .ivsize = AES_BLOCK_SIZE,
1842                         .maxauthsize = SHA256_DIGEST_SIZE,
1843                         },
1844                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1845                 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1846                                    OP_ALG_AAI_HMAC_PRECOMP,
1847                 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
1848         },
1849         {
1850                 .name = "authenc(hmac(sha512),cbc(aes))",
1851                 .driver_name = "authenc-hmac-sha512-cbc-aes-caam",
1852                 .blocksize = AES_BLOCK_SIZE,
1853                 .type = CRYPTO_ALG_TYPE_AEAD,
1854                 .template_aead = {
1855                         .setkey = aead_setkey,
1856                         .setauthsize = aead_setauthsize,
1857                         .encrypt = aead_encrypt,
1858                         .decrypt = aead_decrypt,
1859                         .givencrypt = aead_givencrypt,
1860                         .geniv = "<built-in>",
1861                         .ivsize = AES_BLOCK_SIZE,
1862                         .maxauthsize = SHA512_DIGEST_SIZE,
1863                         },
1864                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1865                 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
1866                                    OP_ALG_AAI_HMAC_PRECOMP,
1867                 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
1868         },
1869         {
1870                 .name = "authenc(hmac(sha1),cbc(des3_ede))",
1871                 .driver_name = "authenc-hmac-sha1-cbc-des3_ede-caam",
1872                 .blocksize = DES3_EDE_BLOCK_SIZE,
1873                 .type = CRYPTO_ALG_TYPE_AEAD,
1874                 .template_aead = {
1875                         .setkey = aead_setkey,
1876                         .setauthsize = aead_setauthsize,
1877                         .encrypt = aead_encrypt,
1878                         .decrypt = aead_decrypt,
1879                         .givencrypt = aead_givencrypt,
1880                         .geniv = "<built-in>",
1881                         .ivsize = DES3_EDE_BLOCK_SIZE,
1882                         .maxauthsize = SHA1_DIGEST_SIZE,
1883                         },
1884                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1885                 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
1886                 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
1887         },
1888         {
1889                 .name = "authenc(hmac(sha256),cbc(des3_ede))",
1890                 .driver_name = "authenc-hmac-sha256-cbc-des3_ede-caam",
1891                 .blocksize = DES3_EDE_BLOCK_SIZE,
1892                 .type = CRYPTO_ALG_TYPE_AEAD,
1893                 .template_aead = {
1894                         .setkey = aead_setkey,
1895                         .setauthsize = aead_setauthsize,
1896                         .encrypt = aead_encrypt,
1897                         .decrypt = aead_decrypt,
1898                         .givencrypt = aead_givencrypt,
1899                         .geniv = "<built-in>",
1900                         .ivsize = DES3_EDE_BLOCK_SIZE,
1901                         .maxauthsize = SHA256_DIGEST_SIZE,
1902                         },
1903                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1904                 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1905                                    OP_ALG_AAI_HMAC_PRECOMP,
1906                 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
1907         },
1908         {
1909                 .name = "authenc(hmac(sha512),cbc(des3_ede))",
1910                 .driver_name = "authenc-hmac-sha512-cbc-des3_ede-caam",
1911                 .blocksize = DES3_EDE_BLOCK_SIZE,
1912                 .type = CRYPTO_ALG_TYPE_AEAD,
1913                 .template_aead = {
1914                         .setkey = aead_setkey,
1915                         .setauthsize = aead_setauthsize,
1916                         .encrypt = aead_encrypt,
1917                         .decrypt = aead_decrypt,
1918                         .givencrypt = aead_givencrypt,
1919                         .geniv = "<built-in>",
1920                         .ivsize = DES3_EDE_BLOCK_SIZE,
1921                         .maxauthsize = SHA512_DIGEST_SIZE,
1922                         },
1923                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1924                 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
1925                                    OP_ALG_AAI_HMAC_PRECOMP,
1926                 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
1927         },
1928         {
1929                 .name = "authenc(hmac(sha1),cbc(des))",
1930                 .driver_name = "authenc-hmac-sha1-cbc-des-caam",
1931                 .blocksize = DES_BLOCK_SIZE,
1932                 .type = CRYPTO_ALG_TYPE_AEAD,
1933                 .template_aead = {
1934                         .setkey = aead_setkey,
1935                         .setauthsize = aead_setauthsize,
1936                         .encrypt = aead_encrypt,
1937                         .decrypt = aead_decrypt,
1938                         .givencrypt = aead_givencrypt,
1939                         .geniv = "<built-in>",
1940                         .ivsize = DES_BLOCK_SIZE,
1941                         .maxauthsize = SHA1_DIGEST_SIZE,
1942                         },
1943                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1944                 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
1945                 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
1946         },
1947         {
1948                 .name = "authenc(hmac(sha256),cbc(des))",
1949                 .driver_name = "authenc-hmac-sha256-cbc-des-caam",
1950                 .blocksize = DES_BLOCK_SIZE,
1951                 .type = CRYPTO_ALG_TYPE_AEAD,
1952                 .template_aead = {
1953                         .setkey = aead_setkey,
1954                         .setauthsize = aead_setauthsize,
1955                         .encrypt = aead_encrypt,
1956                         .decrypt = aead_decrypt,
1957                         .givencrypt = aead_givencrypt,
1958                         .geniv = "<built-in>",
1959                         .ivsize = DES_BLOCK_SIZE,
1960                         .maxauthsize = SHA256_DIGEST_SIZE,
1961                         },
1962                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1963                 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1964                                    OP_ALG_AAI_HMAC_PRECOMP,
1965                 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
1966         },
1967         {
1968                 .name = "authenc(hmac(sha512),cbc(des))",
1969                 .driver_name = "authenc-hmac-sha512-cbc-des-caam",
1970                 .blocksize = DES_BLOCK_SIZE,
1971                 .type = CRYPTO_ALG_TYPE_AEAD,
1972                 .template_aead = {
1973                         .setkey = aead_setkey,
1974                         .setauthsize = aead_setauthsize,
1975                         .encrypt = aead_encrypt,
1976                         .decrypt = aead_decrypt,
1977                         .givencrypt = aead_givencrypt,
1978                         .geniv = "<built-in>",
1979                         .ivsize = DES_BLOCK_SIZE,
1980                         .maxauthsize = SHA512_DIGEST_SIZE,
1981                         },
1982                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1983                 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
1984                                    OP_ALG_AAI_HMAC_PRECOMP,
1985                 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
1986         },
1987         /* ablkcipher descriptor */
1988         {
1989                 .name = "cbc(aes)",
1990                 .driver_name = "cbc-aes-caam",
1991                 .blocksize = AES_BLOCK_SIZE,
1992                 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
1993                 .template_ablkcipher = {
1994                         .setkey = ablkcipher_setkey,
1995                         .encrypt = ablkcipher_encrypt,
1996                         .decrypt = ablkcipher_decrypt,
1997                         .geniv = "eseqiv",
1998                         .min_keysize = AES_MIN_KEY_SIZE,
1999                         .max_keysize = AES_MAX_KEY_SIZE,
2000                         .ivsize = AES_BLOCK_SIZE,
2001                         },
2002                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2003         },
2004         {
2005                 .name = "cbc(des3_ede)",
2006                 .driver_name = "cbc-3des-caam",
2007                 .blocksize = DES3_EDE_BLOCK_SIZE,
2008                 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2009                 .template_ablkcipher = {
2010                         .setkey = ablkcipher_setkey,
2011                         .encrypt = ablkcipher_encrypt,
2012                         .decrypt = ablkcipher_decrypt,
2013                         .geniv = "eseqiv",
2014                         .min_keysize = DES3_EDE_KEY_SIZE,
2015                         .max_keysize = DES3_EDE_KEY_SIZE,
2016                         .ivsize = DES3_EDE_BLOCK_SIZE,
2017                         },
2018                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2019         },
2020         {
2021                 .name = "cbc(des)",
2022                 .driver_name = "cbc-des-caam",
2023                 .blocksize = DES_BLOCK_SIZE,
2024                 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2025                 .template_ablkcipher = {
2026                         .setkey = ablkcipher_setkey,
2027                         .encrypt = ablkcipher_encrypt,
2028                         .decrypt = ablkcipher_decrypt,
2029                         .geniv = "eseqiv",
2030                         .min_keysize = DES_KEY_SIZE,
2031                         .max_keysize = DES_KEY_SIZE,
2032                         .ivsize = DES_BLOCK_SIZE,
2033                         },
2034                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2035         }
2036 };
2037
2038 struct caam_crypto_alg {
2039         struct list_head entry;
2040         struct device *ctrldev;
2041         int class1_alg_type;
2042         int class2_alg_type;
2043         int alg_op;
2044         struct crypto_alg crypto_alg;
2045 };
2046
2047 static int caam_cra_init(struct crypto_tfm *tfm)
2048 {
2049         struct crypto_alg *alg = tfm->__crt_alg;
2050         struct caam_crypto_alg *caam_alg =
2051                  container_of(alg, struct caam_crypto_alg, crypto_alg);
2052         struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
2053         struct caam_drv_private *priv = dev_get_drvdata(caam_alg->ctrldev);
2054         int tgt_jr = atomic_inc_return(&priv->tfm_count);
2055
2056         /*
2057          * distribute tfms across job rings to ensure in-order
2058          * crypto request processing per tfm
2059          */
2060         ctx->jrdev = priv->algapi_jr[(tgt_jr / 2) % priv->num_jrs_for_algapi];
2061
2062         /* copy descriptor header template value */
2063         ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam_alg->class1_alg_type;
2064         ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam_alg->class2_alg_type;
2065         ctx->alg_op = OP_TYPE_CLASS2_ALG | caam_alg->alg_op;
2066
2067         return 0;
2068 }
2069
2070 static void caam_cra_exit(struct crypto_tfm *tfm)
2071 {
2072         struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
2073
2074         if (ctx->sh_desc_enc_dma &&
2075             !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma))
2076                 dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma,
2077                                  desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE);
2078         if (ctx->sh_desc_dec_dma &&
2079             !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma))
2080                 dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma,
2081                                  desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE);
2082         if (ctx->sh_desc_givenc_dma &&
2083             !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma))
2084                 dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma,
2085                                  desc_bytes(ctx->sh_desc_givenc),
2086                                  DMA_TO_DEVICE);
2087 }
2088
2089 static void __exit caam_algapi_exit(void)
2090 {
2091
2092         struct device_node *dev_node;
2093         struct platform_device *pdev;
2094         struct device *ctrldev;
2095         struct caam_drv_private *priv;
2096         struct caam_crypto_alg *t_alg, *n;
2097         int i, err;
2098
2099         dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
2100         if (!dev_node)
2101                 return;
2102
2103         pdev = of_find_device_by_node(dev_node);
2104         if (!pdev)
2105                 return;
2106
2107         ctrldev = &pdev->dev;
2108         of_node_put(dev_node);
2109         priv = dev_get_drvdata(ctrldev);
2110
2111         if (!priv->alg_list.next)
2112                 return;
2113
2114         list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
2115                 crypto_unregister_alg(&t_alg->crypto_alg);
2116                 list_del(&t_alg->entry);
2117                 kfree(t_alg);
2118         }
2119
2120         for (i = 0; i < priv->total_jobrs; i++) {
2121                 err = caam_jr_deregister(priv->algapi_jr[i]);
2122                 if (err < 0)
2123                         break;
2124         }
2125         kfree(priv->algapi_jr);
2126 }
2127
2128 static struct caam_crypto_alg *caam_alg_alloc(struct device *ctrldev,
2129                                               struct caam_alg_template
2130                                               *template)
2131 {
2132         struct caam_crypto_alg *t_alg;
2133         struct crypto_alg *alg;
2134
2135         t_alg = kzalloc(sizeof(struct caam_crypto_alg), GFP_KERNEL);
2136         if (!t_alg) {
2137                 dev_err(ctrldev, "failed to allocate t_alg\n");
2138                 return ERR_PTR(-ENOMEM);
2139         }
2140
2141         alg = &t_alg->crypto_alg;
2142
2143         snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name);
2144         snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
2145                  template->driver_name);
2146         alg->cra_module = THIS_MODULE;
2147         alg->cra_init = caam_cra_init;
2148         alg->cra_exit = caam_cra_exit;
2149         alg->cra_priority = CAAM_CRA_PRIORITY;
2150         alg->cra_blocksize = template->blocksize;
2151         alg->cra_alignmask = 0;
2152         alg->cra_ctxsize = sizeof(struct caam_ctx);
2153         alg->cra_flags = CRYPTO_ALG_ASYNC | template->type;
2154         switch (template->type) {
2155         case CRYPTO_ALG_TYPE_ABLKCIPHER:
2156                 alg->cra_type = &crypto_ablkcipher_type;
2157                 alg->cra_ablkcipher = template->template_ablkcipher;
2158                 break;
2159         case CRYPTO_ALG_TYPE_AEAD:
2160                 alg->cra_type = &crypto_aead_type;
2161                 alg->cra_aead = template->template_aead;
2162                 break;
2163         }
2164
2165         t_alg->class1_alg_type = template->class1_alg_type;
2166         t_alg->class2_alg_type = template->class2_alg_type;
2167         t_alg->alg_op = template->alg_op;
2168         t_alg->ctrldev = ctrldev;
2169
2170         return t_alg;
2171 }
2172
2173 static int __init caam_algapi_init(void)
2174 {
2175         struct device_node *dev_node;
2176         struct platform_device *pdev;
2177         struct device *ctrldev, **jrdev;
2178         struct caam_drv_private *priv;
2179         int i = 0, err = 0;
2180
2181         dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
2182         if (!dev_node)
2183                 return -ENODEV;
2184
2185         pdev = of_find_device_by_node(dev_node);
2186         if (!pdev)
2187                 return -ENODEV;
2188
2189         ctrldev = &pdev->dev;
2190         priv = dev_get_drvdata(ctrldev);
2191         of_node_put(dev_node);
2192
2193         INIT_LIST_HEAD(&priv->alg_list);
2194
2195         jrdev = kmalloc(sizeof(*jrdev) * priv->total_jobrs, GFP_KERNEL);
2196         if (!jrdev)
2197                 return -ENOMEM;
2198
2199         for (i = 0; i < priv->total_jobrs; i++) {
2200                 err = caam_jr_register(ctrldev, &jrdev[i]);
2201                 if (err < 0)
2202                         break;
2203         }
2204         if (err < 0 && i == 0) {
2205                 dev_err(ctrldev, "algapi error in job ring registration: %d\n",
2206                         err);
2207                 kfree(jrdev);
2208                 return err;
2209         }
2210
2211         priv->num_jrs_for_algapi = i;
2212         priv->algapi_jr = jrdev;
2213         atomic_set(&priv->tfm_count, -1);
2214
2215         /* register crypto algorithms the device supports */
2216         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
2217                 /* TODO: check if h/w supports alg */
2218                 struct caam_crypto_alg *t_alg;
2219
2220                 t_alg = caam_alg_alloc(ctrldev, &driver_algs[i]);
2221                 if (IS_ERR(t_alg)) {
2222                         err = PTR_ERR(t_alg);
2223                         dev_warn(ctrldev, "%s alg allocation failed\n",
2224                                  driver_algs[i].driver_name);
2225                         continue;
2226                 }
2227
2228                 err = crypto_register_alg(&t_alg->crypto_alg);
2229                 if (err) {
2230                         dev_warn(ctrldev, "%s alg registration failed\n",
2231                                 t_alg->crypto_alg.cra_driver_name);
2232                         kfree(t_alg);
2233                 } else {
2234                         list_add_tail(&t_alg->entry, &priv->alg_list);
2235                         dev_info(ctrldev, "%s\n",
2236                                  t_alg->crypto_alg.cra_driver_name);
2237                 }
2238         }
2239
2240         return err;
2241 }
2242
2243 module_init(caam_algapi_init);
2244 module_exit(caam_algapi_exit);
2245
2246 MODULE_LICENSE("GPL");
2247 MODULE_DESCRIPTION("FSL CAAM support for crypto API");
2248 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");