arch/x86/crypto/aesni-intel_glue.c

   1 /*
   2  * Support for Intel AES-NI instructions. This file contains glue
   3  * code, the real AES implementation is in intel-aes_asm.S.
   4  *
   5  * Copyright (C) 2008, Intel Corp.
   6  *    Author: Huang Ying <ying.huang@intel.com>
   7  *
   8  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
   9  * interface for 64-bit kernels.
  10  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
  11  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
  12  *             Tadeusz Struk (tadeusz.struk@intel.com)
  13  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
  14  *    Copyright (c) 2010, Intel Corporation.
  15  *
  16  * This program is free software; you can redistribute it and/or modify
  17  * it under the terms of the GNU General Public License as published by
  18  * the Free Software Foundation; either version 2 of the License, or
  19  * (at your option) any later version.
  20  */
  21
  22 #include <linux/hardirq.h>
  23 #include <linux/types.h>
  24 #include <linux/crypto.h>
  25 #include <linux/err.h>
  26 #include <crypto/algapi.h>
  27 #include <crypto/aes.h>
  28 #include <crypto/cryptd.h>
  29 #include <crypto/ctr.h>
  30 #include <asm/i387.h>
  31 #include <asm/aes.h>
  32 #include <crypto/scatterwalk.h>
  33 #include <crypto/internal/aead.h>
  34 #include <linux/workqueue.h>
  35 #include <linux/spinlock.h>
  36
  37 #if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE)
  38 #define HAS_CTR
  39 #endif
  40
  41 #if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE)
  42 #define HAS_LRW
  43 #endif
  44
  45 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE)
  46 #define HAS_PCBC
  47 #endif
  48
  49 #if defined(CONFIG_CRYPTO_XTS) || defined(CONFIG_CRYPTO_XTS_MODULE)
  50 #define HAS_XTS
  51 #endif
  52
  53 struct async_aes_ctx {
  54         struct cryptd_ablkcipher *cryptd_tfm;
  55 };
  56
  57 /* This data is stored at the end of the crypto_tfm struct.
  58  * It's a type of per "session" data storage location.
  59  * This needs to be 16 byte aligned.
  60  */
  61 struct aesni_rfc4106_gcm_ctx {
  62         u8 hash_subkey[16];
  63         struct crypto_aes_ctx aes_key_expanded;
  64         u8 nonce[4];
  65         struct cryptd_aead *cryptd_tfm;
  66 };
  67
  68 struct aesni_gcm_set_hash_subkey_result {
  69         int err;
  70         struct completion completion;
  71 };
  72
  73 struct aesni_hash_subkey_req_data {
  74         u8 iv[16];
  75         struct aesni_gcm_set_hash_subkey_result result;
  76         struct scatterlist sg;
  77 };
  78
  79 #define AESNI_ALIGN     (16)
  80 #define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE-1))
  81 #define RFC4106_HASH_SUBKEY_SIZE 16
  82
  83 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
  84                              unsigned int key_len);
  85 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
  86                           const u8 *in);
  87 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
  88                           const u8 *in);
  89 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
  90                               const u8 *in, unsigned int len);
  91 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
  92                               const u8 *in, unsigned int len);
  93 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
  94                               const u8 *in, unsigned int len, u8 *iv);
  95 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
  96                               const u8 *in, unsigned int len, u8 *iv);
  97
  98 int crypto_fpu_init(void);
  99 void crypto_fpu_exit(void);
 100
 101 #ifdef CONFIG_X86_64
 102 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
 103                               const u8 *in, unsigned int len, u8 *iv);
 104
 105 /* asmlinkage void aesni_gcm_enc()
 106  * void *ctx,  AES Key schedule. Starts on a 16 byte boundary.
 107  * u8 *out, Ciphertext output. Encrypt in-place is allowed.
 108  * const u8 *in, Plaintext input
 109  * unsigned long plaintext_len, Length of data in bytes for encryption.
 110  * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
 111  *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
 112  *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
 113  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 114  * const u8 *aad, Additional Authentication Data (AAD)
 115  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this
 116  *          is going to be 8 or 12 bytes
 117  * u8 *auth_tag, Authenticated Tag output.
 118  * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
 119  *          Valid values are 16 (most likely), 12 or 8.
 120  */
 121 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
 122                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 123                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 124                         u8 *auth_tag, unsigned long auth_tag_len);
 125
 126 /* asmlinkage void aesni_gcm_dec()
 127  * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
 128  * u8 *out, Plaintext output. Decrypt in-place is allowed.
 129  * const u8 *in, Ciphertext input
 130  * unsigned long ciphertext_len, Length of data in bytes for decryption.
 131  * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
 132  *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
 133  *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
 134  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 135  * const u8 *aad, Additional Authentication Data (AAD)
 136  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
 137  * to be 8 or 12 bytes
 138  * u8 *auth_tag, Authenticated Tag output.
 139  * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
 140  * Valid values are 16 (most likely), 12 or 8.
 141  */
 142 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
 143                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 144                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 145                         u8 *auth_tag, unsigned long auth_tag_len);
 146
 147 static inline struct
 148 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
 149 {
 150         return
 151                 (struct aesni_rfc4106_gcm_ctx *)
 152                 PTR_ALIGN((u8 *)
 153                 crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN);
 154 }
 155 #endif
 156
 157 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
 158 {
 159         unsigned long addr = (unsigned long)raw_ctx;
 160         unsigned long align = AESNI_ALIGN;
 161
 162         if (align <= crypto_tfm_ctx_alignment())
 163                 align = 1;
 164         return (struct crypto_aes_ctx *)ALIGN(addr, align);
 165 }
 166
 167 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
 168                               const u8 *in_key, unsigned int key_len)
 169 {
 170         struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
 171         u32 *flags = &tfm->crt_flags;
 172         int err;
 173
 174         if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
 175             key_len != AES_KEYSIZE_256) {
 176                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 177                 return -EINVAL;
 178         }
 179
 180         if (!irq_fpu_usable())
 181                 err = crypto_aes_expand_key(ctx, in_key, key_len);
 182         else {
 183                 kernel_fpu_begin();
 184                 err = aesni_set_key(ctx, in_key, key_len);
 185                 kernel_fpu_end();
 186         }
 187
 188         return err;
 189 }
 190
 191 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 192                        unsigned int key_len)
 193 {
 194         return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
 195 }
 196
 197 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 198 {
 199         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 200
 201         if (!irq_fpu_usable())
 202                 crypto_aes_encrypt_x86(ctx, dst, src);
 203         else {
 204                 kernel_fpu_begin();
 205                 aesni_enc(ctx, dst, src);
 206                 kernel_fpu_end();
 207         }
 208 }
 209
 210 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 211 {
 212         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 213
 214         if (!irq_fpu_usable())
 215                 crypto_aes_decrypt_x86(ctx, dst, src);
 216         else {
 217                 kernel_fpu_begin();
 218                 aesni_dec(ctx, dst, src);
 219                 kernel_fpu_end();
 220         }
 221 }
 222
 223 static struct crypto_alg aesni_alg = {
 224         .cra_name               = "aes",
 225         .cra_driver_name        = "aes-aesni",
 226         .cra_priority           = 300,
 227         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
 228         .cra_blocksize          = AES_BLOCK_SIZE,
 229         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
 230         .cra_alignmask          = 0,
 231         .cra_module             = THIS_MODULE,
 232         .cra_list               = LIST_HEAD_INIT(aesni_alg.cra_list),
 233         .cra_u  = {
 234                 .cipher = {
 235                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
 236                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
 237                         .cia_setkey             = aes_set_key,
 238                         .cia_encrypt            = aes_encrypt,
 239                         .cia_decrypt            = aes_decrypt
 240                 }
 241         }
 242 };
 243
 244 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 245 {
 246         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 247
 248         aesni_enc(ctx, dst, src);
 249 }
 250
 251 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 252 {
 253         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 254
 255         aesni_dec(ctx, dst, src);
 256 }
 257
 258 static struct crypto_alg __aesni_alg = {
 259         .cra_name               = "__aes-aesni",
 260         .cra_driver_name        = "__driver-aes-aesni",
 261         .cra_priority           = 0,
 262         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
 263         .cra_blocksize          = AES_BLOCK_SIZE,
 264         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
 265         .cra_alignmask          = 0,
 266         .cra_module             = THIS_MODULE,
 267         .cra_list               = LIST_HEAD_INIT(__aesni_alg.cra_list),
 268         .cra_u  = {
 269                 .cipher = {
 270                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
 271                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
 272                         .cia_setkey             = aes_set_key,
 273                         .cia_encrypt            = __aes_encrypt,
 274                         .cia_decrypt            = __aes_decrypt
 275                 }
 276         }
 277 };
 278
 279 static int ecb_encrypt(struct blkcipher_desc *desc,
 280                        struct scatterlist *dst, struct scatterlist *src,
 281                        unsigned int nbytes)
 282 {
 283         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 284         struct blkcipher_walk walk;
 285         int err;
 286
 287         blkcipher_walk_init(&walk, dst, src, nbytes);
 288         err = blkcipher_walk_virt(desc, &walk);
 289         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 290
 291         kernel_fpu_begin();
 292         while ((nbytes = walk.nbytes)) {
 293                 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 294                               nbytes & AES_BLOCK_MASK);
 295                 nbytes &= AES_BLOCK_SIZE - 1;
 296                 err = blkcipher_walk_done(desc, &walk, nbytes);
 297         }
 298         kernel_fpu_end();
 299
 300         return err;
 301 }
 302
 303 static int ecb_decrypt(struct blkcipher_desc *desc,
 304                        struct scatterlist *dst, struct scatterlist *src,
 305                        unsigned int nbytes)
 306 {
 307         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 308         struct blkcipher_walk walk;
 309         int err;
 310
 311         blkcipher_walk_init(&walk, dst, src, nbytes);
 312         err = blkcipher_walk_virt(desc, &walk);
 313         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 314
 315         kernel_fpu_begin();
 316         while ((nbytes = walk.nbytes)) {
 317                 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 318                               nbytes & AES_BLOCK_MASK);
 319                 nbytes &= AES_BLOCK_SIZE - 1;
 320                 err = blkcipher_walk_done(desc, &walk, nbytes);
 321         }
 322         kernel_fpu_end();
 323
 324         return err;
 325 }
 326
 327 static struct crypto_alg blk_ecb_alg = {
 328         .cra_name               = "__ecb-aes-aesni",
 329         .cra_driver_name        = "__driver-ecb-aes-aesni",
 330         .cra_priority           = 0,
 331         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 332         .cra_blocksize          = AES_BLOCK_SIZE,
 333         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
 334         .cra_alignmask          = 0,
 335         .cra_type               = &crypto_blkcipher_type,
 336         .cra_module             = THIS_MODULE,
 337         .cra_list               = LIST_HEAD_INIT(blk_ecb_alg.cra_list),
 338         .cra_u = {
 339                 .blkcipher = {
 340                         .min_keysize    = AES_MIN_KEY_SIZE,
 341                         .max_keysize    = AES_MAX_KEY_SIZE,
 342                         .setkey         = aes_set_key,
 343                         .encrypt        = ecb_encrypt,
 344                         .decrypt        = ecb_decrypt,
 345                 },
 346         },
 347 };
 348
 349 static int cbc_encrypt(struct blkcipher_desc *desc,
 350                        struct scatterlist *dst, struct scatterlist *src,
 351                        unsigned int nbytes)
 352 {
 353         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 354         struct blkcipher_walk walk;
 355         int err;
 356
 357         blkcipher_walk_init(&walk, dst, src, nbytes);
 358         err = blkcipher_walk_virt(desc, &walk);
 359         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 360
 361         kernel_fpu_begin();
 362         while ((nbytes = walk.nbytes)) {
 363                 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 364                               nbytes & AES_BLOCK_MASK, walk.iv);
 365                 nbytes &= AES_BLOCK_SIZE - 1;
 366                 err = blkcipher_walk_done(desc, &walk, nbytes);
 367         }
 368         kernel_fpu_end();
 369
 370         return err;
 371 }
 372
 373 static int cbc_decrypt(struct blkcipher_desc *desc,
 374                        struct scatterlist *dst, struct scatterlist *src,
 375                        unsigned int nbytes)
 376 {
 377         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 378         struct blkcipher_walk walk;
 379         int err;
 380
 381         blkcipher_walk_init(&walk, dst, src, nbytes);
 382         err = blkcipher_walk_virt(desc, &walk);
 383         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 384
 385         kernel_fpu_begin();
 386         while ((nbytes = walk.nbytes)) {
 387                 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 388                               nbytes & AES_BLOCK_MASK, walk.iv);
 389                 nbytes &= AES_BLOCK_SIZE - 1;
 390                 err = blkcipher_walk_done(desc, &walk, nbytes);
 391         }
 392         kernel_fpu_end();
 393
 394         return err;
 395 }
 396
 397 static struct crypto_alg blk_cbc_alg = {
 398         .cra_name               = "__cbc-aes-aesni",
 399         .cra_driver_name        = "__driver-cbc-aes-aesni",
 400         .cra_priority           = 0,
 401         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 402         .cra_blocksize          = AES_BLOCK_SIZE,
 403         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
 404         .cra_alignmask          = 0,
 405         .cra_type               = &crypto_blkcipher_type,
 406         .cra_module             = THIS_MODULE,
 407         .cra_list               = LIST_HEAD_INIT(blk_cbc_alg.cra_list),
 408         .cra_u = {
 409                 .blkcipher = {
 410                         .min_keysize    = AES_MIN_KEY_SIZE,
 411                         .max_keysize    = AES_MAX_KEY_SIZE,
 412                         .setkey         = aes_set_key,
 413                         .encrypt        = cbc_encrypt,
 414                         .decrypt        = cbc_decrypt,
 415                 },
 416         },
 417 };
 418
 419 #ifdef CONFIG_X86_64
 420 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
 421                             struct blkcipher_walk *walk)
 422 {
 423         u8 *ctrblk = walk->iv;
 424         u8 keystream[AES_BLOCK_SIZE];
 425         u8 *src = walk->src.virt.addr;
 426         u8 *dst = walk->dst.virt.addr;
 427         unsigned int nbytes = walk->nbytes;
 428
 429         aesni_enc(ctx, keystream, ctrblk);
 430         crypto_xor(keystream, src, nbytes);
 431         memcpy(dst, keystream, nbytes);
 432         crypto_inc(ctrblk, AES_BLOCK_SIZE);
 433 }
 434
 435 static int ctr_crypt(struct blkcipher_desc *desc,
 436                      struct scatterlist *dst, struct scatterlist *src,
 437                      unsigned int nbytes)
 438 {
 439         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 440         struct blkcipher_walk walk;
 441         int err;
 442
 443         blkcipher_walk_init(&walk, dst, src, nbytes);
 444         err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
 445         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 446
 447         kernel_fpu_begin();
 448         while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
 449                 aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 450                               nbytes & AES_BLOCK_MASK, walk.iv);
 451                 nbytes &= AES_BLOCK_SIZE - 1;
 452                 err = blkcipher_walk_done(desc, &walk, nbytes);
 453         }
 454         if (walk.nbytes) {
 455                 ctr_crypt_final(ctx, &walk);
 456                 err = blkcipher_walk_done(desc, &walk, 0);
 457         }
 458         kernel_fpu_end();
 459
 460         return err;
 461 }
 462
 463 static struct crypto_alg blk_ctr_alg = {
 464         .cra_name               = "__ctr-aes-aesni",
 465         .cra_driver_name        = "__driver-ctr-aes-aesni",
 466         .cra_priority           = 0,
 467         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 468         .cra_blocksize          = 1,
 469         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
 470         .cra_alignmask          = 0,
 471         .cra_type               = &crypto_blkcipher_type,
 472         .cra_module             = THIS_MODULE,
 473         .cra_list               = LIST_HEAD_INIT(blk_ctr_alg.cra_list),
 474         .cra_u = {
 475                 .blkcipher = {
 476                         .min_keysize    = AES_MIN_KEY_SIZE,
 477                         .max_keysize    = AES_MAX_KEY_SIZE,
 478                         .ivsize         = AES_BLOCK_SIZE,
 479                         .setkey         = aes_set_key,
 480                         .encrypt        = ctr_crypt,
 481                         .decrypt        = ctr_crypt,
 482                 },
 483         },
 484 };
 485 #endif
 486
 487 static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
 488                         unsigned int key_len)
 489 {
 490         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 491         struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base;
 492         int err;
 493
 494         crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 495         crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm)
 496                                     & CRYPTO_TFM_REQ_MASK);
 497         err = crypto_ablkcipher_setkey(child, key, key_len);
 498         crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child)
 499                                     & CRYPTO_TFM_RES_MASK);
 500         return err;
 501 }
 502
 503 static int ablk_encrypt(struct ablkcipher_request *req)
 504 {
 505         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 506         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 507
 508         if (!irq_fpu_usable()) {
 509                 struct ablkcipher_request *cryptd_req =
 510                         ablkcipher_request_ctx(req);
 511                 memcpy(cryptd_req, req, sizeof(*req));
 512                 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
 513                 return crypto_ablkcipher_encrypt(cryptd_req);
 514         } else {
 515                 struct blkcipher_desc desc;
 516                 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
 517                 desc.info = req->info;
 518                 desc.flags = 0;
 519                 return crypto_blkcipher_crt(desc.tfm)->encrypt(
 520                         &desc, req->dst, req->src, req->nbytes);
 521         }
 522 }
 523
 524 static int ablk_decrypt(struct ablkcipher_request *req)
 525 {
 526         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 527         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 528
 529         if (!irq_fpu_usable()) {
 530                 struct ablkcipher_request *cryptd_req =
 531                         ablkcipher_request_ctx(req);
 532                 memcpy(cryptd_req, req, sizeof(*req));
 533                 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
 534                 return crypto_ablkcipher_decrypt(cryptd_req);
 535         } else {
 536                 struct blkcipher_desc desc;
 537                 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
 538                 desc.info = req->info;
 539                 desc.flags = 0;
 540                 return crypto_blkcipher_crt(desc.tfm)->decrypt(
 541                         &desc, req->dst, req->src, req->nbytes);
 542         }
 543 }
 544
 545 static void ablk_exit(struct crypto_tfm *tfm)
 546 {
 547         struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 548
 549         cryptd_free_ablkcipher(ctx->cryptd_tfm);
 550 }
 551
 552 static void ablk_init_common(struct crypto_tfm *tfm,
 553                              struct cryptd_ablkcipher *cryptd_tfm)
 554 {
 555         struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 556
 557         ctx->cryptd_tfm = cryptd_tfm;
 558         tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
 559                 crypto_ablkcipher_reqsize(&cryptd_tfm->base);
 560 }
 561
 562 static int ablk_ecb_init(struct crypto_tfm *tfm)
 563 {
 564         struct cryptd_ablkcipher *cryptd_tfm;
 565
 566         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ecb-aes-aesni", 0, 0);
 567         if (IS_ERR(cryptd_tfm))
 568                 return PTR_ERR(cryptd_tfm);
 569         ablk_init_common(tfm, cryptd_tfm);
 570         return 0;
 571 }
 572
 573 static struct crypto_alg ablk_ecb_alg = {
 574         .cra_name               = "ecb(aes)",
 575         .cra_driver_name        = "ecb-aes-aesni",
 576         .cra_priority           = 400,
 577         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 578         .cra_blocksize          = AES_BLOCK_SIZE,
 579         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 580         .cra_alignmask          = 0,
 581         .cra_type               = &crypto_ablkcipher_type,
 582         .cra_module             = THIS_MODULE,
 583         .cra_list               = LIST_HEAD_INIT(ablk_ecb_alg.cra_list),
 584         .cra_init               = ablk_ecb_init,
 585         .cra_exit               = ablk_exit,
 586         .cra_u = {
 587                 .ablkcipher = {
 588                         .min_keysize    = AES_MIN_KEY_SIZE,
 589                         .max_keysize    = AES_MAX_KEY_SIZE,
 590                         .setkey         = ablk_set_key,
 591                         .encrypt        = ablk_encrypt,
 592                         .decrypt        = ablk_decrypt,
 593                 },
 594         },
 595 };
 596
 597 static int ablk_cbc_init(struct crypto_tfm *tfm)
 598 {
 599         struct cryptd_ablkcipher *cryptd_tfm;
 600
 601         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-cbc-aes-aesni", 0, 0);
 602         if (IS_ERR(cryptd_tfm))
 603                 return PTR_ERR(cryptd_tfm);
 604         ablk_init_common(tfm, cryptd_tfm);
 605         return 0;
 606 }
 607
 608 static struct crypto_alg ablk_cbc_alg = {
 609         .cra_name               = "cbc(aes)",
 610         .cra_driver_name        = "cbc-aes-aesni",
 611         .cra_priority           = 400,
 612         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 613         .cra_blocksize          = AES_BLOCK_SIZE,
 614         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 615         .cra_alignmask          = 0,
 616         .cra_type               = &crypto_ablkcipher_type,
 617         .cra_module             = THIS_MODULE,
 618         .cra_list               = LIST_HEAD_INIT(ablk_cbc_alg.cra_list),
 619         .cra_init               = ablk_cbc_init,
 620         .cra_exit               = ablk_exit,
 621         .cra_u = {
 622                 .ablkcipher = {
 623                         .min_keysize    = AES_MIN_KEY_SIZE,
 624                         .max_keysize    = AES_MAX_KEY_SIZE,
 625                         .ivsize         = AES_BLOCK_SIZE,
 626                         .setkey         = ablk_set_key,
 627                         .encrypt        = ablk_encrypt,
 628                         .decrypt        = ablk_decrypt,
 629                 },
 630         },
 631 };
 632
 633 #ifdef CONFIG_X86_64
 634 static int ablk_ctr_init(struct crypto_tfm *tfm)
 635 {
 636         struct cryptd_ablkcipher *cryptd_tfm;
 637
 638         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ctr-aes-aesni", 0, 0);
 639         if (IS_ERR(cryptd_tfm))
 640                 return PTR_ERR(cryptd_tfm);
 641         ablk_init_common(tfm, cryptd_tfm);
 642         return 0;
 643 }
 644
 645 static struct crypto_alg ablk_ctr_alg = {
 646         .cra_name               = "ctr(aes)",
 647         .cra_driver_name        = "ctr-aes-aesni",
 648         .cra_priority           = 400,
 649         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 650         .cra_blocksize          = 1,
 651         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 652         .cra_alignmask          = 0,
 653         .cra_type               = &crypto_ablkcipher_type,
 654         .cra_module             = THIS_MODULE,
 655         .cra_list               = LIST_HEAD_INIT(ablk_ctr_alg.cra_list),
 656         .cra_init               = ablk_ctr_init,
 657         .cra_exit               = ablk_exit,
 658         .cra_u = {
 659                 .ablkcipher = {
 660                         .min_keysize    = AES_MIN_KEY_SIZE,
 661                         .max_keysize    = AES_MAX_KEY_SIZE,
 662                         .ivsize         = AES_BLOCK_SIZE,
 663                         .setkey         = ablk_set_key,
 664                         .encrypt        = ablk_encrypt,
 665                         .decrypt        = ablk_encrypt,
 666                         .geniv          = "chainiv",
 667                 },
 668         },
 669 };
 670
 671 #ifdef HAS_CTR
 672 static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm)
 673 {
 674         struct cryptd_ablkcipher *cryptd_tfm;
 675
 676         cryptd_tfm = cryptd_alloc_ablkcipher(
 677                 "rfc3686(__driver-ctr-aes-aesni)", 0, 0);
 678         if (IS_ERR(cryptd_tfm))
 679                 return PTR_ERR(cryptd_tfm);
 680         ablk_init_common(tfm, cryptd_tfm);
 681         return 0;
 682 }
 683
 684 static struct crypto_alg ablk_rfc3686_ctr_alg = {
 685         .cra_name               = "rfc3686(ctr(aes))",
 686         .cra_driver_name        = "rfc3686-ctr-aes-aesni",
 687         .cra_priority           = 400,
 688         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 689         .cra_blocksize          = 1,
 690         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 691         .cra_alignmask          = 0,
 692         .cra_type               = &crypto_ablkcipher_type,
 693         .cra_module             = THIS_MODULE,
 694         .cra_list               = LIST_HEAD_INIT(ablk_rfc3686_ctr_alg.cra_list),
 695         .cra_init               = ablk_rfc3686_ctr_init,
 696         .cra_exit               = ablk_exit,
 697         .cra_u = {
 698                 .ablkcipher = {
 699                         .min_keysize = AES_MIN_KEY_SIZE+CTR_RFC3686_NONCE_SIZE,
 700                         .max_keysize = AES_MAX_KEY_SIZE+CTR_RFC3686_NONCE_SIZE,
 701                         .ivsize      = CTR_RFC3686_IV_SIZE,
 702                         .setkey      = ablk_set_key,
 703                         .encrypt     = ablk_encrypt,
 704                         .decrypt     = ablk_decrypt,
 705                         .geniv       = "seqiv",
 706                 },
 707         },
 708 };
 709 #endif
 710 #endif
 711
 712 #ifdef HAS_LRW
 713 static int ablk_lrw_init(struct crypto_tfm *tfm)
 714 {
 715         struct cryptd_ablkcipher *cryptd_tfm;
 716
 717         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(lrw(__driver-aes-aesni))",
 718                                              0, 0);
 719         if (IS_ERR(cryptd_tfm))
 720                 return PTR_ERR(cryptd_tfm);
 721         ablk_init_common(tfm, cryptd_tfm);
 722         return 0;
 723 }
 724
 725 static struct crypto_alg ablk_lrw_alg = {
 726         .cra_name               = "lrw(aes)",
 727         .cra_driver_name        = "lrw-aes-aesni",
 728         .cra_priority           = 400,
 729         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 730         .cra_blocksize          = AES_BLOCK_SIZE,
 731         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 732         .cra_alignmask          = 0,
 733         .cra_type               = &crypto_ablkcipher_type,
 734         .cra_module             = THIS_MODULE,
 735         .cra_list               = LIST_HEAD_INIT(ablk_lrw_alg.cra_list),
 736         .cra_init               = ablk_lrw_init,
 737         .cra_exit               = ablk_exit,
 738         .cra_u = {
 739                 .ablkcipher = {
 740                         .min_keysize    = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
 741                         .max_keysize    = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
 742                         .ivsize         = AES_BLOCK_SIZE,
 743                         .setkey         = ablk_set_key,
 744                         .encrypt        = ablk_encrypt,
 745                         .decrypt        = ablk_decrypt,
 746                 },
 747         },
 748 };
 749 #endif
 750
 751 #ifdef HAS_PCBC
 752 static int ablk_pcbc_init(struct crypto_tfm *tfm)
 753 {
 754         struct cryptd_ablkcipher *cryptd_tfm;
 755
 756         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(pcbc(__driver-aes-aesni))",
 757                                              0, 0);
 758         if (IS_ERR(cryptd_tfm))
 759                 return PTR_ERR(cryptd_tfm);
 760         ablk_init_common(tfm, cryptd_tfm);
 761         return 0;
 762 }
 763
 764 static struct crypto_alg ablk_pcbc_alg = {
 765         .cra_name               = "pcbc(aes)",
 766         .cra_driver_name        = "pcbc-aes-aesni",
 767         .cra_priority           = 400,
 768         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 769         .cra_blocksize          = AES_BLOCK_SIZE,
 770         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 771         .cra_alignmask          = 0,
 772         .cra_type               = &crypto_ablkcipher_type,
 773         .cra_module             = THIS_MODULE,
 774         .cra_list               = LIST_HEAD_INIT(ablk_pcbc_alg.cra_list),
 775         .cra_init               = ablk_pcbc_init,
 776         .cra_exit               = ablk_exit,
 777         .cra_u = {
 778                 .ablkcipher = {
 779                         .min_keysize    = AES_MIN_KEY_SIZE,
 780                         .max_keysize    = AES_MAX_KEY_SIZE,
 781                         .ivsize         = AES_BLOCK_SIZE,
 782                         .setkey         = ablk_set_key,
 783                         .encrypt        = ablk_encrypt,
 784                         .decrypt        = ablk_decrypt,
 785                 },
 786         },
 787 };
 788 #endif
 789
 790 #ifdef HAS_XTS
 791 static int ablk_xts_init(struct crypto_tfm *tfm)
 792 {
 793         struct cryptd_ablkcipher *cryptd_tfm;
 794
 795         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(xts(__driver-aes-aesni))",
 796                                              0, 0);
 797         if (IS_ERR(cryptd_tfm))
 798                 return PTR_ERR(cryptd_tfm);
 799         ablk_init_common(tfm, cryptd_tfm);
 800         return 0;
 801 }
 802
 803 static struct crypto_alg ablk_xts_alg = {
 804         .cra_name               = "xts(aes)",
 805         .cra_driver_name        = "xts-aes-aesni",
 806         .cra_priority           = 400,
 807         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 808         .cra_blocksize          = AES_BLOCK_SIZE,
 809         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 810         .cra_alignmask          = 0,
 811         .cra_type               = &crypto_ablkcipher_type,
 812         .cra_module             = THIS_MODULE,
 813         .cra_list               = LIST_HEAD_INIT(ablk_xts_alg.cra_list),
 814         .cra_init               = ablk_xts_init,
 815         .cra_exit               = ablk_exit,
 816         .cra_u = {
 817                 .ablkcipher = {
 818                         .min_keysize    = 2 * AES_MIN_KEY_SIZE,
 819                         .max_keysize    = 2 * AES_MAX_KEY_SIZE,
 820                         .ivsize         = AES_BLOCK_SIZE,
 821                         .setkey         = ablk_set_key,
 822                         .encrypt        = ablk_encrypt,
 823                         .decrypt        = ablk_decrypt,
 824                 },
 825         },
 826 };
 827 #endif
 828
 829 #ifdef CONFIG_X86_64
 830 static int rfc4106_init(struct crypto_tfm *tfm)
 831 {
 832         struct cryptd_aead *cryptd_tfm;
 833         struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *)
 834                 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
 835         struct crypto_aead *cryptd_child;
 836         struct aesni_rfc4106_gcm_ctx *child_ctx;
 837         cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0);
 838         if (IS_ERR(cryptd_tfm))
 839                 return PTR_ERR(cryptd_tfm);
 840
 841         cryptd_child = cryptd_aead_child(cryptd_tfm);
 842         child_ctx = aesni_rfc4106_gcm_ctx_get(cryptd_child);
 843         memcpy(child_ctx, ctx, sizeof(*ctx));
 844         ctx->cryptd_tfm = cryptd_tfm;
 845         tfm->crt_aead.reqsize = sizeof(struct aead_request)
 846                 + crypto_aead_reqsize(&cryptd_tfm->base);
 847         return 0;
 848 }
 849
 850 static void rfc4106_exit(struct crypto_tfm *tfm)
 851 {
 852         struct aesni_rfc4106_gcm_ctx *ctx =
 853                 (struct aesni_rfc4106_gcm_ctx *)
 854                 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
 855         if (!IS_ERR(ctx->cryptd_tfm))
 856                 cryptd_free_aead(ctx->cryptd_tfm);
 857         return;
 858 }
 859
 860 static void
 861 rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err)
 862 {
 863         struct aesni_gcm_set_hash_subkey_result *result = req->data;
 864
 865         if (err == -EINPROGRESS)
 866                 return;
 867         result->err = err;
 868         complete(&result->completion);
 869 }
 870
 871 static int
 872 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
 873 {
 874         struct crypto_ablkcipher *ctr_tfm;
 875         struct ablkcipher_request *req;
 876         int ret = -EINVAL;
 877         struct aesni_hash_subkey_req_data *req_data;
 878
 879         ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0);
 880         if (IS_ERR(ctr_tfm))
 881                 return PTR_ERR(ctr_tfm);
 882
 883         crypto_ablkcipher_clear_flags(ctr_tfm, ~0);
 884
 885         ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len);
 886         if (ret)
 887                 goto out_free_ablkcipher;
 888
 889         ret = -ENOMEM;
 890         req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL);
 891         if (!req)
 892                 goto out_free_ablkcipher;
 893
 894         req_data = kmalloc(sizeof(*req_data), GFP_KERNEL);
 895         if (!req_data)
 896                 goto out_free_request;
 897
 898         memset(req_data->iv, 0, sizeof(req_data->iv));
 899
 900         /* Clear the data in the hash sub key container to zero.*/
 901         /* We want to cipher all zeros to create the hash sub key. */
 902         memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
 903
 904         init_completion(&req_data->result.completion);
 905         sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE);
 906         ablkcipher_request_set_tfm(req, ctr_tfm);
 907         ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
 908                                         CRYPTO_TFM_REQ_MAY_BACKLOG,
 909                                         rfc4106_set_hash_subkey_done,
 910                                         &req_data->result);
 911
 912         ablkcipher_request_set_crypt(req, &req_data->sg,
 913                 &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv);
 914
 915         ret = crypto_ablkcipher_encrypt(req);
 916         if (ret == -EINPROGRESS || ret == -EBUSY) {
 917                 ret = wait_for_completion_interruptible
 918                         (&req_data->result.completion);
 919                 if (!ret)
 920                         ret = req_data->result.err;
 921         }
 922         kfree(req_data);
 923 out_free_request:
 924         ablkcipher_request_free(req);
 925 out_free_ablkcipher:
 926         crypto_free_ablkcipher(ctr_tfm);
 927         return ret;
 928 }
 929
 930 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
 931                                                    unsigned int key_len)
 932 {
 933         int ret = 0;
 934         struct crypto_tfm *tfm = crypto_aead_tfm(parent);
 935         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
 936         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
 937         struct aesni_rfc4106_gcm_ctx *child_ctx =
 938                                  aesni_rfc4106_gcm_ctx_get(cryptd_child);
 939         u8 *new_key_mem = NULL;
 940
 941         if (key_len < 4) {
 942                 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 943                 return -EINVAL;
 944         }
 945         /*Account for 4 byte nonce at the end.*/
 946         key_len -= 4;
 947         if (key_len != AES_KEYSIZE_128) {
 948                 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 949                 return -EINVAL;
 950         }
 951
 952         memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
 953         /*This must be on a 16 byte boundary!*/
 954         if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN)
 955                 return -EINVAL;
 956
 957         if ((unsigned long)key % AESNI_ALIGN) {
 958                 /*key is not aligned: use an auxuliar aligned pointer*/
 959                 new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL);
 960                 if (!new_key_mem)
 961                         return -ENOMEM;
 962
 963                 new_key_mem = PTR_ALIGN(new_key_mem, AESNI_ALIGN);
 964                 memcpy(new_key_mem, key, key_len);
 965                 key = new_key_mem;
 966         }
 967
 968         if (!irq_fpu_usable())
 969                 ret = crypto_aes_expand_key(&(ctx->aes_key_expanded),
 970                 key, key_len);
 971         else {
 972                 kernel_fpu_begin();
 973                 ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len);
 974                 kernel_fpu_end();
 975         }
 976         /*This must be on a 16 byte boundary!*/
 977         if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) {
 978                 ret = -EINVAL;
 979                 goto exit;
 980         }
 981         ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
 982         memcpy(child_ctx, ctx, sizeof(*ctx));
 983 exit:
 984         kfree(new_key_mem);
 985         return ret;
 986 }
 987
 988 /* This is the Integrity Check Value (aka the authentication tag length and can
 989  * be 8, 12 or 16 bytes long. */
 990 static int rfc4106_set_authsize(struct crypto_aead *parent,
 991                                 unsigned int authsize)
 992 {
 993         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
 994         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
 995
 996         switch (authsize) {
 997         case 8:
 998         case 12:
 999         case 16:
1000                 break;
1001         default:
1002                 return -EINVAL;
1003         }
1004         crypto_aead_crt(parent)->authsize = authsize;
1005         crypto_aead_crt(cryptd_child)->authsize = authsize;
1006         return 0;
1007 }
1008
1009 static int rfc4106_encrypt(struct aead_request *req)
1010 {
1011         int ret;
1012         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1013         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1014
1015         if (!irq_fpu_usable()) {
1016                 struct aead_request *cryptd_req =
1017                         (struct aead_request *) aead_request_ctx(req);
1018                 memcpy(cryptd_req, req, sizeof(*req));
1019                 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
1020                 return crypto_aead_encrypt(cryptd_req);
1021         } else {
1022                 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
1023                 kernel_fpu_begin();
1024                 ret = cryptd_child->base.crt_aead.encrypt(req);
1025                 kernel_fpu_end();
1026                 return ret;
1027         }
1028 }
1029
1030 static int rfc4106_decrypt(struct aead_request *req)
1031 {
1032         int ret;
1033         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1034         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1035
1036         if (!irq_fpu_usable()) {
1037                 struct aead_request *cryptd_req =
1038                         (struct aead_request *) aead_request_ctx(req);
1039                 memcpy(cryptd_req, req, sizeof(*req));
1040                 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
1041                 return crypto_aead_decrypt(cryptd_req);
1042         } else {
1043                 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
1044                 kernel_fpu_begin();
1045                 ret = cryptd_child->base.crt_aead.decrypt(req);
1046                 kernel_fpu_end();
1047                 return ret;
1048         }
1049 }
1050
1051 static struct crypto_alg rfc4106_alg = {
1052         .cra_name = "rfc4106(gcm(aes))",
1053         .cra_driver_name = "rfc4106-gcm-aesni",
1054         .cra_priority = 400,
1055         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1056         .cra_blocksize = 1,
1057         .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN,
1058         .cra_alignmask = 0,
1059         .cra_type = &crypto_nivaead_type,
1060         .cra_module = THIS_MODULE,
1061         .cra_list = LIST_HEAD_INIT(rfc4106_alg.cra_list),
1062         .cra_init = rfc4106_init,
1063         .cra_exit = rfc4106_exit,
1064         .cra_u = {
1065                 .aead = {
1066                         .setkey = rfc4106_set_key,
1067                         .setauthsize = rfc4106_set_authsize,
1068                         .encrypt = rfc4106_encrypt,
1069                         .decrypt = rfc4106_decrypt,
1070                         .geniv = "seqiv",
1071                         .ivsize = 8,
1072                         .maxauthsize = 16,
1073                 },
1074         },
1075 };
1076
1077 static int __driver_rfc4106_encrypt(struct aead_request *req)
1078 {
1079         u8 one_entry_in_sg = 0;
1080         u8 *src, *dst, *assoc;
1081         __be32 counter = cpu_to_be32(1);
1082         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1083         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1084         void *aes_ctx = &(ctx->aes_key_expanded);
1085         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
1086         u8 iv_tab[16+AESNI_ALIGN];
1087         u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN);
1088         struct scatter_walk src_sg_walk;
1089         struct scatter_walk assoc_sg_walk;
1090         struct scatter_walk dst_sg_walk;
1091         unsigned int i;
1092
1093         /* Assuming we are supporting rfc4106 64-bit extended */
1094         /* sequence numbers We need to have the AAD length equal */
1095         /* to 8 or 12 bytes */
1096         if (unlikely(req->assoclen != 8 && req->assoclen != 12))
1097                 return -EINVAL;
1098         /* IV below built */
1099         for (i = 0; i < 4; i++)
1100                 *(iv+i) = ctx->nonce[i];
1101         for (i = 0; i < 8; i++)
1102                 *(iv+4+i) = req->iv[i];
1103         *((__be32 *)(iv+12)) = counter;
1104
1105         if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
1106                 one_entry_in_sg = 1;
1107                 scatterwalk_start(&src_sg_walk, req->src);
1108                 scatterwalk_start(&assoc_sg_walk, req->assoc);
1109                 src = scatterwalk_map(&src_sg_walk, 0);
1110                 assoc = scatterwalk_map(&assoc_sg_walk, 0);
1111                 dst = src;
1112                 if (unlikely(req->src != req->dst)) {
1113                         scatterwalk_start(&dst_sg_walk, req->dst);
1114                         dst = scatterwalk_map(&dst_sg_walk, 0);
1115                 }
1116
1117         } else {
1118                 /* Allocate memory for src, dst, assoc */
1119                 src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
1120                         GFP_ATOMIC);
1121                 if (unlikely(!src))
1122                         return -ENOMEM;
1123                 assoc = (src + req->cryptlen + auth_tag_len);
1124                 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
1125                 scatterwalk_map_and_copy(assoc, req->assoc, 0,
1126                                         req->assoclen, 0);
1127                 dst = src;
1128         }
1129
1130         aesni_gcm_enc(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv,
1131                 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst
1132                 + ((unsigned long)req->cryptlen), auth_tag_len);
1133
1134         /* The authTag (aka the Integrity Check Value) needs to be written
1135          * back to the packet. */
1136         if (one_entry_in_sg) {
1137                 if (unlikely(req->src != req->dst)) {
1138                         scatterwalk_unmap(dst, 0);
1139                         scatterwalk_done(&dst_sg_walk, 0, 0);
1140                 }
1141                 scatterwalk_unmap(src, 0);
1142                 scatterwalk_unmap(assoc, 0);
1143                 scatterwalk_done(&src_sg_walk, 0, 0);
1144                 scatterwalk_done(&assoc_sg_walk, 0, 0);
1145         } else {
1146                 scatterwalk_map_and_copy(dst, req->dst, 0,
1147                         req->cryptlen + auth_tag_len, 1);
1148                 kfree(src);
1149         }
1150         return 0;
1151 }
1152
1153 static int __driver_rfc4106_decrypt(struct aead_request *req)
1154 {
1155         u8 one_entry_in_sg = 0;
1156         u8 *src, *dst, *assoc;
1157         unsigned long tempCipherLen = 0;
1158         __be32 counter = cpu_to_be32(1);
1159         int retval = 0;
1160         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1161         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1162         void *aes_ctx = &(ctx->aes_key_expanded);
1163         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
1164         u8 iv_and_authTag[32+AESNI_ALIGN];
1165         u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN);
1166         u8 *authTag = iv + 16;
1167         struct scatter_walk src_sg_walk;
1168         struct scatter_walk assoc_sg_walk;
1169         struct scatter_walk dst_sg_walk;
1170         unsigned int i;
1171
1172         if (unlikely((req->cryptlen < auth_tag_len) ||
1173                 (req->assoclen != 8 && req->assoclen != 12)))
1174                 return -EINVAL;
1175         /* Assuming we are supporting rfc4106 64-bit extended */
1176         /* sequence numbers We need to have the AAD length */
1177         /* equal to 8 or 12 bytes */
1178
1179         tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
1180         /* IV below built */
1181         for (i = 0; i < 4; i++)
1182                 *(iv+i) = ctx->nonce[i];
1183         for (i = 0; i < 8; i++)
1184                 *(iv+4+i) = req->iv[i];
1185         *((__be32 *)(iv+12)) = counter;
1186
1187         if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
1188                 one_entry_in_sg = 1;
1189                 scatterwalk_start(&src_sg_walk, req->src);
1190                 scatterwalk_start(&assoc_sg_walk, req->assoc);
1191                 src = scatterwalk_map(&src_sg_walk, 0);
1192                 assoc = scatterwalk_map(&assoc_sg_walk, 0);
1193                 dst = src;
1194                 if (unlikely(req->src != req->dst)) {
1195                         scatterwalk_start(&dst_sg_walk, req->dst);
1196                         dst = scatterwalk_map(&dst_sg_walk, 0);
1197                 }
1198
1199         } else {
1200                 /* Allocate memory for src, dst, assoc */
1201                 src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
1202                 if (!src)
1203                         return -ENOMEM;
1204                 assoc = (src + req->cryptlen + auth_tag_len);
1205                 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
1206                 scatterwalk_map_and_copy(assoc, req->assoc, 0,
1207                         req->assoclen, 0);
1208                 dst = src;
1209         }
1210
1211         aesni_gcm_dec(aes_ctx, dst, src, tempCipherLen, iv,
1212                 ctx->hash_subkey, assoc, (unsigned long)req->assoclen,
1213                 authTag, auth_tag_len);
1214
1215         /* Compare generated tag with passed in tag. */
1216         retval = memcmp(src + tempCipherLen, authTag, auth_tag_len) ?
1217                 -EBADMSG : 0;
1218
1219         if (one_entry_in_sg) {
1220                 if (unlikely(req->src != req->dst)) {
1221                         scatterwalk_unmap(dst, 0);
1222                         scatterwalk_done(&dst_sg_walk, 0, 0);
1223                 }
1224                 scatterwalk_unmap(src, 0);
1225                 scatterwalk_unmap(assoc, 0);
1226                 scatterwalk_done(&src_sg_walk, 0, 0);
1227                 scatterwalk_done(&assoc_sg_walk, 0, 0);
1228         } else {
1229                 scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1);
1230                 kfree(src);
1231         }
1232         return retval;
1233 }
1234
1235 static struct crypto_alg __rfc4106_alg = {
1236         .cra_name               = "__gcm-aes-aesni",
1237         .cra_driver_name        = "__driver-gcm-aes-aesni",
1238         .cra_priority           = 0,
1239         .cra_flags              = CRYPTO_ALG_TYPE_AEAD,
1240         .cra_blocksize          = 1,
1241         .cra_ctxsize    = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN,
1242         .cra_alignmask          = 0,
1243         .cra_type               = &crypto_aead_type,
1244         .cra_module             = THIS_MODULE,
1245         .cra_list               = LIST_HEAD_INIT(__rfc4106_alg.cra_list),
1246         .cra_u = {
1247                 .aead = {
1248                         .encrypt        = __driver_rfc4106_encrypt,
1249                         .decrypt        = __driver_rfc4106_decrypt,
1250                 },
1251         },
1252 };
1253 #endif
1254
1255 static int __init aesni_init(void)
1256 {
1257         int err;
1258
1259         if (!cpu_has_aes) {
1260                 printk(KERN_INFO "Intel AES-NI instructions are not detected.\n");
1261                 return -ENODEV;
1262         }
1263
1264         if ((err = crypto_fpu_init()))
1265                 goto fpu_err;
1266         if ((err = crypto_register_alg(&aesni_alg)))
1267                 goto aes_err;
1268         if ((err = crypto_register_alg(&__aesni_alg)))
1269                 goto __aes_err;
1270         if ((err = crypto_register_alg(&blk_ecb_alg)))
1271                 goto blk_ecb_err;
1272         if ((err = crypto_register_alg(&blk_cbc_alg)))
1273                 goto blk_cbc_err;
1274         if ((err = crypto_register_alg(&ablk_ecb_alg)))
1275                 goto ablk_ecb_err;
1276         if ((err = crypto_register_alg(&ablk_cbc_alg)))
1277                 goto ablk_cbc_err;
1278 #ifdef CONFIG_X86_64
1279         if ((err = crypto_register_alg(&blk_ctr_alg)))
1280                 goto blk_ctr_err;
1281         if ((err = crypto_register_alg(&ablk_ctr_alg)))
1282                 goto ablk_ctr_err;
1283         if ((err = crypto_register_alg(&__rfc4106_alg)))
1284                 goto __aead_gcm_err;
1285         if ((err = crypto_register_alg(&rfc4106_alg)))
1286                 goto aead_gcm_err;
1287 #ifdef HAS_CTR
1288         if ((err = crypto_register_alg(&ablk_rfc3686_ctr_alg)))
1289                 goto ablk_rfc3686_ctr_err;
1290 #endif
1291 #endif
1292 #ifdef HAS_LRW
1293         if ((err = crypto_register_alg(&ablk_lrw_alg)))
1294                 goto ablk_lrw_err;
1295 #endif
1296 #ifdef HAS_PCBC
1297         if ((err = crypto_register_alg(&ablk_pcbc_alg)))
1298                 goto ablk_pcbc_err;
1299 #endif
1300 #ifdef HAS_XTS
1301         if ((err = crypto_register_alg(&ablk_xts_alg)))
1302                 goto ablk_xts_err;
1303 #endif
1304         return err;
1305
1306 #ifdef HAS_XTS
1307 ablk_xts_err:
1308 #endif
1309 #ifdef HAS_PCBC
1310         crypto_unregister_alg(&ablk_pcbc_alg);
1311 ablk_pcbc_err:
1312 #endif
1313 #ifdef HAS_LRW
1314         crypto_unregister_alg(&ablk_lrw_alg);
1315 ablk_lrw_err:
1316 #endif
1317 #ifdef CONFIG_X86_64
1318 #ifdef HAS_CTR
1319         crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
1320 ablk_rfc3686_ctr_err:
1321 #endif
1322         crypto_unregister_alg(&rfc4106_alg);
1323 aead_gcm_err:
1324         crypto_unregister_alg(&__rfc4106_alg);
1325 __aead_gcm_err:
1326         crypto_unregister_alg(&ablk_ctr_alg);
1327 ablk_ctr_err:
1328         crypto_unregister_alg(&blk_ctr_alg);
1329 blk_ctr_err:
1330 #endif
1331         crypto_unregister_alg(&ablk_cbc_alg);
1332 ablk_cbc_err:
1333         crypto_unregister_alg(&ablk_ecb_alg);
1334 ablk_ecb_err:
1335         crypto_unregister_alg(&blk_cbc_alg);
1336 blk_cbc_err:
1337         crypto_unregister_alg(&blk_ecb_alg);
1338 blk_ecb_err:
1339         crypto_unregister_alg(&__aesni_alg);
1340 __aes_err:
1341         crypto_unregister_alg(&aesni_alg);
1342 aes_err:
1343 fpu_err:
1344         return err;
1345 }
1346
1347 static void __exit aesni_exit(void)
1348 {
1349 #ifdef HAS_XTS
1350         crypto_unregister_alg(&ablk_xts_alg);
1351 #endif
1352 #ifdef HAS_PCBC
1353         crypto_unregister_alg(&ablk_pcbc_alg);
1354 #endif
1355 #ifdef HAS_LRW
1356         crypto_unregister_alg(&ablk_lrw_alg);
1357 #endif
1358 #ifdef CONFIG_X86_64
1359 #ifdef HAS_CTR
1360         crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
1361 #endif
1362         crypto_unregister_alg(&rfc4106_alg);
1363         crypto_unregister_alg(&__rfc4106_alg);
1364         crypto_unregister_alg(&ablk_ctr_alg);
1365         crypto_unregister_alg(&blk_ctr_alg);
1366 #endif
1367         crypto_unregister_alg(&ablk_cbc_alg);
1368         crypto_unregister_alg(&ablk_ecb_alg);
1369         crypto_unregister_alg(&blk_cbc_alg);
1370         crypto_unregister_alg(&blk_ecb_alg);
1371         crypto_unregister_alg(&__aesni_alg);
1372         crypto_unregister_alg(&aesni_alg);
1373
1374         crypto_fpu_exit();
1375 }
1376
1377 module_init(aesni_init);
1378 module_exit(aesni_exit);
1379
1380 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1381 MODULE_LICENSE("GPL");
1382 MODULE_ALIAS("aes");