arch/x86/crypto/aesni-intel_glue.c

   1 /*
   2  * Support for Intel AES-NI instructions. This file contains glue
   3  * code, the real AES implementation is in intel-aes_asm.S.
   4  *
   5  * Copyright (C) 2008, Intel Corp.
   6  *    Author: Huang Ying <ying.huang@intel.com>
   7  *
   8  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
   9  * interface for 64-bit kernels.
  10  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
  11  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
  12  *             Tadeusz Struk (tadeusz.struk@intel.com)
  13  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
  14  *    Copyright (c) 2010, Intel Corporation.
  15  *
  16  * This program is free software; you can redistribute it and/or modify
  17  * it under the terms of the GNU General Public License as published by
  18  * the Free Software Foundation; either version 2 of the License, or
  19  * (at your option) any later version.
  20  */
  21
  22 #include <linux/hardirq.h>
  23 #include <linux/types.h>
  24 #include <linux/crypto.h>
  25 #include <linux/err.h>
  26 #include <crypto/algapi.h>
  27 #include <crypto/aes.h>
  28 #include <crypto/cryptd.h>
  29 #include <crypto/ctr.h>
  30 #include <asm/i387.h>
  31 #include <asm/aes.h>
  32 #include <crypto/scatterwalk.h>
  33 #include <crypto/internal/aead.h>
  34 #include <linux/workqueue.h>
  35 #include <linux/spinlock.h>
  36
  37 #if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE)
  38 #define HAS_CTR
  39 #endif
  40
  41 #if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE)
  42 #define HAS_LRW
  43 #endif
  44
  45 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE)
  46 #define HAS_PCBC
  47 #endif
  48
  49 #if defined(CONFIG_CRYPTO_XTS) || defined(CONFIG_CRYPTO_XTS_MODULE)
  50 #define HAS_XTS
  51 #endif
  52
  53 struct async_aes_ctx {
  54         struct cryptd_ablkcipher *cryptd_tfm;
  55 };
  56
  57 /* This data is stored at the end of the crypto_tfm struct.
  58  * It's a type of per "session" data storage location.
  59  * This needs to be 16 byte aligned.
  60  */
  61 struct aesni_rfc4106_gcm_ctx {
  62         u8 hash_subkey[16];
  63         struct crypto_aes_ctx aes_key_expanded;
  64         u8 nonce[4];
  65         struct cryptd_aead *cryptd_tfm;
  66 };
  67
  68 struct aesni_gcm_set_hash_subkey_result {
  69         int err;
  70         struct completion completion;
  71 };
  72
  73 struct aesni_hash_subkey_req_data {
  74         u8 iv[16];
  75         struct aesni_gcm_set_hash_subkey_result result;
  76         struct scatterlist sg;
  77 };
  78
  79 #define AESNI_ALIGN     (16)
  80 #define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE-1))
  81 #define RFC4106_HASH_SUBKEY_SIZE 16
  82
  83 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
  84                              unsigned int key_len);
  85 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
  86                           const u8 *in);
  87 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
  88                           const u8 *in);
  89 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
  90                               const u8 *in, unsigned int len);
  91 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
  92                               const u8 *in, unsigned int len);
  93 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
  94                               const u8 *in, unsigned int len, u8 *iv);
  95 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
  96                               const u8 *in, unsigned int len, u8 *iv);
  97 #ifdef CONFIG_X86_64
  98 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
  99                               const u8 *in, unsigned int len, u8 *iv);
 100
 101 /* asmlinkage void aesni_gcm_enc()
 102  * void *ctx,  AES Key schedule. Starts on a 16 byte boundary.
 103  * u8 *out, Ciphertext output. Encrypt in-place is allowed.
 104  * const u8 *in, Plaintext input
 105  * unsigned long plaintext_len, Length of data in bytes for encryption.
 106  * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
 107  *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
 108  *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
 109  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 110  * const u8 *aad, Additional Authentication Data (AAD)
 111  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this
 112  *          is going to be 8 or 12 bytes
 113  * u8 *auth_tag, Authenticated Tag output.
 114  * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
 115  *          Valid values are 16 (most likely), 12 or 8.
 116  */
 117 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
 118                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 119                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 120                         u8 *auth_tag, unsigned long auth_tag_len);
 121
 122 /* asmlinkage void aesni_gcm_dec()
 123  * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
 124  * u8 *out, Plaintext output. Decrypt in-place is allowed.
 125  * const u8 *in, Ciphertext input
 126  * unsigned long ciphertext_len, Length of data in bytes for decryption.
 127  * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
 128  *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
 129  *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
 130  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 131  * const u8 *aad, Additional Authentication Data (AAD)
 132  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
 133  * to be 8 or 12 bytes
 134  * u8 *auth_tag, Authenticated Tag output.
 135  * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
 136  * Valid values are 16 (most likely), 12 or 8.
 137  */
 138 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
 139                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 140                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 141                         u8 *auth_tag, unsigned long auth_tag_len);
 142
 143 static inline struct
 144 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
 145 {
 146         return
 147                 (struct aesni_rfc4106_gcm_ctx *)
 148                 PTR_ALIGN((u8 *)
 149                 crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN);
 150 }
 151 #endif
 152
 153 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
 154 {
 155         unsigned long addr = (unsigned long)raw_ctx;
 156         unsigned long align = AESNI_ALIGN;
 157
 158         if (align <= crypto_tfm_ctx_alignment())
 159                 align = 1;
 160         return (struct crypto_aes_ctx *)ALIGN(addr, align);
 161 }
 162
 163 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
 164                               const u8 *in_key, unsigned int key_len)
 165 {
 166         struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
 167         u32 *flags = &tfm->crt_flags;
 168         int err;
 169
 170         if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
 171             key_len != AES_KEYSIZE_256) {
 172                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 173                 return -EINVAL;
 174         }
 175
 176         if (!irq_fpu_usable())
 177                 err = crypto_aes_expand_key(ctx, in_key, key_len);
 178         else {
 179                 kernel_fpu_begin();
 180                 err = aesni_set_key(ctx, in_key, key_len);
 181                 kernel_fpu_end();
 182         }
 183
 184         return err;
 185 }
 186
 187 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 188                        unsigned int key_len)
 189 {
 190         return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
 191 }
 192
 193 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 194 {
 195         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 196
 197         if (!irq_fpu_usable())
 198                 crypto_aes_encrypt_x86(ctx, dst, src);
 199         else {
 200                 kernel_fpu_begin();
 201                 aesni_enc(ctx, dst, src);
 202                 kernel_fpu_end();
 203         }
 204 }
 205
 206 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 207 {
 208         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 209
 210         if (!irq_fpu_usable())
 211                 crypto_aes_decrypt_x86(ctx, dst, src);
 212         else {
 213                 kernel_fpu_begin();
 214                 aesni_dec(ctx, dst, src);
 215                 kernel_fpu_end();
 216         }
 217 }
 218
 219 static struct crypto_alg aesni_alg = {
 220         .cra_name               = "aes",
 221         .cra_driver_name        = "aes-aesni",
 222         .cra_priority           = 300,
 223         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
 224         .cra_blocksize          = AES_BLOCK_SIZE,
 225         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
 226         .cra_alignmask          = 0,
 227         .cra_module             = THIS_MODULE,
 228         .cra_list               = LIST_HEAD_INIT(aesni_alg.cra_list),
 229         .cra_u  = {
 230                 .cipher = {
 231                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
 232                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
 233                         .cia_setkey             = aes_set_key,
 234                         .cia_encrypt            = aes_encrypt,
 235                         .cia_decrypt            = aes_decrypt
 236                 }
 237         }
 238 };
 239
 240 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 241 {
 242         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 243
 244         aesni_enc(ctx, dst, src);
 245 }
 246
 247 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 248 {
 249         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 250
 251         aesni_dec(ctx, dst, src);
 252 }
 253
 254 static struct crypto_alg __aesni_alg = {
 255         .cra_name               = "__aes-aesni",
 256         .cra_driver_name        = "__driver-aes-aesni",
 257         .cra_priority           = 0,
 258         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
 259         .cra_blocksize          = AES_BLOCK_SIZE,
 260         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
 261         .cra_alignmask          = 0,
 262         .cra_module             = THIS_MODULE,
 263         .cra_list               = LIST_HEAD_INIT(__aesni_alg.cra_list),
 264         .cra_u  = {
 265                 .cipher = {
 266                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
 267                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
 268                         .cia_setkey             = aes_set_key,
 269                         .cia_encrypt            = __aes_encrypt,
 270                         .cia_decrypt            = __aes_decrypt
 271                 }
 272         }
 273 };
 274
 275 static int ecb_encrypt(struct blkcipher_desc *desc,
 276                        struct scatterlist *dst, struct scatterlist *src,
 277                        unsigned int nbytes)
 278 {
 279         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 280         struct blkcipher_walk walk;
 281         int err;
 282
 283         blkcipher_walk_init(&walk, dst, src, nbytes);
 284         err = blkcipher_walk_virt(desc, &walk);
 285         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 286
 287         kernel_fpu_begin();
 288         while ((nbytes = walk.nbytes)) {
 289                 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 290                               nbytes & AES_BLOCK_MASK);
 291                 nbytes &= AES_BLOCK_SIZE - 1;
 292                 err = blkcipher_walk_done(desc, &walk, nbytes);
 293         }
 294         kernel_fpu_end();
 295
 296         return err;
 297 }
 298
 299 static int ecb_decrypt(struct blkcipher_desc *desc,
 300                        struct scatterlist *dst, struct scatterlist *src,
 301                        unsigned int nbytes)
 302 {
 303         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 304         struct blkcipher_walk walk;
 305         int err;
 306
 307         blkcipher_walk_init(&walk, dst, src, nbytes);
 308         err = blkcipher_walk_virt(desc, &walk);
 309         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 310
 311         kernel_fpu_begin();
 312         while ((nbytes = walk.nbytes)) {
 313                 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 314                               nbytes & AES_BLOCK_MASK);
 315                 nbytes &= AES_BLOCK_SIZE - 1;
 316                 err = blkcipher_walk_done(desc, &walk, nbytes);
 317         }
 318         kernel_fpu_end();
 319
 320         return err;
 321 }
 322
 323 static struct crypto_alg blk_ecb_alg = {
 324         .cra_name               = "__ecb-aes-aesni",
 325         .cra_driver_name        = "__driver-ecb-aes-aesni",
 326         .cra_priority           = 0,
 327         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 328         .cra_blocksize          = AES_BLOCK_SIZE,
 329         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
 330         .cra_alignmask          = 0,
 331         .cra_type               = &crypto_blkcipher_type,
 332         .cra_module             = THIS_MODULE,
 333         .cra_list               = LIST_HEAD_INIT(blk_ecb_alg.cra_list),
 334         .cra_u = {
 335                 .blkcipher = {
 336                         .min_keysize    = AES_MIN_KEY_SIZE,
 337                         .max_keysize    = AES_MAX_KEY_SIZE,
 338                         .setkey         = aes_set_key,
 339                         .encrypt        = ecb_encrypt,
 340                         .decrypt        = ecb_decrypt,
 341                 },
 342         },
 343 };
 344
 345 static int cbc_encrypt(struct blkcipher_desc *desc,
 346                        struct scatterlist *dst, struct scatterlist *src,
 347                        unsigned int nbytes)
 348 {
 349         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 350         struct blkcipher_walk walk;
 351         int err;
 352
 353         blkcipher_walk_init(&walk, dst, src, nbytes);
 354         err = blkcipher_walk_virt(desc, &walk);
 355         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 356
 357         kernel_fpu_begin();
 358         while ((nbytes = walk.nbytes)) {
 359                 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 360                               nbytes & AES_BLOCK_MASK, walk.iv);
 361                 nbytes &= AES_BLOCK_SIZE - 1;
 362                 err = blkcipher_walk_done(desc, &walk, nbytes);
 363         }
 364         kernel_fpu_end();
 365
 366         return err;
 367 }
 368
 369 static int cbc_decrypt(struct blkcipher_desc *desc,
 370                        struct scatterlist *dst, struct scatterlist *src,
 371                        unsigned int nbytes)
 372 {
 373         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 374         struct blkcipher_walk walk;
 375         int err;
 376
 377         blkcipher_walk_init(&walk, dst, src, nbytes);
 378         err = blkcipher_walk_virt(desc, &walk);
 379         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 380
 381         kernel_fpu_begin();
 382         while ((nbytes = walk.nbytes)) {
 383                 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 384                               nbytes & AES_BLOCK_MASK, walk.iv);
 385                 nbytes &= AES_BLOCK_SIZE - 1;
 386                 err = blkcipher_walk_done(desc, &walk, nbytes);
 387         }
 388         kernel_fpu_end();
 389
 390         return err;
 391 }
 392
 393 static struct crypto_alg blk_cbc_alg = {
 394         .cra_name               = "__cbc-aes-aesni",
 395         .cra_driver_name        = "__driver-cbc-aes-aesni",
 396         .cra_priority           = 0,
 397         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 398         .cra_blocksize          = AES_BLOCK_SIZE,
 399         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
 400         .cra_alignmask          = 0,
 401         .cra_type               = &crypto_blkcipher_type,
 402         .cra_module             = THIS_MODULE,
 403         .cra_list               = LIST_HEAD_INIT(blk_cbc_alg.cra_list),
 404         .cra_u = {
 405                 .blkcipher = {
 406                         .min_keysize    = AES_MIN_KEY_SIZE,
 407                         .max_keysize    = AES_MAX_KEY_SIZE,
 408                         .setkey         = aes_set_key,
 409                         .encrypt        = cbc_encrypt,
 410                         .decrypt        = cbc_decrypt,
 411                 },
 412         },
 413 };
 414
 415 #ifdef CONFIG_X86_64
 416 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
 417                             struct blkcipher_walk *walk)
 418 {
 419         u8 *ctrblk = walk->iv;
 420         u8 keystream[AES_BLOCK_SIZE];
 421         u8 *src = walk->src.virt.addr;
 422         u8 *dst = walk->dst.virt.addr;
 423         unsigned int nbytes = walk->nbytes;
 424
 425         aesni_enc(ctx, keystream, ctrblk);
 426         crypto_xor(keystream, src, nbytes);
 427         memcpy(dst, keystream, nbytes);
 428         crypto_inc(ctrblk, AES_BLOCK_SIZE);
 429 }
 430
 431 static int ctr_crypt(struct blkcipher_desc *desc,
 432                      struct scatterlist *dst, struct scatterlist *src,
 433                      unsigned int nbytes)
 434 {
 435         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 436         struct blkcipher_walk walk;
 437         int err;
 438
 439         blkcipher_walk_init(&walk, dst, src, nbytes);
 440         err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
 441         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 442
 443         kernel_fpu_begin();
 444         while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
 445                 aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 446                               nbytes & AES_BLOCK_MASK, walk.iv);
 447                 nbytes &= AES_BLOCK_SIZE - 1;
 448                 err = blkcipher_walk_done(desc, &walk, nbytes);
 449         }
 450         if (walk.nbytes) {
 451                 ctr_crypt_final(ctx, &walk);
 452                 err = blkcipher_walk_done(desc, &walk, 0);
 453         }
 454         kernel_fpu_end();
 455
 456         return err;
 457 }
 458
 459 static struct crypto_alg blk_ctr_alg = {
 460         .cra_name               = "__ctr-aes-aesni",
 461         .cra_driver_name        = "__driver-ctr-aes-aesni",
 462         .cra_priority           = 0,
 463         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 464         .cra_blocksize          = 1,
 465         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
 466         .cra_alignmask          = 0,
 467         .cra_type               = &crypto_blkcipher_type,
 468         .cra_module             = THIS_MODULE,
 469         .cra_list               = LIST_HEAD_INIT(blk_ctr_alg.cra_list),
 470         .cra_u = {
 471                 .blkcipher = {
 472                         .min_keysize    = AES_MIN_KEY_SIZE,
 473                         .max_keysize    = AES_MAX_KEY_SIZE,
 474                         .ivsize         = AES_BLOCK_SIZE,
 475                         .setkey         = aes_set_key,
 476                         .encrypt        = ctr_crypt,
 477                         .decrypt        = ctr_crypt,
 478                 },
 479         },
 480 };
 481 #endif
 482
 483 static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
 484                         unsigned int key_len)
 485 {
 486         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 487         struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base;
 488         int err;
 489
 490         crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 491         crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm)
 492                                     & CRYPTO_TFM_REQ_MASK);
 493         err = crypto_ablkcipher_setkey(child, key, key_len);
 494         crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child)
 495                                     & CRYPTO_TFM_RES_MASK);
 496         return err;
 497 }
 498
 499 static int ablk_encrypt(struct ablkcipher_request *req)
 500 {
 501         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 502         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 503
 504         if (!irq_fpu_usable()) {
 505                 struct ablkcipher_request *cryptd_req =
 506                         ablkcipher_request_ctx(req);
 507                 memcpy(cryptd_req, req, sizeof(*req));
 508                 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
 509                 return crypto_ablkcipher_encrypt(cryptd_req);
 510         } else {
 511                 struct blkcipher_desc desc;
 512                 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
 513                 desc.info = req->info;
 514                 desc.flags = 0;
 515                 return crypto_blkcipher_crt(desc.tfm)->encrypt(
 516                         &desc, req->dst, req->src, req->nbytes);
 517         }
 518 }
 519
 520 static int ablk_decrypt(struct ablkcipher_request *req)
 521 {
 522         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 523         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 524
 525         if (!irq_fpu_usable()) {
 526                 struct ablkcipher_request *cryptd_req =
 527                         ablkcipher_request_ctx(req);
 528                 memcpy(cryptd_req, req, sizeof(*req));
 529                 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
 530                 return crypto_ablkcipher_decrypt(cryptd_req);
 531         } else {
 532                 struct blkcipher_desc desc;
 533                 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
 534                 desc.info = req->info;
 535                 desc.flags = 0;
 536                 return crypto_blkcipher_crt(desc.tfm)->decrypt(
 537                         &desc, req->dst, req->src, req->nbytes);
 538         }
 539 }
 540
 541 static void ablk_exit(struct crypto_tfm *tfm)
 542 {
 543         struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 544
 545         cryptd_free_ablkcipher(ctx->cryptd_tfm);
 546 }
 547
 548 static void ablk_init_common(struct crypto_tfm *tfm,
 549                              struct cryptd_ablkcipher *cryptd_tfm)
 550 {
 551         struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 552
 553         ctx->cryptd_tfm = cryptd_tfm;
 554         tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
 555                 crypto_ablkcipher_reqsize(&cryptd_tfm->base);
 556 }
 557
 558 static int ablk_ecb_init(struct crypto_tfm *tfm)
 559 {
 560         struct cryptd_ablkcipher *cryptd_tfm;
 561
 562         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ecb-aes-aesni", 0, 0);
 563         if (IS_ERR(cryptd_tfm))
 564                 return PTR_ERR(cryptd_tfm);
 565         ablk_init_common(tfm, cryptd_tfm);
 566         return 0;
 567 }
 568
 569 static struct crypto_alg ablk_ecb_alg = {
 570         .cra_name               = "ecb(aes)",
 571         .cra_driver_name        = "ecb-aes-aesni",
 572         .cra_priority           = 400,
 573         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 574         .cra_blocksize          = AES_BLOCK_SIZE,
 575         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 576         .cra_alignmask          = 0,
 577         .cra_type               = &crypto_ablkcipher_type,
 578         .cra_module             = THIS_MODULE,
 579         .cra_list               = LIST_HEAD_INIT(ablk_ecb_alg.cra_list),
 580         .cra_init               = ablk_ecb_init,
 581         .cra_exit               = ablk_exit,
 582         .cra_u = {
 583                 .ablkcipher = {
 584                         .min_keysize    = AES_MIN_KEY_SIZE,
 585                         .max_keysize    = AES_MAX_KEY_SIZE,
 586                         .setkey         = ablk_set_key,
 587                         .encrypt        = ablk_encrypt,
 588                         .decrypt        = ablk_decrypt,
 589                 },
 590         },
 591 };
 592
 593 static int ablk_cbc_init(struct crypto_tfm *tfm)
 594 {
 595         struct cryptd_ablkcipher *cryptd_tfm;
 596
 597         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-cbc-aes-aesni", 0, 0);
 598         if (IS_ERR(cryptd_tfm))
 599                 return PTR_ERR(cryptd_tfm);
 600         ablk_init_common(tfm, cryptd_tfm);
 601         return 0;
 602 }
 603
 604 static struct crypto_alg ablk_cbc_alg = {
 605         .cra_name               = "cbc(aes)",
 606         .cra_driver_name        = "cbc-aes-aesni",
 607         .cra_priority           = 400,
 608         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 609         .cra_blocksize          = AES_BLOCK_SIZE,
 610         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 611         .cra_alignmask          = 0,
 612         .cra_type               = &crypto_ablkcipher_type,
 613         .cra_module             = THIS_MODULE,
 614         .cra_list               = LIST_HEAD_INIT(ablk_cbc_alg.cra_list),
 615         .cra_init               = ablk_cbc_init,
 616         .cra_exit               = ablk_exit,
 617         .cra_u = {
 618                 .ablkcipher = {
 619                         .min_keysize    = AES_MIN_KEY_SIZE,
 620                         .max_keysize    = AES_MAX_KEY_SIZE,
 621                         .ivsize         = AES_BLOCK_SIZE,
 622                         .setkey         = ablk_set_key,
 623                         .encrypt        = ablk_encrypt,
 624                         .decrypt        = ablk_decrypt,
 625                 },
 626         },
 627 };
 628
 629 #ifdef CONFIG_X86_64
 630 static int ablk_ctr_init(struct crypto_tfm *tfm)
 631 {
 632         struct cryptd_ablkcipher *cryptd_tfm;
 633
 634         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ctr-aes-aesni", 0, 0);
 635         if (IS_ERR(cryptd_tfm))
 636                 return PTR_ERR(cryptd_tfm);
 637         ablk_init_common(tfm, cryptd_tfm);
 638         return 0;
 639 }
 640
 641 static struct crypto_alg ablk_ctr_alg = {
 642         .cra_name               = "ctr(aes)",
 643         .cra_driver_name        = "ctr-aes-aesni",
 644         .cra_priority           = 400,
 645         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 646         .cra_blocksize          = 1,
 647         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 648         .cra_alignmask          = 0,
 649         .cra_type               = &crypto_ablkcipher_type,
 650         .cra_module             = THIS_MODULE,
 651         .cra_list               = LIST_HEAD_INIT(ablk_ctr_alg.cra_list),
 652         .cra_init               = ablk_ctr_init,
 653         .cra_exit               = ablk_exit,
 654         .cra_u = {
 655                 .ablkcipher = {
 656                         .min_keysize    = AES_MIN_KEY_SIZE,
 657                         .max_keysize    = AES_MAX_KEY_SIZE,
 658                         .ivsize         = AES_BLOCK_SIZE,
 659                         .setkey         = ablk_set_key,
 660                         .encrypt        = ablk_encrypt,
 661                         .decrypt        = ablk_encrypt,
 662                         .geniv          = "chainiv",
 663                 },
 664         },
 665 };
 666
 667 #ifdef HAS_CTR
 668 static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm)
 669 {
 670         struct cryptd_ablkcipher *cryptd_tfm;
 671
 672         cryptd_tfm = cryptd_alloc_ablkcipher(
 673                 "rfc3686(__driver-ctr-aes-aesni)", 0, 0);
 674         if (IS_ERR(cryptd_tfm))
 675                 return PTR_ERR(cryptd_tfm);
 676         ablk_init_common(tfm, cryptd_tfm);
 677         return 0;
 678 }
 679
 680 static struct crypto_alg ablk_rfc3686_ctr_alg = {
 681         .cra_name               = "rfc3686(ctr(aes))",
 682         .cra_driver_name        = "rfc3686-ctr-aes-aesni",
 683         .cra_priority           = 400,
 684         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 685         .cra_blocksize          = 1,
 686         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 687         .cra_alignmask          = 0,
 688         .cra_type               = &crypto_ablkcipher_type,
 689         .cra_module             = THIS_MODULE,
 690         .cra_list               = LIST_HEAD_INIT(ablk_rfc3686_ctr_alg.cra_list),
 691         .cra_init               = ablk_rfc3686_ctr_init,
 692         .cra_exit               = ablk_exit,
 693         .cra_u = {
 694                 .ablkcipher = {
 695                         .min_keysize = AES_MIN_KEY_SIZE+CTR_RFC3686_NONCE_SIZE,
 696                         .max_keysize = AES_MAX_KEY_SIZE+CTR_RFC3686_NONCE_SIZE,
 697                         .ivsize      = CTR_RFC3686_IV_SIZE,
 698                         .setkey      = ablk_set_key,
 699                         .encrypt     = ablk_encrypt,
 700                         .decrypt     = ablk_decrypt,
 701                         .geniv       = "seqiv",
 702                 },
 703         },
 704 };
 705 #endif
 706 #endif
 707
 708 #ifdef HAS_LRW
 709 static int ablk_lrw_init(struct crypto_tfm *tfm)
 710 {
 711         struct cryptd_ablkcipher *cryptd_tfm;
 712
 713         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(lrw(__driver-aes-aesni))",
 714                                              0, 0);
 715         if (IS_ERR(cryptd_tfm))
 716                 return PTR_ERR(cryptd_tfm);
 717         ablk_init_common(tfm, cryptd_tfm);
 718         return 0;
 719 }
 720
 721 static struct crypto_alg ablk_lrw_alg = {
 722         .cra_name               = "lrw(aes)",
 723         .cra_driver_name        = "lrw-aes-aesni",
 724         .cra_priority           = 400,
 725         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 726         .cra_blocksize          = AES_BLOCK_SIZE,
 727         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 728         .cra_alignmask          = 0,
 729         .cra_type               = &crypto_ablkcipher_type,
 730         .cra_module             = THIS_MODULE,
 731         .cra_list               = LIST_HEAD_INIT(ablk_lrw_alg.cra_list),
 732         .cra_init               = ablk_lrw_init,
 733         .cra_exit               = ablk_exit,
 734         .cra_u = {
 735                 .ablkcipher = {
 736                         .min_keysize    = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
 737                         .max_keysize    = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
 738                         .ivsize         = AES_BLOCK_SIZE,
 739                         .setkey         = ablk_set_key,
 740                         .encrypt        = ablk_encrypt,
 741                         .decrypt        = ablk_decrypt,
 742                 },
 743         },
 744 };
 745 #endif
 746
 747 #ifdef HAS_PCBC
 748 static int ablk_pcbc_init(struct crypto_tfm *tfm)
 749 {
 750         struct cryptd_ablkcipher *cryptd_tfm;
 751
 752         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(pcbc(__driver-aes-aesni))",
 753                                              0, 0);
 754         if (IS_ERR(cryptd_tfm))
 755                 return PTR_ERR(cryptd_tfm);
 756         ablk_init_common(tfm, cryptd_tfm);
 757         return 0;
 758 }
 759
 760 static struct crypto_alg ablk_pcbc_alg = {
 761         .cra_name               = "pcbc(aes)",
 762         .cra_driver_name        = "pcbc-aes-aesni",
 763         .cra_priority           = 400,
 764         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 765         .cra_blocksize          = AES_BLOCK_SIZE,
 766         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 767         .cra_alignmask          = 0,
 768         .cra_type               = &crypto_ablkcipher_type,
 769         .cra_module             = THIS_MODULE,
 770         .cra_list               = LIST_HEAD_INIT(ablk_pcbc_alg.cra_list),
 771         .cra_init               = ablk_pcbc_init,
 772         .cra_exit               = ablk_exit,
 773         .cra_u = {
 774                 .ablkcipher = {
 775                         .min_keysize    = AES_MIN_KEY_SIZE,
 776                         .max_keysize    = AES_MAX_KEY_SIZE,
 777                         .ivsize         = AES_BLOCK_SIZE,
 778                         .setkey         = ablk_set_key,
 779                         .encrypt        = ablk_encrypt,
 780                         .decrypt        = ablk_decrypt,
 781                 },
 782         },
 783 };
 784 #endif
 785
 786 #ifdef HAS_XTS
 787 static int ablk_xts_init(struct crypto_tfm *tfm)
 788 {
 789         struct cryptd_ablkcipher *cryptd_tfm;
 790
 791         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(xts(__driver-aes-aesni))",
 792                                              0, 0);
 793         if (IS_ERR(cryptd_tfm))
 794                 return PTR_ERR(cryptd_tfm);
 795         ablk_init_common(tfm, cryptd_tfm);
 796         return 0;
 797 }
 798
 799 static struct crypto_alg ablk_xts_alg = {
 800         .cra_name               = "xts(aes)",
 801         .cra_driver_name        = "xts-aes-aesni",
 802         .cra_priority           = 400,
 803         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 804         .cra_blocksize          = AES_BLOCK_SIZE,
 805         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 806         .cra_alignmask          = 0,
 807         .cra_type               = &crypto_ablkcipher_type,
 808         .cra_module             = THIS_MODULE,
 809         .cra_list               = LIST_HEAD_INIT(ablk_xts_alg.cra_list),
 810         .cra_init               = ablk_xts_init,
 811         .cra_exit               = ablk_exit,
 812         .cra_u = {
 813                 .ablkcipher = {
 814                         .min_keysize    = 2 * AES_MIN_KEY_SIZE,
 815                         .max_keysize    = 2 * AES_MAX_KEY_SIZE,
 816                         .ivsize         = AES_BLOCK_SIZE,
 817                         .setkey         = ablk_set_key,
 818                         .encrypt        = ablk_encrypt,
 819                         .decrypt        = ablk_decrypt,
 820                 },
 821         },
 822 };
 823 #endif
 824
 825 #ifdef CONFIG_X86_64
 826 static int rfc4106_init(struct crypto_tfm *tfm)
 827 {
 828         struct cryptd_aead *cryptd_tfm;
 829         struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *)
 830                 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
 831         cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0);
 832         if (IS_ERR(cryptd_tfm))
 833                 return PTR_ERR(cryptd_tfm);
 834         ctx->cryptd_tfm = cryptd_tfm;
 835         tfm->crt_aead.reqsize = sizeof(struct aead_request)
 836                 + crypto_aead_reqsize(&cryptd_tfm->base);
 837         return 0;
 838 }
 839
 840 static void rfc4106_exit(struct crypto_tfm *tfm)
 841 {
 842         struct aesni_rfc4106_gcm_ctx *ctx =
 843                 (struct aesni_rfc4106_gcm_ctx *)
 844                 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
 845         if (!IS_ERR(ctx->cryptd_tfm))
 846                 cryptd_free_aead(ctx->cryptd_tfm);
 847         return;
 848 }
 849
 850 static void
 851 rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err)
 852 {
 853         struct aesni_gcm_set_hash_subkey_result *result = req->data;
 854
 855         if (err == -EINPROGRESS)
 856                 return;
 857         result->err = err;
 858         complete(&result->completion);
 859 }
 860
 861 static int
 862 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
 863 {
 864         struct crypto_ablkcipher *ctr_tfm;
 865         struct ablkcipher_request *req;
 866         int ret = -EINVAL;
 867         struct aesni_hash_subkey_req_data *req_data;
 868
 869         ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0);
 870         if (IS_ERR(ctr_tfm))
 871                 return PTR_ERR(ctr_tfm);
 872
 873         crypto_ablkcipher_clear_flags(ctr_tfm, ~0);
 874
 875         ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len);
 876         if (ret) {
 877                 crypto_free_ablkcipher(ctr_tfm);
 878                 return ret;
 879         }
 880
 881         req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL);
 882         if (!req) {
 883                 crypto_free_ablkcipher(ctr_tfm);
 884                 return -EINVAL;
 885         }
 886
 887         req_data = kmalloc(sizeof(*req_data), GFP_KERNEL);
 888         if (!req_data) {
 889                 crypto_free_ablkcipher(ctr_tfm);
 890                 return -ENOMEM;
 891         }
 892         memset(req_data->iv, 0, sizeof(req_data->iv));
 893
 894         /* Clear the data in the hash sub key container to zero.*/
 895         /* We want to cipher all zeros to create the hash sub key. */
 896         memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
 897
 898         init_completion(&req_data->result.completion);
 899         sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE);
 900         ablkcipher_request_set_tfm(req, ctr_tfm);
 901         ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
 902                                         CRYPTO_TFM_REQ_MAY_BACKLOG,
 903                                         rfc4106_set_hash_subkey_done,
 904                                         &req_data->result);
 905
 906         ablkcipher_request_set_crypt(req, &req_data->sg,
 907                 &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv);
 908
 909         ret = crypto_ablkcipher_encrypt(req);
 910         if (ret == -EINPROGRESS || ret == -EBUSY) {
 911                 ret = wait_for_completion_interruptible
 912                         (&req_data->result.completion);
 913                 if (!ret)
 914                         ret = req_data->result.err;
 915         }
 916         ablkcipher_request_free(req);
 917         kfree(req_data);
 918         crypto_free_ablkcipher(ctr_tfm);
 919         return ret;
 920 }
 921
 922 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
 923                                                    unsigned int key_len)
 924 {
 925         int ret = 0;
 926         struct crypto_tfm *tfm = crypto_aead_tfm(parent);
 927         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
 928         u8 *new_key_mem = NULL;
 929
 930         if (key_len < 4) {
 931                 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 932                 return -EINVAL;
 933         }
 934         /*Account for 4 byte nonce at the end.*/
 935         key_len -= 4;
 936         if (key_len != AES_KEYSIZE_128) {
 937                 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 938                 return -EINVAL;
 939         }
 940
 941         memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
 942         /*This must be on a 16 byte boundary!*/
 943         if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN)
 944                 return -EINVAL;
 945
 946         if ((unsigned long)key % AESNI_ALIGN) {
 947                 /*key is not aligned: use an auxuliar aligned pointer*/
 948                 new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL);
 949                 if (!new_key_mem)
 950                         return -ENOMEM;
 951
 952                 new_key_mem = PTR_ALIGN(new_key_mem, AESNI_ALIGN);
 953                 memcpy(new_key_mem, key, key_len);
 954                 key = new_key_mem;
 955         }
 956
 957         if (!irq_fpu_usable())
 958                 ret = crypto_aes_expand_key(&(ctx->aes_key_expanded),
 959                 key, key_len);
 960         else {
 961                 kernel_fpu_begin();
 962                 ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len);
 963                 kernel_fpu_end();
 964         }
 965         /*This must be on a 16 byte boundary!*/
 966         if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) {
 967                 ret = -EINVAL;
 968                 goto exit;
 969         }
 970         ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
 971 exit:
 972         kfree(new_key_mem);
 973         return ret;
 974 }
 975
 976 /* This is the Integrity Check Value (aka the authentication tag length and can
 977  * be 8, 12 or 16 bytes long. */
 978 static int rfc4106_set_authsize(struct crypto_aead *parent,
 979                                 unsigned int authsize)
 980 {
 981         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
 982         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
 983
 984         switch (authsize) {
 985         case 8:
 986         case 12:
 987         case 16:
 988                 break;
 989         default:
 990                 return -EINVAL;
 991         }
 992         crypto_aead_crt(parent)->authsize = authsize;
 993         crypto_aead_crt(cryptd_child)->authsize = authsize;
 994         return 0;
 995 }
 996
 997 static int rfc4106_encrypt(struct aead_request *req)
 998 {
 999         int ret;
1000         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1001         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1002         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
1003
1004         if (!irq_fpu_usable()) {
1005                 struct aead_request *cryptd_req =
1006                         (struct aead_request *) aead_request_ctx(req);
1007                 memcpy(cryptd_req, req, sizeof(*req));
1008                 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
1009                 return crypto_aead_encrypt(cryptd_req);
1010         } else {
1011                 kernel_fpu_begin();
1012                 ret = cryptd_child->base.crt_aead.encrypt(req);
1013                 kernel_fpu_end();
1014                 return ret;
1015         }
1016 }
1017
1018 static int rfc4106_decrypt(struct aead_request *req)
1019 {
1020         int ret;
1021         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1022         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1023         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
1024
1025         if (!irq_fpu_usable()) {
1026                 struct aead_request *cryptd_req =
1027                         (struct aead_request *) aead_request_ctx(req);
1028                 memcpy(cryptd_req, req, sizeof(*req));
1029                 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
1030                 return crypto_aead_decrypt(cryptd_req);
1031         } else {
1032                 kernel_fpu_begin();
1033                 ret = cryptd_child->base.crt_aead.decrypt(req);
1034                 kernel_fpu_end();
1035                 return ret;
1036         }
1037 }
1038
1039 static struct crypto_alg rfc4106_alg = {
1040         .cra_name = "rfc4106(gcm(aes))",
1041         .cra_driver_name = "rfc4106-gcm-aesni",
1042         .cra_priority = 400,
1043         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1044         .cra_blocksize = 1,
1045         .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN,
1046         .cra_alignmask = 0,
1047         .cra_type = &crypto_nivaead_type,
1048         .cra_module = THIS_MODULE,
1049         .cra_list = LIST_HEAD_INIT(rfc4106_alg.cra_list),
1050         .cra_init = rfc4106_init,
1051         .cra_exit = rfc4106_exit,
1052         .cra_u = {
1053                 .aead = {
1054                         .setkey = rfc4106_set_key,
1055                         .setauthsize = rfc4106_set_authsize,
1056                         .encrypt = rfc4106_encrypt,
1057                         .decrypt = rfc4106_decrypt,
1058                         .geniv = "seqiv",
1059                         .ivsize = 8,
1060                         .maxauthsize = 16,
1061                 },
1062         },
1063 };
1064
1065 static int __driver_rfc4106_encrypt(struct aead_request *req)
1066 {
1067         u8 one_entry_in_sg = 0;
1068         u8 *src, *dst, *assoc;
1069         __be32 counter = cpu_to_be32(1);
1070         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1071         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1072         void *aes_ctx = &(ctx->aes_key_expanded);
1073         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
1074         u8 iv_tab[16+AESNI_ALIGN];
1075         u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN);
1076         struct scatter_walk src_sg_walk;
1077         struct scatter_walk assoc_sg_walk;
1078         struct scatter_walk dst_sg_walk;
1079         unsigned int i;
1080
1081         /* Assuming we are supporting rfc4106 64-bit extended */
1082         /* sequence numbers We need to have the AAD length equal */
1083         /* to 8 or 12 bytes */
1084         if (unlikely(req->assoclen != 8 && req->assoclen != 12))
1085                 return -EINVAL;
1086         /* IV below built */
1087         for (i = 0; i < 4; i++)
1088                 *(iv+i) = ctx->nonce[i];
1089         for (i = 0; i < 8; i++)
1090                 *(iv+4+i) = req->iv[i];
1091         *((__be32 *)(iv+12)) = counter;
1092
1093         if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
1094                 one_entry_in_sg = 1;
1095                 scatterwalk_start(&src_sg_walk, req->src);
1096                 scatterwalk_start(&assoc_sg_walk, req->assoc);
1097                 src = scatterwalk_map(&src_sg_walk, 0);
1098                 assoc = scatterwalk_map(&assoc_sg_walk, 0);
1099                 dst = src;
1100                 if (unlikely(req->src != req->dst)) {
1101                         scatterwalk_start(&dst_sg_walk, req->dst);
1102                         dst = scatterwalk_map(&dst_sg_walk, 0);
1103                 }
1104
1105         } else {
1106                 /* Allocate memory for src, dst, assoc */
1107                 src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
1108                         GFP_ATOMIC);
1109                 if (unlikely(!src))
1110                         return -ENOMEM;
1111                 assoc = (src + req->cryptlen + auth_tag_len);
1112                 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
1113                 scatterwalk_map_and_copy(assoc, req->assoc, 0,
1114                                         req->assoclen, 0);
1115                 dst = src;
1116         }
1117
1118         aesni_gcm_enc(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv,
1119                 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst
1120                 + ((unsigned long)req->cryptlen), auth_tag_len);
1121
1122         /* The authTag (aka the Integrity Check Value) needs to be written
1123          * back to the packet. */
1124         if (one_entry_in_sg) {
1125                 if (unlikely(req->src != req->dst)) {
1126                         scatterwalk_unmap(dst, 0);
1127                         scatterwalk_done(&dst_sg_walk, 0, 0);
1128                 }
1129                 scatterwalk_unmap(src, 0);
1130                 scatterwalk_unmap(assoc, 0);
1131                 scatterwalk_done(&src_sg_walk, 0, 0);
1132                 scatterwalk_done(&assoc_sg_walk, 0, 0);
1133         } else {
1134                 scatterwalk_map_and_copy(dst, req->dst, 0,
1135                         req->cryptlen + auth_tag_len, 1);
1136                 kfree(src);
1137         }
1138         return 0;
1139 }
1140
1141 static int __driver_rfc4106_decrypt(struct aead_request *req)
1142 {
1143         u8 one_entry_in_sg = 0;
1144         u8 *src, *dst, *assoc;
1145         unsigned long tempCipherLen = 0;
1146         __be32 counter = cpu_to_be32(1);
1147         int retval = 0;
1148         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1149         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1150         void *aes_ctx = &(ctx->aes_key_expanded);
1151         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
1152         u8 iv_and_authTag[32+AESNI_ALIGN];
1153         u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN);
1154         u8 *authTag = iv + 16;
1155         struct scatter_walk src_sg_walk;
1156         struct scatter_walk assoc_sg_walk;
1157         struct scatter_walk dst_sg_walk;
1158         unsigned int i;
1159
1160         if (unlikely((req->cryptlen < auth_tag_len) ||
1161                 (req->assoclen != 8 && req->assoclen != 12)))
1162                 return -EINVAL;
1163         /* Assuming we are supporting rfc4106 64-bit extended */
1164         /* sequence numbers We need to have the AAD length */
1165         /* equal to 8 or 12 bytes */
1166
1167         tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
1168         /* IV below built */
1169         for (i = 0; i < 4; i++)
1170                 *(iv+i) = ctx->nonce[i];
1171         for (i = 0; i < 8; i++)
1172                 *(iv+4+i) = req->iv[i];
1173         *((__be32 *)(iv+12)) = counter;
1174
1175         if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
1176                 one_entry_in_sg = 1;
1177                 scatterwalk_start(&src_sg_walk, req->src);
1178                 scatterwalk_start(&assoc_sg_walk, req->assoc);
1179                 src = scatterwalk_map(&src_sg_walk, 0);
1180                 assoc = scatterwalk_map(&assoc_sg_walk, 0);
1181                 dst = src;
1182                 if (unlikely(req->src != req->dst)) {
1183                         scatterwalk_start(&dst_sg_walk, req->dst);
1184                         dst = scatterwalk_map(&dst_sg_walk, 0);
1185                 }
1186
1187         } else {
1188                 /* Allocate memory for src, dst, assoc */
1189                 src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
1190                 if (!src)
1191                         return -ENOMEM;
1192                 assoc = (src + req->cryptlen + auth_tag_len);
1193                 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
1194                 scatterwalk_map_and_copy(assoc, req->assoc, 0,
1195                         req->assoclen, 0);
1196                 dst = src;
1197         }
1198
1199         aesni_gcm_dec(aes_ctx, dst, src, tempCipherLen, iv,
1200                 ctx->hash_subkey, assoc, (unsigned long)req->assoclen,
1201                 authTag, auth_tag_len);
1202
1203         /* Compare generated tag with passed in tag. */
1204         retval = memcmp(src + tempCipherLen, authTag, auth_tag_len) ?
1205                 -EBADMSG : 0;
1206
1207         if (one_entry_in_sg) {
1208                 if (unlikely(req->src != req->dst)) {
1209                         scatterwalk_unmap(dst, 0);
1210                         scatterwalk_done(&dst_sg_walk, 0, 0);
1211                 }
1212                 scatterwalk_unmap(src, 0);
1213                 scatterwalk_unmap(assoc, 0);
1214                 scatterwalk_done(&src_sg_walk, 0, 0);
1215                 scatterwalk_done(&assoc_sg_walk, 0, 0);
1216         } else {
1217                 scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1);
1218                 kfree(src);
1219         }
1220         return retval;
1221 }
1222
1223 static struct crypto_alg __rfc4106_alg = {
1224         .cra_name               = "__gcm-aes-aesni",
1225         .cra_driver_name        = "__driver-gcm-aes-aesni",
1226         .cra_priority           = 0,
1227         .cra_flags              = CRYPTO_ALG_TYPE_AEAD,
1228         .cra_blocksize          = 1,
1229         .cra_ctxsize    = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN,
1230         .cra_alignmask          = 0,
1231         .cra_type               = &crypto_aead_type,
1232         .cra_module             = THIS_MODULE,
1233         .cra_list               = LIST_HEAD_INIT(__rfc4106_alg.cra_list),
1234         .cra_u = {
1235                 .aead = {
1236                         .encrypt        = __driver_rfc4106_encrypt,
1237                         .decrypt        = __driver_rfc4106_decrypt,
1238                 },
1239         },
1240 };
1241 #endif
1242
1243 static int __init aesni_init(void)
1244 {
1245         int err;
1246
1247         if (!cpu_has_aes) {
1248                 printk(KERN_INFO "Intel AES-NI instructions are not detected.\n");
1249                 return -ENODEV;
1250         }
1251
1252         if ((err = crypto_register_alg(&aesni_alg)))
1253                 goto aes_err;
1254         if ((err = crypto_register_alg(&__aesni_alg)))
1255                 goto __aes_err;
1256         if ((err = crypto_register_alg(&blk_ecb_alg)))
1257                 goto blk_ecb_err;
1258         if ((err = crypto_register_alg(&blk_cbc_alg)))
1259                 goto blk_cbc_err;
1260         if ((err = crypto_register_alg(&ablk_ecb_alg)))
1261                 goto ablk_ecb_err;
1262         if ((err = crypto_register_alg(&ablk_cbc_alg)))
1263                 goto ablk_cbc_err;
1264 #ifdef CONFIG_X86_64
1265         if ((err = crypto_register_alg(&blk_ctr_alg)))
1266                 goto blk_ctr_err;
1267         if ((err = crypto_register_alg(&ablk_ctr_alg)))
1268                 goto ablk_ctr_err;
1269         if ((err = crypto_register_alg(&__rfc4106_alg)))
1270                 goto __aead_gcm_err;
1271         if ((err = crypto_register_alg(&rfc4106_alg)))
1272                 goto aead_gcm_err;
1273 #ifdef HAS_CTR
1274         if ((err = crypto_register_alg(&ablk_rfc3686_ctr_alg)))
1275                 goto ablk_rfc3686_ctr_err;
1276 #endif
1277 #endif
1278 #ifdef HAS_LRW
1279         if ((err = crypto_register_alg(&ablk_lrw_alg)))
1280                 goto ablk_lrw_err;
1281 #endif
1282 #ifdef HAS_PCBC
1283         if ((err = crypto_register_alg(&ablk_pcbc_alg)))
1284                 goto ablk_pcbc_err;
1285 #endif
1286 #ifdef HAS_XTS
1287         if ((err = crypto_register_alg(&ablk_xts_alg)))
1288                 goto ablk_xts_err;
1289 #endif
1290         return err;
1291
1292 #ifdef HAS_XTS
1293 ablk_xts_err:
1294 #endif
1295 #ifdef HAS_PCBC
1296         crypto_unregister_alg(&ablk_pcbc_alg);
1297 ablk_pcbc_err:
1298 #endif
1299 #ifdef HAS_LRW
1300         crypto_unregister_alg(&ablk_lrw_alg);
1301 ablk_lrw_err:
1302 #endif
1303 #ifdef CONFIG_X86_64
1304 #ifdef HAS_CTR
1305         crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
1306 ablk_rfc3686_ctr_err:
1307 #endif
1308         crypto_unregister_alg(&rfc4106_alg);
1309 aead_gcm_err:
1310         crypto_unregister_alg(&__rfc4106_alg);
1311 __aead_gcm_err:
1312         crypto_unregister_alg(&ablk_ctr_alg);
1313 ablk_ctr_err:
1314         crypto_unregister_alg(&blk_ctr_alg);
1315 blk_ctr_err:
1316 #endif
1317         crypto_unregister_alg(&ablk_cbc_alg);
1318 ablk_cbc_err:
1319         crypto_unregister_alg(&ablk_ecb_alg);
1320 ablk_ecb_err:
1321         crypto_unregister_alg(&blk_cbc_alg);
1322 blk_cbc_err:
1323         crypto_unregister_alg(&blk_ecb_alg);
1324 blk_ecb_err:
1325         crypto_unregister_alg(&__aesni_alg);
1326 __aes_err:
1327         crypto_unregister_alg(&aesni_alg);
1328 aes_err:
1329         return err;
1330 }
1331
1332 static void __exit aesni_exit(void)
1333 {
1334 #ifdef HAS_XTS
1335         crypto_unregister_alg(&ablk_xts_alg);
1336 #endif
1337 #ifdef HAS_PCBC
1338         crypto_unregister_alg(&ablk_pcbc_alg);
1339 #endif
1340 #ifdef HAS_LRW
1341         crypto_unregister_alg(&ablk_lrw_alg);
1342 #endif
1343 #ifdef CONFIG_X86_64
1344 #ifdef HAS_CTR
1345         crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
1346 #endif
1347         crypto_unregister_alg(&rfc4106_alg);
1348         crypto_unregister_alg(&__rfc4106_alg);
1349         crypto_unregister_alg(&ablk_ctr_alg);
1350         crypto_unregister_alg(&blk_ctr_alg);
1351 #endif
1352         crypto_unregister_alg(&ablk_cbc_alg);
1353         crypto_unregister_alg(&ablk_ecb_alg);
1354         crypto_unregister_alg(&blk_cbc_alg);
1355         crypto_unregister_alg(&blk_ecb_alg);
1356         crypto_unregister_alg(&__aesni_alg);
1357         crypto_unregister_alg(&aesni_alg);
1358 }
1359
1360 module_init(aesni_init);
1361 module_exit(aesni_exit);
1362
1363 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1364 MODULE_LICENSE("GPL");
1365 MODULE_ALIAS("aes");