arch/x86/crypto/twofish_glue_3way.c

   1 /*
   2  * Glue Code for 3-way parallel assembler optimized version of Twofish
   3  *
   4  * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
   5  *
   6  * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
   7  *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
   8  * CTR part based on code (crypto/ctr.c) by:
   9  *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
  10  *
  11  * This program is free software; you can redistribute it and/or modify
  12  * it under the terms of the GNU General Public License as published by
  13  * the Free Software Foundation; either version 2 of the License, or
  14  * (at your option) any later version.
  15  *
  16  * This program is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  * GNU General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU General Public License
  22  * along with this program; if not, write to the Free Software
  23  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
  24  * USA
  25  *
  26  */
  27
  28 #include <linux/crypto.h>
  29 #include <linux/init.h>
  30 #include <linux/module.h>
  31 #include <linux/types.h>
  32 #include <crypto/algapi.h>
  33 #include <crypto/twofish.h>
  34 #include <crypto/b128ops.h>
  35
  36 /* regular block cipher functions from twofish_x86_64 module */
  37 asmlinkage void twofish_enc_blk(struct twofish_ctx *ctx, u8 *dst,
  38                                 const u8 *src);
  39 asmlinkage void twofish_dec_blk(struct twofish_ctx *ctx, u8 *dst,
  40                                 const u8 *src);
  41
  42 /* 3-way parallel cipher functions */
  43 asmlinkage void __twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
  44                                        const u8 *src, bool xor);
  45 asmlinkage void twofish_dec_blk_3way(struct twofish_ctx *ctx, u8 *dst,
  46                                      const u8 *src);
  47
  48 static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
  49                                         const u8 *src)
  50 {
  51         __twofish_enc_blk_3way(ctx, dst, src, false);
  52 }
  53
  54 static inline void twofish_enc_blk_xor_3way(struct twofish_ctx *ctx, u8 *dst,
  55                                             const u8 *src)
  56 {
  57         __twofish_enc_blk_3way(ctx, dst, src, true);
  58 }
  59
  60 static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
  61                      void (*fn)(struct twofish_ctx *, u8 *, const u8 *),
  62                      void (*fn_3way)(struct twofish_ctx *, u8 *, const u8 *))
  63 {
  64         struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
  65         unsigned int bsize = TF_BLOCK_SIZE;
  66         unsigned int nbytes;
  67         int err;
  68
  69         err = blkcipher_walk_virt(desc, walk);
  70
  71         while ((nbytes = walk->nbytes)) {
  72                 u8 *wsrc = walk->src.virt.addr;
  73                 u8 *wdst = walk->dst.virt.addr;
  74
  75                 /* Process three block batch */
  76                 if (nbytes >= bsize * 3) {
  77                         do {
  78                                 fn_3way(ctx, wdst, wsrc);
  79
  80                                 wsrc += bsize * 3;
  81                                 wdst += bsize * 3;
  82                                 nbytes -= bsize * 3;
  83                         } while (nbytes >= bsize * 3);
  84
  85                         if (nbytes < bsize)
  86                                 goto done;
  87                 }
  88
  89                 /* Handle leftovers */
  90                 do {
  91                         fn(ctx, wdst, wsrc);
  92
  93                         wsrc += bsize;
  94                         wdst += bsize;
  95                         nbytes -= bsize;
  96                 } while (nbytes >= bsize);
  97
  98 done:
  99                 err = blkcipher_walk_done(desc, walk, nbytes);
 100         }
 101
 102         return err;
 103 }
 104
 105 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 106                        struct scatterlist *src, unsigned int nbytes)
 107 {
 108         struct blkcipher_walk walk;
 109
 110         blkcipher_walk_init(&walk, dst, src, nbytes);
 111         return ecb_crypt(desc, &walk, twofish_enc_blk, twofish_enc_blk_3way);
 112 }
 113
 114 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 115                        struct scatterlist *src, unsigned int nbytes)
 116 {
 117         struct blkcipher_walk walk;
 118
 119         blkcipher_walk_init(&walk, dst, src, nbytes);
 120         return ecb_crypt(desc, &walk, twofish_dec_blk, twofish_dec_blk_3way);
 121 }
 122
 123 static struct crypto_alg blk_ecb_alg = {
 124         .cra_name               = "ecb(twofish)",
 125         .cra_driver_name        = "ecb-twofish-3way",
 126         .cra_priority           = 300,
 127         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 128         .cra_blocksize          = TF_BLOCK_SIZE,
 129         .cra_ctxsize            = sizeof(struct twofish_ctx),
 130         .cra_alignmask          = 0,
 131         .cra_type               = &crypto_blkcipher_type,
 132         .cra_module             = THIS_MODULE,
 133         .cra_list               = LIST_HEAD_INIT(blk_ecb_alg.cra_list),
 134         .cra_u = {
 135                 .blkcipher = {
 136                         .min_keysize    = TF_MIN_KEY_SIZE,
 137                         .max_keysize    = TF_MAX_KEY_SIZE,
 138                         .setkey         = twofish_setkey,
 139                         .encrypt        = ecb_encrypt,
 140                         .decrypt        = ecb_decrypt,
 141                 },
 142         },
 143 };
 144
 145 static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
 146                                   struct blkcipher_walk *walk)
 147 {
 148         struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 149         unsigned int bsize = TF_BLOCK_SIZE;
 150         unsigned int nbytes = walk->nbytes;
 151         u128 *src = (u128 *)walk->src.virt.addr;
 152         u128 *dst = (u128 *)walk->dst.virt.addr;
 153         u128 *iv = (u128 *)walk->iv;
 154
 155         do {
 156                 u128_xor(dst, src, iv);
 157                 twofish_enc_blk(ctx, (u8 *)dst, (u8 *)dst);
 158                 iv = dst;
 159
 160                 src += 1;
 161                 dst += 1;
 162                 nbytes -= bsize;
 163         } while (nbytes >= bsize);
 164
 165         u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv);
 166         return nbytes;
 167 }
 168
 169 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 170                        struct scatterlist *src, unsigned int nbytes)
 171 {
 172         struct blkcipher_walk walk;
 173         int err;
 174
 175         blkcipher_walk_init(&walk, dst, src, nbytes);
 176         err = blkcipher_walk_virt(desc, &walk);
 177
 178         while ((nbytes = walk.nbytes)) {
 179                 nbytes = __cbc_encrypt(desc, &walk);
 180                 err = blkcipher_walk_done(desc, &walk, nbytes);
 181         }
 182
 183         return err;
 184 }
 185
 186 static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
 187                                   struct blkcipher_walk *walk)
 188 {
 189         struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 190         unsigned int bsize = TF_BLOCK_SIZE;
 191         unsigned int nbytes = walk->nbytes;
 192         u128 *src = (u128 *)walk->src.virt.addr;
 193         u128 *dst = (u128 *)walk->dst.virt.addr;
 194         u128 ivs[3 - 1];
 195         u128 last_iv;
 196
 197         /* Start of the last block. */
 198         src += nbytes / bsize - 1;
 199         dst += nbytes / bsize - 1;
 200
 201         last_iv = *src;
 202
 203         /* Process three block batch */
 204         if (nbytes >= bsize * 3) {
 205                 do {
 206                         nbytes -= bsize * (3 - 1);
 207                         src -= 3 - 1;
 208                         dst -= 3 - 1;
 209
 210                         ivs[0] = src[0];
 211                         ivs[1] = src[1];
 212
 213                         twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src);
 214
 215                         u128_xor(dst + 1, dst + 1, ivs + 0);
 216                         u128_xor(dst + 2, dst + 2, ivs + 1);
 217
 218                         nbytes -= bsize;
 219                         if (nbytes < bsize)
 220                                 goto done;
 221
 222                         u128_xor(dst, dst, src - 1);
 223                         src -= 1;
 224                         dst -= 1;
 225                 } while (nbytes >= bsize * 3);
 226
 227                 if (nbytes < bsize)
 228                         goto done;
 229         }
 230
 231         /* Handle leftovers */
 232         for (;;) {
 233                 twofish_dec_blk(ctx, (u8 *)dst, (u8 *)src);
 234
 235                 nbytes -= bsize;
 236                 if (nbytes < bsize)
 237                         break;
 238
 239                 u128_xor(dst, dst, src - 1);
 240                 src -= 1;
 241                 dst -= 1;
 242         }
 243
 244 done:
 245         u128_xor(dst, dst, (u128 *)walk->iv);
 246         *(u128 *)walk->iv = last_iv;
 247
 248         return nbytes;
 249 }
 250
 251 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 252                        struct scatterlist *src, unsigned int nbytes)
 253 {
 254         struct blkcipher_walk walk;
 255         int err;
 256
 257         blkcipher_walk_init(&walk, dst, src, nbytes);
 258         err = blkcipher_walk_virt(desc, &walk);
 259
 260         while ((nbytes = walk.nbytes)) {
 261                 nbytes = __cbc_decrypt(desc, &walk);
 262                 err = blkcipher_walk_done(desc, &walk, nbytes);
 263         }
 264
 265         return err;
 266 }
 267
 268 static struct crypto_alg blk_cbc_alg = {
 269         .cra_name               = "cbc(twofish)",
 270         .cra_driver_name        = "cbc-twofish-3way",
 271         .cra_priority           = 300,
 272         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 273         .cra_blocksize          = TF_BLOCK_SIZE,
 274         .cra_ctxsize            = sizeof(struct twofish_ctx),
 275         .cra_alignmask          = 0,
 276         .cra_type               = &crypto_blkcipher_type,
 277         .cra_module             = THIS_MODULE,
 278         .cra_list               = LIST_HEAD_INIT(blk_cbc_alg.cra_list),
 279         .cra_u = {
 280                 .blkcipher = {
 281                         .min_keysize    = TF_MIN_KEY_SIZE,
 282                         .max_keysize    = TF_MAX_KEY_SIZE,
 283                         .ivsize         = TF_BLOCK_SIZE,
 284                         .setkey         = twofish_setkey,
 285                         .encrypt        = cbc_encrypt,
 286                         .decrypt        = cbc_decrypt,
 287                 },
 288         },
 289 };
 290
 291 static inline void u128_to_be128(be128 *dst, const u128 *src)
 292 {
 293         dst->a = cpu_to_be64(src->a);
 294         dst->b = cpu_to_be64(src->b);
 295 }
 296
 297 static inline void be128_to_u128(u128 *dst, const be128 *src)
 298 {
 299         dst->a = be64_to_cpu(src->a);
 300         dst->b = be64_to_cpu(src->b);
 301 }
 302
 303 static inline void u128_inc(u128 *i)
 304 {
 305         i->b++;
 306         if (!i->b)
 307                 i->a++;
 308 }
 309
 310 static void ctr_crypt_final(struct blkcipher_desc *desc,
 311                             struct blkcipher_walk *walk)
 312 {
 313         struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 314         u8 *ctrblk = walk->iv;
 315         u8 keystream[TF_BLOCK_SIZE];
 316         u8 *src = walk->src.virt.addr;
 317         u8 *dst = walk->dst.virt.addr;
 318         unsigned int nbytes = walk->nbytes;
 319
 320         twofish_enc_blk(ctx, keystream, ctrblk);
 321         crypto_xor(keystream, src, nbytes);
 322         memcpy(dst, keystream, nbytes);
 323
 324         crypto_inc(ctrblk, TF_BLOCK_SIZE);
 325 }
 326
 327 static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
 328                                 struct blkcipher_walk *walk)
 329 {
 330         struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 331         unsigned int bsize = TF_BLOCK_SIZE;
 332         unsigned int nbytes = walk->nbytes;
 333         u128 *src = (u128 *)walk->src.virt.addr;
 334         u128 *dst = (u128 *)walk->dst.virt.addr;
 335         u128 ctrblk;
 336         be128 ctrblocks[3];
 337
 338         be128_to_u128(&ctrblk, (be128 *)walk->iv);
 339
 340         /* Process three block batch */
 341         if (nbytes >= bsize * 3) {
 342                 do {
 343                         if (dst != src) {
 344                                 dst[0] = src[0];
 345                                 dst[1] = src[1];
 346                                 dst[2] = src[2];
 347                         }
 348
 349                         /* create ctrblks for parallel encrypt */
 350                         u128_to_be128(&ctrblocks[0], &ctrblk);
 351                         u128_inc(&ctrblk);
 352                         u128_to_be128(&ctrblocks[1], &ctrblk);
 353                         u128_inc(&ctrblk);
 354                         u128_to_be128(&ctrblocks[2], &ctrblk);
 355                         u128_inc(&ctrblk);
 356
 357                         twofish_enc_blk_xor_3way(ctx, (u8 *)dst,
 358                                                  (u8 *)ctrblocks);
 359
 360                         src += 3;
 361                         dst += 3;
 362                         nbytes -= bsize * 3;
 363                 } while (nbytes >= bsize * 3);
 364
 365                 if (nbytes < bsize)
 366                         goto done;
 367         }
 368
 369         /* Handle leftovers */
 370         do {
 371                 if (dst != src)
 372                         *dst = *src;
 373
 374                 u128_to_be128(&ctrblocks[0], &ctrblk);
 375                 u128_inc(&ctrblk);
 376
 377                 twofish_enc_blk(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks);
 378                 u128_xor(dst, dst, (u128 *)ctrblocks);
 379
 380                 src += 1;
 381                 dst += 1;
 382                 nbytes -= bsize;
 383         } while (nbytes >= bsize);
 384
 385 done:
 386         u128_to_be128((be128 *)walk->iv, &ctrblk);
 387         return nbytes;
 388 }
 389
 390 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 391                      struct scatterlist *src, unsigned int nbytes)
 392 {
 393         struct blkcipher_walk walk;
 394         int err;
 395
 396         blkcipher_walk_init(&walk, dst, src, nbytes);
 397         err = blkcipher_walk_virt_block(desc, &walk, TF_BLOCK_SIZE);
 398
 399         while ((nbytes = walk.nbytes) >= TF_BLOCK_SIZE) {
 400                 nbytes = __ctr_crypt(desc, &walk);
 401                 err = blkcipher_walk_done(desc, &walk, nbytes);
 402         }
 403
 404         if (walk.nbytes) {
 405                 ctr_crypt_final(desc, &walk);
 406                 err = blkcipher_walk_done(desc, &walk, 0);
 407         }
 408
 409         return err;
 410 }
 411
 412 static struct crypto_alg blk_ctr_alg = {
 413         .cra_name               = "ctr(twofish)",
 414         .cra_driver_name        = "ctr-twofish-3way",
 415         .cra_priority           = 300,
 416         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 417         .cra_blocksize          = 1,
 418         .cra_ctxsize            = sizeof(struct twofish_ctx),
 419         .cra_alignmask          = 0,
 420         .cra_type               = &crypto_blkcipher_type,
 421         .cra_module             = THIS_MODULE,
 422         .cra_list               = LIST_HEAD_INIT(blk_ctr_alg.cra_list),
 423         .cra_u = {
 424                 .blkcipher = {
 425                         .min_keysize    = TF_MIN_KEY_SIZE,
 426                         .max_keysize    = TF_MAX_KEY_SIZE,
 427                         .ivsize         = TF_BLOCK_SIZE,
 428                         .setkey         = twofish_setkey,
 429                         .encrypt        = ctr_crypt,
 430                         .decrypt        = ctr_crypt,
 431                 },
 432         },
 433 };
 434
 435 int __init init(void)
 436 {
 437         int err;
 438
 439         err = crypto_register_alg(&blk_ecb_alg);
 440         if (err)
 441                 goto ecb_err;
 442         err = crypto_register_alg(&blk_cbc_alg);
 443         if (err)
 444                 goto cbc_err;
 445         err = crypto_register_alg(&blk_ctr_alg);
 446         if (err)
 447                 goto ctr_err;
 448
 449         return 0;
 450
 451 ctr_err:
 452         crypto_unregister_alg(&blk_cbc_alg);
 453 cbc_err:
 454         crypto_unregister_alg(&blk_ecb_alg);
 455 ecb_err:
 456         return err;
 457 }
 458
 459 void __exit fini(void)
 460 {
 461         crypto_unregister_alg(&blk_ctr_alg);
 462         crypto_unregister_alg(&blk_cbc_alg);
 463         crypto_unregister_alg(&blk_ecb_alg);
 464 }
 465
 466 module_init(init);
 467 module_exit(fini);
 468
 469 MODULE_LICENSE("GPL");
 470 MODULE_DESCRIPTION("Twofish Cipher Algorithm, 3-way parallel asm optimized");
 471 MODULE_ALIAS_CRYPTO("twofish");
 472 MODULE_ALIAS_CRYPTO("twofish-asm");