drivers/crypto/picoxcell_crypto.c

   1 /*
   2  * Copyright (c) 2010-2011 Picochip Ltd., Jamie Iles
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  17  */
  18 #include <crypto/aead.h>
  19 #include <crypto/aes.h>
  20 #include <crypto/algapi.h>
  21 #include <crypto/authenc.h>
  22 #include <crypto/des.h>
  23 #include <crypto/md5.h>
  24 #include <crypto/sha.h>
  25 #include <crypto/internal/skcipher.h>
  26 #include <linux/clk.h>
  27 #include <linux/crypto.h>
  28 #include <linux/delay.h>
  29 #include <linux/dma-mapping.h>
  30 #include <linux/dmapool.h>
  31 #include <linux/err.h>
  32 #include <linux/init.h>
  33 #include <linux/interrupt.h>
  34 #include <linux/io.h>
  35 #include <linux/list.h>
  36 #include <linux/module.h>
  37 #include <linux/platform_device.h>
  38 #include <linux/pm.h>
  39 #include <linux/rtnetlink.h>
  40 #include <linux/scatterlist.h>
  41 #include <linux/sched.h>
  42 #include <linux/slab.h>
  43 #include <linux/timer.h>
  44
  45 #include "picoxcell_crypto_regs.h"
  46
  47 /*
  48  * The threshold for the number of entries in the CMD FIFO available before
  49  * the CMD0_CNT interrupt is raised. Increasing this value will reduce the
  50  * number of interrupts raised to the CPU.
  51  */
  52 #define CMD0_IRQ_THRESHOLD   1
  53
  54 /*
  55  * The timeout period (in jiffies) for a PDU. When the the number of PDUs in
  56  * flight is greater than the STAT_IRQ_THRESHOLD or 0 the timer is disabled.
  57  * When there are packets in flight but lower than the threshold, we enable
  58  * the timer and at expiry, attempt to remove any processed packets from the
  59  * queue and if there are still packets left, schedule the timer again.
  60  */
  61 #define PACKET_TIMEOUT      1
  62
  63 /* The priority to register each algorithm with. */
  64 #define SPACC_CRYPTO_ALG_PRIORITY       10000
  65
  66 #define SPACC_CRYPTO_KASUMI_F8_KEY_LEN  16
  67 #define SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ 64
  68 #define SPACC_CRYPTO_IPSEC_HASH_PG_SZ   64
  69 #define SPACC_CRYPTO_IPSEC_MAX_CTXS     32
  70 #define SPACC_CRYPTO_IPSEC_FIFO_SZ      32
  71 #define SPACC_CRYPTO_L2_CIPHER_PG_SZ    64
  72 #define SPACC_CRYPTO_L2_HASH_PG_SZ      64
  73 #define SPACC_CRYPTO_L2_MAX_CTXS        128
  74 #define SPACC_CRYPTO_L2_FIFO_SZ         128
  75
  76 #define MAX_DDT_LEN                     16
  77
  78 /* DDT format. This must match the hardware DDT format exactly. */
  79 struct spacc_ddt {
  80         dma_addr_t      p;
  81         u32             len;
  82 };
  83
  84 /*
  85  * Asynchronous crypto request structure.
  86  *
  87  * This structure defines a request that is either queued for processing or
  88  * being processed.
  89  */
  90 struct spacc_req {
  91         struct list_head                list;
  92         struct spacc_engine             *engine;
  93         struct crypto_async_request     *req;
  94         int                             result;
  95         bool                            is_encrypt;
  96         unsigned                        ctx_id;
  97         dma_addr_t                      src_addr, dst_addr;
  98         struct spacc_ddt                *src_ddt, *dst_ddt;
  99         void                            (*complete)(struct spacc_req *req);
 100
 101         /* AEAD specific bits. */
 102         u8                              *giv;
 103         size_t                          giv_len;
 104         dma_addr_t                      giv_pa;
 105 };
 106
 107 struct spacc_engine {
 108         void __iomem                    *regs;
 109         struct list_head                pending;
 110         int                             next_ctx;
 111         spinlock_t                      hw_lock;
 112         int                             in_flight;
 113         struct list_head                completed;
 114         struct list_head                in_progress;
 115         struct tasklet_struct           complete;
 116         unsigned long                   fifo_sz;
 117         void __iomem                    *cipher_ctx_base;
 118         void __iomem                    *hash_key_base;
 119         struct spacc_alg                *algs;
 120         unsigned                        num_algs;
 121         struct list_head                registered_algs;
 122         size_t                          cipher_pg_sz;
 123         size_t                          hash_pg_sz;
 124         const char                      *name;
 125         struct clk                      *clk;
 126         struct device                   *dev;
 127         unsigned                        max_ctxs;
 128         struct timer_list               packet_timeout;
 129         unsigned                        stat_irq_thresh;
 130         struct dma_pool                 *req_pool;
 131 };
 132
 133 /* Algorithm type mask. */
 134 #define SPACC_CRYPTO_ALG_MASK           0x7
 135
 136 /* SPACC definition of a crypto algorithm. */
 137 struct spacc_alg {
 138         unsigned long                   ctrl_default;
 139         unsigned long                   type;
 140         struct crypto_alg               alg;
 141         struct spacc_engine             *engine;
 142         struct list_head                entry;
 143         int                             key_offs;
 144         int                             iv_offs;
 145 };
 146
 147 /* Generic context structure for any algorithm type. */
 148 struct spacc_generic_ctx {
 149         struct spacc_engine             *engine;
 150         int                             flags;
 151         int                             key_offs;
 152         int                             iv_offs;
 153 };
 154
 155 /* Block cipher context. */
 156 struct spacc_ablk_ctx {
 157         struct spacc_generic_ctx        generic;
 158         u8                              key[AES_MAX_KEY_SIZE];
 159         u8                              key_len;
 160         /*
 161          * The fallback cipher. If the operation can't be done in hardware,
 162          * fallback to a software version.
 163          */
 164         struct crypto_ablkcipher        *sw_cipher;
 165 };
 166
 167 /* AEAD cipher context. */
 168 struct spacc_aead_ctx {
 169         struct spacc_generic_ctx        generic;
 170         u8                              cipher_key[AES_MAX_KEY_SIZE];
 171         u8                              hash_ctx[SPACC_CRYPTO_IPSEC_HASH_PG_SZ];
 172         u8                              cipher_key_len;
 173         u8                              hash_key_len;
 174         struct crypto_aead              *sw_cipher;
 175         size_t                          auth_size;
 176         u8                              salt[AES_BLOCK_SIZE];
 177 };
 178
 179 static int spacc_ablk_submit(struct spacc_req *req);
 180
 181 static inline struct spacc_alg *to_spacc_alg(struct crypto_alg *alg)
 182 {
 183         return alg ? container_of(alg, struct spacc_alg, alg) : NULL;
 184 }
 185
 186 static inline int spacc_fifo_cmd_full(struct spacc_engine *engine)
 187 {
 188         u32 fifo_stat = readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET);
 189
 190         return fifo_stat & SPA_FIFO_CMD_FULL;
 191 }
 192
 193 /*
 194  * Given a cipher context, and a context number, get the base address of the
 195  * context page.
 196  *
 197  * Returns the address of the context page where the key/context may
 198  * be written.
 199  */
 200 static inline void __iomem *spacc_ctx_page_addr(struct spacc_generic_ctx *ctx,
 201                                                 unsigned indx,
 202                                                 bool is_cipher_ctx)
 203 {
 204         return is_cipher_ctx ? ctx->engine->cipher_ctx_base +
 205                         (indx * ctx->engine->cipher_pg_sz) :
 206                 ctx->engine->hash_key_base + (indx * ctx->engine->hash_pg_sz);
 207 }
 208
 209 /* The context pages can only be written with 32-bit accesses. */
 210 static inline void memcpy_toio32(u32 __iomem *dst, const void *src,
 211                                  unsigned count)
 212 {
 213         const u32 *src32 = (const u32 *) src;
 214
 215         while (count--)
 216                 writel(*src32++, dst++);
 217 }
 218
 219 static void spacc_cipher_write_ctx(struct spacc_generic_ctx *ctx,
 220                                    void __iomem *page_addr, const u8 *key,
 221                                    size_t key_len, const u8 *iv, size_t iv_len)
 222 {
 223         void __iomem *key_ptr = page_addr + ctx->key_offs;
 224         void __iomem *iv_ptr = page_addr + ctx->iv_offs;
 225
 226         memcpy_toio32(key_ptr, key, key_len / 4);
 227         memcpy_toio32(iv_ptr, iv, iv_len / 4);
 228 }
 229
 230 /*
 231  * Load a context into the engines context memory.
 232  *
 233  * Returns the index of the context page where the context was loaded.
 234  */
 235 static unsigned spacc_load_ctx(struct spacc_generic_ctx *ctx,
 236                                const u8 *ciph_key, size_t ciph_len,
 237                                const u8 *iv, size_t ivlen, const u8 *hash_key,
 238                                size_t hash_len)
 239 {
 240         unsigned indx = ctx->engine->next_ctx++;
 241         void __iomem *ciph_page_addr, *hash_page_addr;
 242
 243         ciph_page_addr = spacc_ctx_page_addr(ctx, indx, 1);
 244         hash_page_addr = spacc_ctx_page_addr(ctx, indx, 0);
 245
 246         ctx->engine->next_ctx &= ctx->engine->fifo_sz - 1;
 247         spacc_cipher_write_ctx(ctx, ciph_page_addr, ciph_key, ciph_len, iv,
 248                                ivlen);
 249         writel(ciph_len | (indx << SPA_KEY_SZ_CTX_INDEX_OFFSET) |
 250                (1 << SPA_KEY_SZ_CIPHER_OFFSET),
 251                ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET);
 252
 253         if (hash_key) {
 254                 memcpy_toio32(hash_page_addr, hash_key, hash_len / 4);
 255                 writel(hash_len | (indx << SPA_KEY_SZ_CTX_INDEX_OFFSET),
 256                        ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET);
 257         }
 258
 259         return indx;
 260 }
 261
 262 /* Count the number of scatterlist entries in a scatterlist. */
 263 static int sg_count(struct scatterlist *sg_list, int nbytes)
 264 {
 265         struct scatterlist *sg = sg_list;
 266         int sg_nents = 0;
 267
 268         while (nbytes > 0) {
 269                 ++sg_nents;
 270                 nbytes -= sg->length;
 271                 sg = sg_next(sg);
 272         }
 273
 274         return sg_nents;
 275 }
 276
 277 static inline void ddt_set(struct spacc_ddt *ddt, dma_addr_t phys, size_t len)
 278 {
 279         ddt->p = phys;
 280         ddt->len = len;
 281 }
 282
 283 /*
 284  * Take a crypto request and scatterlists for the data and turn them into DDTs
 285  * for passing to the crypto engines. This also DMA maps the data so that the
 286  * crypto engines can DMA to/from them.
 287  */
 288 static struct spacc_ddt *spacc_sg_to_ddt(struct spacc_engine *engine,
 289                                          struct scatterlist *payload,
 290                                          unsigned nbytes,
 291                                          enum dma_data_direction dir,
 292                                          dma_addr_t *ddt_phys)
 293 {
 294         unsigned nents, mapped_ents;
 295         struct scatterlist *cur;
 296         struct spacc_ddt *ddt;
 297         int i;
 298
 299         nents = sg_count(payload, nbytes);
 300         mapped_ents = dma_map_sg(engine->dev, payload, nents, dir);
 301
 302         if (mapped_ents + 1 > MAX_DDT_LEN)
 303                 goto out;
 304
 305         ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, ddt_phys);
 306         if (!ddt)
 307                 goto out;
 308
 309         for_each_sg(payload, cur, mapped_ents, i)
 310                 ddt_set(&ddt[i], sg_dma_address(cur), sg_dma_len(cur));
 311         ddt_set(&ddt[mapped_ents], 0, 0);
 312
 313         return ddt;
 314
 315 out:
 316         dma_unmap_sg(engine->dev, payload, nents, dir);
 317         return NULL;
 318 }
 319
 320 static int spacc_aead_make_ddts(struct spacc_req *req, u8 *giv)
 321 {
 322         struct aead_request *areq = container_of(req->req, struct aead_request,
 323                                                  base);
 324         struct spacc_engine *engine = req->engine;
 325         struct spacc_ddt *src_ddt, *dst_ddt;
 326         unsigned ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(areq));
 327         unsigned nents = sg_count(areq->src, areq->cryptlen);
 328         dma_addr_t iv_addr;
 329         struct scatterlist *cur;
 330         int i, dst_ents, src_ents, assoc_ents;
 331         u8 *iv = giv ? giv : areq->iv;
 332
 333         src_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->src_addr);
 334         if (!src_ddt)
 335                 return -ENOMEM;
 336
 337         dst_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->dst_addr);
 338         if (!dst_ddt) {
 339                 dma_pool_free(engine->req_pool, src_ddt, req->src_addr);
 340                 return -ENOMEM;
 341         }
 342
 343         req->src_ddt = src_ddt;
 344         req->dst_ddt = dst_ddt;
 345
 346         assoc_ents = dma_map_sg(engine->dev, areq->assoc,
 347                 sg_count(areq->assoc, areq->assoclen), DMA_TO_DEVICE);
 348         if (areq->src != areq->dst) {
 349                 src_ents = dma_map_sg(engine->dev, areq->src, nents,
 350                                       DMA_TO_DEVICE);
 351                 dst_ents = dma_map_sg(engine->dev, areq->dst, nents,
 352                                       DMA_FROM_DEVICE);
 353         } else {
 354                 src_ents = dma_map_sg(engine->dev, areq->src, nents,
 355                                       DMA_BIDIRECTIONAL);
 356                 dst_ents = 0;
 357         }
 358
 359         /*
 360          * Map the IV/GIV. For the GIV it needs to be bidirectional as it is
 361          * formed by the crypto block and sent as the ESP IV for IPSEC.
 362          */
 363         iv_addr = dma_map_single(engine->dev, iv, ivsize,
 364                                  giv ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
 365         req->giv_pa = iv_addr;
 366
 367         /*
 368          * Map the associated data. For decryption we don't copy the
 369          * associated data.
 370          */
 371         for_each_sg(areq->assoc, cur, assoc_ents, i) {
 372                 ddt_set(src_ddt++, sg_dma_address(cur), sg_dma_len(cur));
 373                 if (req->is_encrypt)
 374                         ddt_set(dst_ddt++, sg_dma_address(cur),
 375                                 sg_dma_len(cur));
 376         }
 377         ddt_set(src_ddt++, iv_addr, ivsize);
 378
 379         if (giv || req->is_encrypt)
 380                 ddt_set(dst_ddt++, iv_addr, ivsize);
 381
 382         /*
 383          * Now map in the payload for the source and destination and terminate
 384          * with the NULL pointers.
 385          */
 386         for_each_sg(areq->src, cur, src_ents, i) {
 387                 ddt_set(src_ddt++, sg_dma_address(cur), sg_dma_len(cur));
 388                 if (areq->src == areq->dst)
 389                         ddt_set(dst_ddt++, sg_dma_address(cur),
 390                                 sg_dma_len(cur));
 391         }
 392
 393         for_each_sg(areq->dst, cur, dst_ents, i)
 394                 ddt_set(dst_ddt++, sg_dma_address(cur),
 395                         sg_dma_len(cur));
 396
 397         ddt_set(src_ddt, 0, 0);
 398         ddt_set(dst_ddt, 0, 0);
 399
 400         return 0;
 401 }
 402
 403 static void spacc_aead_free_ddts(struct spacc_req *req)
 404 {
 405         struct aead_request *areq = container_of(req->req, struct aead_request,
 406                                                  base);
 407         struct spacc_alg *alg = to_spacc_alg(req->req->tfm->__crt_alg);
 408         struct spacc_ablk_ctx *aead_ctx = crypto_tfm_ctx(req->req->tfm);
 409         struct spacc_engine *engine = aead_ctx->generic.engine;
 410         unsigned ivsize = alg->alg.cra_aead.ivsize;
 411         unsigned nents = sg_count(areq->src, areq->cryptlen);
 412
 413         if (areq->src != areq->dst) {
 414                 dma_unmap_sg(engine->dev, areq->src, nents, DMA_TO_DEVICE);
 415                 dma_unmap_sg(engine->dev, areq->dst,
 416                              sg_count(areq->dst, areq->cryptlen),
 417                              DMA_FROM_DEVICE);
 418         } else
 419                 dma_unmap_sg(engine->dev, areq->src, nents, DMA_BIDIRECTIONAL);
 420
 421         dma_unmap_sg(engine->dev, areq->assoc,
 422                      sg_count(areq->assoc, areq->assoclen), DMA_TO_DEVICE);
 423
 424         dma_unmap_single(engine->dev, req->giv_pa, ivsize, DMA_BIDIRECTIONAL);
 425
 426         dma_pool_free(engine->req_pool, req->src_ddt, req->src_addr);
 427         dma_pool_free(engine->req_pool, req->dst_ddt, req->dst_addr);
 428 }
 429
 430 static void spacc_free_ddt(struct spacc_req *req, struct spacc_ddt *ddt,
 431                            dma_addr_t ddt_addr, struct scatterlist *payload,
 432                            unsigned nbytes, enum dma_data_direction dir)
 433 {
 434         unsigned nents = sg_count(payload, nbytes);
 435
 436         dma_unmap_sg(req->engine->dev, payload, nents, dir);
 437         dma_pool_free(req->engine->req_pool, ddt, ddt_addr);
 438 }
 439
 440 /*
 441  * Set key for a DES operation in an AEAD cipher. This also performs weak key
 442  * checking if required.
 443  */
 444 static int spacc_aead_des_setkey(struct crypto_aead *aead, const u8 *key,
 445                                  unsigned int len)
 446 {
 447         struct crypto_tfm *tfm = crypto_aead_tfm(aead);
 448         struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm);
 449         u32 tmp[DES_EXPKEY_WORDS];
 450
 451         if (unlikely(!des_ekey(tmp, key)) &&
 452             (crypto_aead_get_flags(aead)) & CRYPTO_TFM_REQ_WEAK_KEY) {
 453                 tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
 454                 return -EINVAL;
 455         }
 456
 457         memcpy(ctx->cipher_key, key, len);
 458         ctx->cipher_key_len = len;
 459
 460         return 0;
 461 }
 462
 463 /* Set the key for the AES block cipher component of the AEAD transform. */
 464 static int spacc_aead_aes_setkey(struct crypto_aead *aead, const u8 *key,
 465                                  unsigned int len)
 466 {
 467         struct crypto_tfm *tfm = crypto_aead_tfm(aead);
 468         struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm);
 469
 470         /*
 471          * IPSec engine only supports 128 and 256 bit AES keys. If we get a
 472          * request for any other size (192 bits) then we need to do a software
 473          * fallback.
 474          */
 475         if (len != AES_KEYSIZE_128 && len != AES_KEYSIZE_256) {
 476                 /*
 477                  * Set the fallback transform to use the same request flags as
 478                  * the hardware transform.
 479                  */
 480                 ctx->sw_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
 481                 ctx->sw_cipher->base.crt_flags |=
 482                         tfm->crt_flags & CRYPTO_TFM_REQ_MASK;
 483                 return crypto_aead_setkey(ctx->sw_cipher, key, len);
 484         }
 485
 486         memcpy(ctx->cipher_key, key, len);
 487         ctx->cipher_key_len = len;
 488
 489         return 0;
 490 }
 491
 492 static int spacc_aead_setkey(struct crypto_aead *tfm, const u8 *key,
 493                              unsigned int keylen)
 494 {
 495         struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 496         struct spacc_alg *alg = to_spacc_alg(tfm->base.__crt_alg);
 497         struct rtattr *rta = (void *)key;
 498         struct crypto_authenc_key_param *param;
 499         unsigned int authkeylen, enckeylen;
 500         int err = -EINVAL;
 501
 502         if (!RTA_OK(rta, keylen))
 503                 goto badkey;
 504
 505         if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
 506                 goto badkey;
 507
 508         if (RTA_PAYLOAD(rta) < sizeof(*param))
 509                 goto badkey;
 510
 511         param = RTA_DATA(rta);
 512         enckeylen = be32_to_cpu(param->enckeylen);
 513
 514         key += RTA_ALIGN(rta->rta_len);
 515         keylen -= RTA_ALIGN(rta->rta_len);
 516
 517         if (keylen < enckeylen)
 518                 goto badkey;
 519
 520         authkeylen = keylen - enckeylen;
 521
 522         if (enckeylen > AES_MAX_KEY_SIZE)
 523                 goto badkey;
 524
 525         if ((alg->ctrl_default & SPACC_CRYPTO_ALG_MASK) ==
 526             SPA_CTRL_CIPH_ALG_AES)
 527                 err = spacc_aead_aes_setkey(tfm, key + authkeylen, enckeylen);
 528         else
 529                 err = spacc_aead_des_setkey(tfm, key + authkeylen, enckeylen);
 530
 531         if (err)
 532                 goto badkey;
 533
 534         memcpy(ctx->hash_ctx, key, authkeylen);
 535         ctx->hash_key_len = authkeylen;
 536
 537         return 0;
 538
 539 badkey:
 540         crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 541         return -EINVAL;
 542 }
 543
 544 static int spacc_aead_setauthsize(struct crypto_aead *tfm,
 545                                   unsigned int authsize)
 546 {
 547         struct spacc_aead_ctx *ctx = crypto_tfm_ctx(crypto_aead_tfm(tfm));
 548
 549         ctx->auth_size = authsize;
 550
 551         return 0;
 552 }
 553
 554 /*
 555  * Check if an AEAD request requires a fallback operation. Some requests can't
 556  * be completed in hardware because the hardware may not support certain key
 557  * sizes. In these cases we need to complete the request in software.
 558  */
 559 static int spacc_aead_need_fallback(struct spacc_req *req)
 560 {
 561         struct aead_request *aead_req;
 562         struct crypto_tfm *tfm = req->req->tfm;
 563         struct crypto_alg *alg = req->req->tfm->__crt_alg;
 564         struct spacc_alg *spacc_alg = to_spacc_alg(alg);
 565         struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm);
 566
 567         aead_req = container_of(req->req, struct aead_request, base);
 568         /*
 569          * If we have a non-supported key-length, then we need to do a
 570          * software fallback.
 571          */
 572         if ((spacc_alg->ctrl_default & SPACC_CRYPTO_ALG_MASK) ==
 573             SPA_CTRL_CIPH_ALG_AES &&
 574             ctx->cipher_key_len != AES_KEYSIZE_128 &&
 575             ctx->cipher_key_len != AES_KEYSIZE_256)
 576                 return 1;
 577
 578         return 0;
 579 }
 580
 581 static int spacc_aead_do_fallback(struct aead_request *req, unsigned alg_type,
 582                                   bool is_encrypt)
 583 {
 584         struct crypto_tfm *old_tfm = crypto_aead_tfm(crypto_aead_reqtfm(req));
 585         struct spacc_aead_ctx *ctx = crypto_tfm_ctx(old_tfm);
 586         int err;
 587
 588         if (ctx->sw_cipher) {
 589                 /*
 590                  * Change the request to use the software fallback transform,
 591                  * and once the ciphering has completed, put the old transform
 592                  * back into the request.
 593                  */
 594                 aead_request_set_tfm(req, ctx->sw_cipher);
 595                 err = is_encrypt ? crypto_aead_encrypt(req) :
 596                     crypto_aead_decrypt(req);
 597                 aead_request_set_tfm(req, __crypto_aead_cast(old_tfm));
 598         } else
 599                 err = -EINVAL;
 600
 601         return err;
 602 }
 603
 604 static void spacc_aead_complete(struct spacc_req *req)
 605 {
 606         spacc_aead_free_ddts(req);
 607         req->req->complete(req->req, req->result);
 608 }
 609
 610 static int spacc_aead_submit(struct spacc_req *req)
 611 {
 612         struct crypto_tfm *tfm = req->req->tfm;
 613         struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm);
 614         struct crypto_alg *alg = req->req->tfm->__crt_alg;
 615         struct spacc_alg *spacc_alg = to_spacc_alg(alg);
 616         struct spacc_engine *engine = ctx->generic.engine;
 617         u32 ctrl, proc_len, assoc_len;
 618         struct aead_request *aead_req =
 619                 container_of(req->req, struct aead_request, base);
 620
 621         req->result = -EINPROGRESS;
 622         req->ctx_id = spacc_load_ctx(&ctx->generic, ctx->cipher_key,
 623                 ctx->cipher_key_len, aead_req->iv, alg->cra_aead.ivsize,
 624                 ctx->hash_ctx, ctx->hash_key_len);
 625
 626         /* Set the source and destination DDT pointers. */
 627         writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET);
 628         writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET);
 629         writel(0, engine->regs + SPA_OFFSET_REG_OFFSET);
 630
 631         assoc_len = aead_req->assoclen;
 632         proc_len = aead_req->cryptlen + assoc_len;
 633
 634         /*
 635          * If we aren't generating an IV, then we need to include the IV in the
 636          * associated data so that it is included in the hash.
 637          */
 638         if (!req->giv) {
 639                 assoc_len += crypto_aead_ivsize(crypto_aead_reqtfm(aead_req));
 640                 proc_len += crypto_aead_ivsize(crypto_aead_reqtfm(aead_req));
 641         } else
 642                 proc_len += req->giv_len;
 643
 644         /*
 645          * If we are decrypting, we need to take the length of the ICV out of
 646          * the processing length.
 647          */
 648         if (!req->is_encrypt)
 649                 proc_len -= ctx->auth_size;
 650
 651         writel(proc_len, engine->regs + SPA_PROC_LEN_REG_OFFSET);
 652         writel(assoc_len, engine->regs + SPA_AAD_LEN_REG_OFFSET);
 653         writel(ctx->auth_size, engine->regs + SPA_ICV_LEN_REG_OFFSET);
 654         writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET);
 655         writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET);
 656
 657         ctrl = spacc_alg->ctrl_default | (req->ctx_id << SPA_CTRL_CTX_IDX) |
 658                 (1 << SPA_CTRL_ICV_APPEND);
 659         if (req->is_encrypt)
 660                 ctrl |= (1 << SPA_CTRL_ENCRYPT_IDX) | (1 << SPA_CTRL_AAD_COPY);
 661         else
 662                 ctrl |= (1 << SPA_CTRL_KEY_EXP);
 663
 664         mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT);
 665
 666         writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET);
 667
 668         return -EINPROGRESS;
 669 }
 670
 671 static int spacc_req_submit(struct spacc_req *req);
 672
 673 static void spacc_push(struct spacc_engine *engine)
 674 {
 675         struct spacc_req *req;
 676
 677         while (!list_empty(&engine->pending) &&
 678                engine->in_flight + 1 <= engine->fifo_sz) {
 679
 680                 ++engine->in_flight;
 681                 req = list_first_entry(&engine->pending, struct spacc_req,
 682                                        list);
 683                 list_move_tail(&req->list, &engine->in_progress);
 684
 685                 req->result = spacc_req_submit(req);
 686         }
 687 }
 688
 689 /*
 690  * Setup an AEAD request for processing. This will configure the engine, load
 691  * the context and then start the packet processing.
 692  *
 693  * @giv Pointer to destination address for a generated IV. If the
 694  *      request does not need to generate an IV then this should be set to NULL.
 695  */
 696 static int spacc_aead_setup(struct aead_request *req, u8 *giv,
 697                             unsigned alg_type, bool is_encrypt)
 698 {
 699         struct crypto_alg *alg = req->base.tfm->__crt_alg;
 700         struct spacc_engine *engine = to_spacc_alg(alg)->engine;
 701         struct spacc_req *dev_req = aead_request_ctx(req);
 702         int err = -EINPROGRESS;
 703         unsigned long flags;
 704         unsigned ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req));
 705
 706         dev_req->giv            = giv;
 707         dev_req->giv_len        = ivsize;
 708         dev_req->req            = &req->base;
 709         dev_req->is_encrypt     = is_encrypt;
 710         dev_req->result         = -EBUSY;
 711         dev_req->engine         = engine;
 712         dev_req->complete       = spacc_aead_complete;
 713
 714         if (unlikely(spacc_aead_need_fallback(dev_req)))
 715                 return spacc_aead_do_fallback(req, alg_type, is_encrypt);
 716
 717         spacc_aead_make_ddts(dev_req, dev_req->giv);
 718
 719         err = -EINPROGRESS;
 720         spin_lock_irqsave(&engine->hw_lock, flags);
 721         if (unlikely(spacc_fifo_cmd_full(engine)) ||
 722             engine->in_flight + 1 > engine->fifo_sz) {
 723                 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
 724                         err = -EBUSY;
 725                         spin_unlock_irqrestore(&engine->hw_lock, flags);
 726                         goto out_free_ddts;
 727                 }
 728                 list_add_tail(&dev_req->list, &engine->pending);
 729         } else {
 730                 list_add_tail(&dev_req->list, &engine->pending);
 731                 spacc_push(engine);
 732         }
 733         spin_unlock_irqrestore(&engine->hw_lock, flags);
 734
 735         goto out;
 736
 737 out_free_ddts:
 738         spacc_aead_free_ddts(dev_req);
 739 out:
 740         return err;
 741 }
 742
 743 static int spacc_aead_encrypt(struct aead_request *req)
 744 {
 745         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 746         struct crypto_tfm *tfm = crypto_aead_tfm(aead);
 747         struct spacc_alg *alg = to_spacc_alg(tfm->__crt_alg);
 748
 749         return spacc_aead_setup(req, NULL, alg->type, 1);
 750 }
 751
 752 static int spacc_aead_givencrypt(struct aead_givcrypt_request *req)
 753 {
 754         struct crypto_aead *tfm = aead_givcrypt_reqtfm(req);
 755         struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 756         size_t ivsize = crypto_aead_ivsize(tfm);
 757         struct spacc_alg *alg = to_spacc_alg(tfm->base.__crt_alg);
 758         unsigned len;
 759         __be64 seq;
 760
 761         memcpy(req->areq.iv, ctx->salt, ivsize);
 762         len = ivsize;
 763         if (ivsize > sizeof(u64)) {
 764                 memset(req->giv, 0, ivsize - sizeof(u64));
 765                 len = sizeof(u64);
 766         }
 767         seq = cpu_to_be64(req->seq);
 768         memcpy(req->giv + ivsize - len, &seq, len);
 769
 770         return spacc_aead_setup(&req->areq, req->giv, alg->type, 1);
 771 }
 772
 773 static int spacc_aead_decrypt(struct aead_request *req)
 774 {
 775         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 776         struct crypto_tfm *tfm = crypto_aead_tfm(aead);
 777         struct spacc_alg *alg = to_spacc_alg(tfm->__crt_alg);
 778
 779         return spacc_aead_setup(req, NULL, alg->type, 0);
 780 }
 781
 782 /*
 783  * Initialise a new AEAD context. This is responsible for allocating the
 784  * fallback cipher and initialising the context.
 785  */
 786 static int spacc_aead_cra_init(struct crypto_tfm *tfm)
 787 {
 788         struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm);
 789         struct crypto_alg *alg = tfm->__crt_alg;
 790         struct spacc_alg *spacc_alg = to_spacc_alg(alg);
 791         struct spacc_engine *engine = spacc_alg->engine;
 792
 793         ctx->generic.flags = spacc_alg->type;
 794         ctx->generic.engine = engine;
 795         ctx->sw_cipher = crypto_alloc_aead(alg->cra_name, 0,
 796                                            CRYPTO_ALG_ASYNC |
 797                                            CRYPTO_ALG_NEED_FALLBACK);
 798         if (IS_ERR(ctx->sw_cipher)) {
 799                 dev_warn(engine->dev, "failed to allocate fallback for %s\n",
 800                          alg->cra_name);
 801                 ctx->sw_cipher = NULL;
 802         }
 803         ctx->generic.key_offs = spacc_alg->key_offs;
 804         ctx->generic.iv_offs = spacc_alg->iv_offs;
 805
 806         get_random_bytes(ctx->salt, sizeof(ctx->salt));
 807
 808         tfm->crt_aead.reqsize = sizeof(struct spacc_req);
 809
 810         return 0;
 811 }
 812
 813 /*
 814  * Destructor for an AEAD context. This is called when the transform is freed
 815  * and must free the fallback cipher.
 816  */
 817 static void spacc_aead_cra_exit(struct crypto_tfm *tfm)
 818 {
 819         struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm);
 820
 821         if (ctx->sw_cipher)
 822                 crypto_free_aead(ctx->sw_cipher);
 823         ctx->sw_cipher = NULL;
 824 }
 825
 826 /*
 827  * Set the DES key for a block cipher transform. This also performs weak key
 828  * checking if the transform has requested it.
 829  */
 830 static int spacc_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
 831                             unsigned int len)
 832 {
 833         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
 834         struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm);
 835         u32 tmp[DES_EXPKEY_WORDS];
 836
 837         if (len > DES3_EDE_KEY_SIZE) {
 838                 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
 839                 return -EINVAL;
 840         }
 841
 842         if (unlikely(!des_ekey(tmp, key)) &&
 843             (crypto_ablkcipher_get_flags(cipher) & CRYPTO_TFM_REQ_WEAK_KEY)) {
 844                 tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
 845                 return -EINVAL;
 846         }
 847
 848         memcpy(ctx->key, key, len);
 849         ctx->key_len = len;
 850
 851         return 0;
 852 }
 853
 854 /*
 855  * Set the key for an AES block cipher. Some key lengths are not supported in
 856  * hardware so this must also check whether a fallback is needed.
 857  */
 858 static int spacc_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
 859                             unsigned int len)
 860 {
 861         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
 862         struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm);
 863         int err = 0;
 864
 865         if (len > AES_MAX_KEY_SIZE) {
 866                 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
 867                 return -EINVAL;
 868         }
 869
 870         /*
 871          * IPSec engine only supports 128 and 256 bit AES keys. If we get a
 872          * request for any other size (192 bits) then we need to do a software
 873          * fallback.
 874          */
 875         if ((len != AES_KEYSIZE_128 || len != AES_KEYSIZE_256) &&
 876             ctx->sw_cipher) {
 877                 /*
 878                  * Set the fallback transform to use the same request flags as
 879                  * the hardware transform.
 880                  */
 881                 ctx->sw_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
 882                 ctx->sw_cipher->base.crt_flags |=
 883                         cipher->base.crt_flags & CRYPTO_TFM_REQ_MASK;
 884
 885                 err = crypto_ablkcipher_setkey(ctx->sw_cipher, key, len);
 886                 if (err)
 887                         goto sw_setkey_failed;
 888         } else if ((len != AES_KEYSIZE_128 || len != AES_KEYSIZE_256) &&
 889                    !ctx->sw_cipher)
 890                 err = -EINVAL;
 891
 892         memcpy(ctx->key, key, len);
 893         ctx->key_len = len;
 894
 895 sw_setkey_failed:
 896         if (err && ctx->sw_cipher) {
 897                 tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
 898                 tfm->crt_flags |=
 899                         ctx->sw_cipher->base.crt_flags & CRYPTO_TFM_RES_MASK;
 900         }
 901
 902         return err;
 903 }
 904
 905 static int spacc_kasumi_f8_setkey(struct crypto_ablkcipher *cipher,
 906                                   const u8 *key, unsigned int len)
 907 {
 908         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
 909         struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm);
 910         int err = 0;
 911
 912         if (len > AES_MAX_KEY_SIZE) {
 913                 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
 914                 err = -EINVAL;
 915                 goto out;
 916         }
 917
 918         memcpy(ctx->key, key, len);
 919         ctx->key_len = len;
 920
 921 out:
 922         return err;
 923 }
 924
 925 static int spacc_ablk_need_fallback(struct spacc_req *req)
 926 {
 927         struct spacc_ablk_ctx *ctx;
 928         struct crypto_tfm *tfm = req->req->tfm;
 929         struct crypto_alg *alg = req->req->tfm->__crt_alg;
 930         struct spacc_alg *spacc_alg = to_spacc_alg(alg);
 931
 932         ctx = crypto_tfm_ctx(tfm);
 933
 934         return (spacc_alg->ctrl_default & SPACC_CRYPTO_ALG_MASK) ==
 935                         SPA_CTRL_CIPH_ALG_AES &&
 936                         ctx->key_len != AES_KEYSIZE_128 &&
 937                         ctx->key_len != AES_KEYSIZE_256;
 938 }
 939
 940 static void spacc_ablk_complete(struct spacc_req *req)
 941 {
 942         struct ablkcipher_request *ablk_req =
 943                 container_of(req->req, struct ablkcipher_request, base);
 944
 945         if (ablk_req->src != ablk_req->dst) {
 946                 spacc_free_ddt(req, req->src_ddt, req->src_addr, ablk_req->src,
 947                                ablk_req->nbytes, DMA_TO_DEVICE);
 948                 spacc_free_ddt(req, req->dst_ddt, req->dst_addr, ablk_req->dst,
 949                                ablk_req->nbytes, DMA_FROM_DEVICE);
 950         } else
 951                 spacc_free_ddt(req, req->dst_ddt, req->dst_addr, ablk_req->dst,
 952                                ablk_req->nbytes, DMA_BIDIRECTIONAL);
 953
 954         req->req->complete(req->req, req->result);
 955 }
 956
 957 static int spacc_ablk_submit(struct spacc_req *req)
 958 {
 959         struct crypto_tfm *tfm = req->req->tfm;
 960         struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm);
 961         struct ablkcipher_request *ablk_req = ablkcipher_request_cast(req->req);
 962         struct crypto_alg *alg = req->req->tfm->__crt_alg;
 963         struct spacc_alg *spacc_alg = to_spacc_alg(alg);
 964         struct spacc_engine *engine = ctx->generic.engine;
 965         u32 ctrl;
 966
 967         req->ctx_id = spacc_load_ctx(&ctx->generic, ctx->key,
 968                 ctx->key_len, ablk_req->info, alg->cra_ablkcipher.ivsize,
 969                 NULL, 0);
 970
 971         writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET);
 972         writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET);
 973         writel(0, engine->regs + SPA_OFFSET_REG_OFFSET);
 974
 975         writel(ablk_req->nbytes, engine->regs + SPA_PROC_LEN_REG_OFFSET);
 976         writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET);
 977         writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET);
 978         writel(0, engine->regs + SPA_AAD_LEN_REG_OFFSET);
 979
 980         ctrl = spacc_alg->ctrl_default | (req->ctx_id << SPA_CTRL_CTX_IDX) |
 981                 (req->is_encrypt ? (1 << SPA_CTRL_ENCRYPT_IDX) :
 982                  (1 << SPA_CTRL_KEY_EXP));
 983
 984         mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT);
 985
 986         writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET);
 987
 988         return -EINPROGRESS;
 989 }
 990
 991 static int spacc_ablk_do_fallback(struct ablkcipher_request *req,
 992                                   unsigned alg_type, bool is_encrypt)
 993 {
 994         struct crypto_tfm *old_tfm =
 995             crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
 996         struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(old_tfm);
 997         int err;
 998
 999         if (!ctx->sw_cipher)
1000                 return -EINVAL;
1001
1002         /*
1003          * Change the request to use the software fallback transform, and once
1004          * the ciphering has completed, put the old transform back into the
1005          * request.
1006          */
1007         ablkcipher_request_set_tfm(req, ctx->sw_cipher);
1008         err = is_encrypt ? crypto_ablkcipher_encrypt(req) :
1009                 crypto_ablkcipher_decrypt(req);
1010         ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(old_tfm));
1011
1012         return err;
1013 }
1014
1015 static int spacc_ablk_setup(struct ablkcipher_request *req, unsigned alg_type,
1016                             bool is_encrypt)
1017 {
1018         struct crypto_alg *alg = req->base.tfm->__crt_alg;
1019         struct spacc_engine *engine = to_spacc_alg(alg)->engine;
1020         struct spacc_req *dev_req = ablkcipher_request_ctx(req);
1021         unsigned long flags;
1022         int err = -ENOMEM;
1023
1024         dev_req->req            = &req->base;
1025         dev_req->is_encrypt     = is_encrypt;
1026         dev_req->engine         = engine;
1027         dev_req->complete       = spacc_ablk_complete;
1028         dev_req->result         = -EINPROGRESS;
1029
1030         if (unlikely(spacc_ablk_need_fallback(dev_req)))
1031                 return spacc_ablk_do_fallback(req, alg_type, is_encrypt);
1032
1033         /*
1034          * Create the DDT's for the engine. If we share the same source and
1035          * destination then we can optimize by reusing the DDT's.
1036          */
1037         if (req->src != req->dst) {
1038                 dev_req->src_ddt = spacc_sg_to_ddt(engine, req->src,
1039                         req->nbytes, DMA_TO_DEVICE, &dev_req->src_addr);
1040                 if (!dev_req->src_ddt)
1041                         goto out;
1042
1043                 dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst,
1044                         req->nbytes, DMA_FROM_DEVICE, &dev_req->dst_addr);
1045                 if (!dev_req->dst_ddt)
1046                         goto out_free_src;
1047         } else {
1048                 dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst,
1049                         req->nbytes, DMA_BIDIRECTIONAL, &dev_req->dst_addr);
1050                 if (!dev_req->dst_ddt)
1051                         goto out;
1052
1053                 dev_req->src_ddt = NULL;
1054                 dev_req->src_addr = dev_req->dst_addr;
1055         }
1056
1057         err = -EINPROGRESS;
1058         spin_lock_irqsave(&engine->hw_lock, flags);
1059         /*
1060          * Check if the engine will accept the operation now. If it won't then
1061          * we either stick it on the end of a pending list if we can backlog,
1062          * or bailout with an error if not.
1063          */
1064         if (unlikely(spacc_fifo_cmd_full(engine)) ||
1065             engine->in_flight + 1 > engine->fifo_sz) {
1066                 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1067                         err = -EBUSY;
1068                         spin_unlock_irqrestore(&engine->hw_lock, flags);
1069                         goto out_free_ddts;
1070                 }
1071                 list_add_tail(&dev_req->list, &engine->pending);
1072         } else {
1073                 list_add_tail(&dev_req->list, &engine->pending);
1074                 spacc_push(engine);
1075         }
1076         spin_unlock_irqrestore(&engine->hw_lock, flags);
1077
1078         goto out;
1079
1080 out_free_ddts:
1081         spacc_free_ddt(dev_req, dev_req->dst_ddt, dev_req->dst_addr, req->dst,
1082                        req->nbytes, req->src == req->dst ?
1083                        DMA_BIDIRECTIONAL : DMA_FROM_DEVICE);
1084 out_free_src:
1085         if (req->src != req->dst)
1086                 spacc_free_ddt(dev_req, dev_req->src_ddt, dev_req->src_addr,
1087                                req->src, req->nbytes, DMA_TO_DEVICE);
1088 out:
1089         return err;
1090 }
1091
1092 static int spacc_ablk_cra_init(struct crypto_tfm *tfm)
1093 {
1094         struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm);
1095         struct crypto_alg *alg = tfm->__crt_alg;
1096         struct spacc_alg *spacc_alg = to_spacc_alg(alg);
1097         struct spacc_engine *engine = spacc_alg->engine;
1098
1099         ctx->generic.flags = spacc_alg->type;
1100         ctx->generic.engine = engine;
1101         if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
1102                 ctx->sw_cipher = crypto_alloc_ablkcipher(alg->cra_name, 0,
1103                                 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
1104                 if (IS_ERR(ctx->sw_cipher)) {
1105                         dev_warn(engine->dev, "failed to allocate fallback for %s\n",
1106                                  alg->cra_name);
1107                         ctx->sw_cipher = NULL;
1108                 }
1109         }
1110         ctx->generic.key_offs = spacc_alg->key_offs;
1111         ctx->generic.iv_offs = spacc_alg->iv_offs;
1112
1113         tfm->crt_ablkcipher.reqsize = sizeof(struct spacc_req);
1114
1115         return 0;
1116 }
1117
1118 static void spacc_ablk_cra_exit(struct crypto_tfm *tfm)
1119 {
1120         struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm);
1121
1122         if (ctx->sw_cipher)
1123                 crypto_free_ablkcipher(ctx->sw_cipher);
1124         ctx->sw_cipher = NULL;
1125 }
1126
1127 static int spacc_ablk_encrypt(struct ablkcipher_request *req)
1128 {
1129         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(req);
1130         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
1131         struct spacc_alg *alg = to_spacc_alg(tfm->__crt_alg);
1132
1133         return spacc_ablk_setup(req, alg->type, 1);
1134 }
1135
1136 static int spacc_ablk_decrypt(struct ablkcipher_request *req)
1137 {
1138         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(req);
1139         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
1140         struct spacc_alg *alg = to_spacc_alg(tfm->__crt_alg);
1141
1142         return spacc_ablk_setup(req, alg->type, 0);
1143 }
1144
1145 static inline int spacc_fifo_stat_empty(struct spacc_engine *engine)
1146 {
1147         return readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET) &
1148                 SPA_FIFO_STAT_EMPTY;
1149 }
1150
1151 static void spacc_process_done(struct spacc_engine *engine)
1152 {
1153         struct spacc_req *req;
1154         unsigned long flags;
1155
1156         spin_lock_irqsave(&engine->hw_lock, flags);
1157
1158         while (!spacc_fifo_stat_empty(engine)) {
1159                 req = list_first_entry(&engine->in_progress, struct spacc_req,
1160                                        list);
1161                 list_move_tail(&req->list, &engine->completed);
1162                 --engine->in_flight;
1163
1164                 /* POP the status register. */
1165                 writel(~0, engine->regs + SPA_STAT_POP_REG_OFFSET);
1166                 req->result = (readl(engine->regs + SPA_STATUS_REG_OFFSET) &
1167                      SPA_STATUS_RES_CODE_MASK) >> SPA_STATUS_RES_CODE_OFFSET;
1168
1169                 /*
1170                  * Convert the SPAcc error status into the standard POSIX error
1171                  * codes.
1172                  */
1173                 if (unlikely(req->result)) {
1174                         switch (req->result) {
1175                         case SPA_STATUS_ICV_FAIL:
1176                                 req->result = -EBADMSG;
1177                                 break;
1178
1179                         case SPA_STATUS_MEMORY_ERROR:
1180                                 dev_warn(engine->dev,
1181                                          "memory error triggered\n");
1182                                 req->result = -EFAULT;
1183                                 break;
1184
1185                         case SPA_STATUS_BLOCK_ERROR:
1186                                 dev_warn(engine->dev,
1187                                          "block error triggered\n");
1188                                 req->result = -EIO;
1189                                 break;
1190                         }
1191                 }
1192         }
1193
1194         tasklet_schedule(&engine->complete);
1195
1196         spin_unlock_irqrestore(&engine->hw_lock, flags);
1197 }
1198
1199 static irqreturn_t spacc_spacc_irq(int irq, void *dev)
1200 {
1201         struct spacc_engine *engine = (struct spacc_engine *)dev;
1202         u32 spacc_irq_stat = readl(engine->regs + SPA_IRQ_STAT_REG_OFFSET);
1203
1204         writel(spacc_irq_stat, engine->regs + SPA_IRQ_STAT_REG_OFFSET);
1205         spacc_process_done(engine);
1206
1207         return IRQ_HANDLED;
1208 }
1209
1210 static void spacc_packet_timeout(unsigned long data)
1211 {
1212         struct spacc_engine *engine = (struct spacc_engine *)data;
1213
1214         spacc_process_done(engine);
1215 }
1216
1217 static int spacc_req_submit(struct spacc_req *req)
1218 {
1219         struct crypto_alg *alg = req->req->tfm->__crt_alg;
1220
1221         if (CRYPTO_ALG_TYPE_AEAD == (CRYPTO_ALG_TYPE_MASK & alg->cra_flags))
1222                 return spacc_aead_submit(req);
1223         else
1224                 return spacc_ablk_submit(req);
1225 }
1226
1227 static void spacc_spacc_complete(unsigned long data)
1228 {
1229         struct spacc_engine *engine = (struct spacc_engine *)data;
1230         struct spacc_req *req, *tmp;
1231         unsigned long flags;
1232         LIST_HEAD(completed);
1233
1234         spin_lock_irqsave(&engine->hw_lock, flags);
1235
1236         list_splice_init(&engine->completed, &completed);
1237         spacc_push(engine);
1238         if (engine->in_flight)
1239                 mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT);
1240
1241         spin_unlock_irqrestore(&engine->hw_lock, flags);
1242
1243         list_for_each_entry_safe(req, tmp, &completed, list) {
1244                 req->complete(req);
1245                 list_del(&req->list);
1246         }
1247 }
1248
1249 #ifdef CONFIG_PM
1250 static int spacc_suspend(struct device *dev)
1251 {
1252         struct platform_device *pdev = to_platform_device(dev);
1253         struct spacc_engine *engine = platform_get_drvdata(pdev);
1254
1255         /*
1256          * We only support standby mode. All we have to do is gate the clock to
1257          * the spacc. The hardware will preserve state until we turn it back
1258          * on again.
1259          */
1260         clk_disable(engine->clk);
1261
1262         return 0;
1263 }
1264
1265 static int spacc_resume(struct device *dev)
1266 {
1267         struct platform_device *pdev = to_platform_device(dev);
1268         struct spacc_engine *engine = platform_get_drvdata(pdev);
1269
1270         return clk_enable(engine->clk);
1271 }
1272
1273 static const struct dev_pm_ops spacc_pm_ops = {
1274         .suspend        = spacc_suspend,
1275         .resume         = spacc_resume,
1276 };
1277 #endif /* CONFIG_PM */
1278
1279 static inline struct spacc_engine *spacc_dev_to_engine(struct device *dev)
1280 {
1281         return dev ? platform_get_drvdata(to_platform_device(dev)) : NULL;
1282 }
1283
1284 static ssize_t spacc_stat_irq_thresh_show(struct device *dev,
1285                                           struct device_attribute *attr,
1286                                           char *buf)
1287 {
1288         struct spacc_engine *engine = spacc_dev_to_engine(dev);
1289
1290         return snprintf(buf, PAGE_SIZE, "%u\n", engine->stat_irq_thresh);
1291 }
1292
1293 static ssize_t spacc_stat_irq_thresh_store(struct device *dev,
1294                                            struct device_attribute *attr,
1295                                            const char *buf, size_t len)
1296 {
1297         struct spacc_engine *engine = spacc_dev_to_engine(dev);
1298         unsigned long thresh;
1299
1300         if (strict_strtoul(buf, 0, &thresh))
1301                 return -EINVAL;
1302
1303         thresh = clamp(thresh, 1UL, engine->fifo_sz - 1);
1304
1305         engine->stat_irq_thresh = thresh;
1306         writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET,
1307                engine->regs + SPA_IRQ_CTRL_REG_OFFSET);
1308
1309         return len;
1310 }
1311 static DEVICE_ATTR(stat_irq_thresh, 0644, spacc_stat_irq_thresh_show,
1312                    spacc_stat_irq_thresh_store);
1313
1314 static struct spacc_alg ipsec_engine_algs[] = {
1315         {
1316                 .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_CBC,
1317                 .key_offs = 0,
1318                 .iv_offs = AES_MAX_KEY_SIZE,
1319                 .alg = {
1320                         .cra_name = "cbc(aes)",
1321                         .cra_driver_name = "cbc-aes-picoxcell",
1322                         .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1323                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1324                                      CRYPTO_ALG_ASYNC |
1325                                      CRYPTO_ALG_NEED_FALLBACK,
1326                         .cra_blocksize = AES_BLOCK_SIZE,
1327                         .cra_ctxsize = sizeof(struct spacc_ablk_ctx),
1328                         .cra_type = &crypto_ablkcipher_type,
1329                         .cra_module = THIS_MODULE,
1330                         .cra_ablkcipher = {
1331                                 .setkey = spacc_aes_setkey,
1332                                 .encrypt = spacc_ablk_encrypt,
1333                                 .decrypt = spacc_ablk_decrypt,
1334                                 .min_keysize = AES_MIN_KEY_SIZE,
1335                                 .max_keysize = AES_MAX_KEY_SIZE,
1336                                 .ivsize = AES_BLOCK_SIZE,
1337                         },
1338                         .cra_init = spacc_ablk_cra_init,
1339                         .cra_exit = spacc_ablk_cra_exit,
1340                 },
1341         },
1342         {
1343                 .key_offs = 0,
1344                 .iv_offs = AES_MAX_KEY_SIZE,
1345                 .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_ECB,
1346                 .alg = {
1347                         .cra_name = "ecb(aes)",
1348                         .cra_driver_name = "ecb-aes-picoxcell",
1349                         .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1350                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1351                                 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1352                         .cra_blocksize = AES_BLOCK_SIZE,
1353                         .cra_ctxsize = sizeof(struct spacc_ablk_ctx),
1354                         .cra_type = &crypto_ablkcipher_type,
1355                         .cra_module = THIS_MODULE,
1356                         .cra_ablkcipher = {
1357                                 .setkey = spacc_aes_setkey,
1358                                 .encrypt = spacc_ablk_encrypt,
1359                                 .decrypt = spacc_ablk_decrypt,
1360                                 .min_keysize = AES_MIN_KEY_SIZE,
1361                                 .max_keysize = AES_MAX_KEY_SIZE,
1362                         },
1363                         .cra_init = spacc_ablk_cra_init,
1364                         .cra_exit = spacc_ablk_cra_exit,
1365                 },
1366         },
1367         {
1368                 .key_offs = DES_BLOCK_SIZE,
1369                 .iv_offs = 0,
1370                 .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_CBC,
1371                 .alg = {
1372                         .cra_name = "cbc(des)",
1373                         .cra_driver_name = "cbc-des-picoxcell",
1374                         .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1375                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1376                         .cra_blocksize = DES_BLOCK_SIZE,
1377                         .cra_ctxsize = sizeof(struct spacc_ablk_ctx),
1378                         .cra_type = &crypto_ablkcipher_type,
1379                         .cra_module = THIS_MODULE,
1380                         .cra_ablkcipher = {
1381                                 .setkey = spacc_des_setkey,
1382                                 .encrypt = spacc_ablk_encrypt,
1383                                 .decrypt = spacc_ablk_decrypt,
1384                                 .min_keysize = DES_KEY_SIZE,
1385                                 .max_keysize = DES_KEY_SIZE,
1386                                 .ivsize = DES_BLOCK_SIZE,
1387                         },
1388                         .cra_init = spacc_ablk_cra_init,
1389                         .cra_exit = spacc_ablk_cra_exit,
1390                 },
1391         },
1392         {
1393                 .key_offs = DES_BLOCK_SIZE,
1394                 .iv_offs = 0,
1395                 .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_ECB,
1396                 .alg = {
1397                         .cra_name = "ecb(des)",
1398                         .cra_driver_name = "ecb-des-picoxcell",
1399                         .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1400                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1401                         .cra_blocksize = DES_BLOCK_SIZE,
1402                         .cra_ctxsize = sizeof(struct spacc_ablk_ctx),
1403                         .cra_type = &crypto_ablkcipher_type,
1404                         .cra_module = THIS_MODULE,
1405                         .cra_ablkcipher = {
1406                                 .setkey = spacc_des_setkey,
1407                                 .encrypt = spacc_ablk_encrypt,
1408                                 .decrypt = spacc_ablk_decrypt,
1409                                 .min_keysize = DES_KEY_SIZE,
1410                                 .max_keysize = DES_KEY_SIZE,
1411                         },
1412                         .cra_init = spacc_ablk_cra_init,
1413                         .cra_exit = spacc_ablk_cra_exit,
1414                 },
1415         },
1416         {
1417                 .key_offs = DES_BLOCK_SIZE,
1418                 .iv_offs = 0,
1419                 .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_CBC,
1420                 .alg = {
1421                         .cra_name = "cbc(des3_ede)",
1422                         .cra_driver_name = "cbc-des3-ede-picoxcell",
1423                         .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1424                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1425                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1426                         .cra_ctxsize = sizeof(struct spacc_ablk_ctx),
1427                         .cra_type = &crypto_ablkcipher_type,
1428                         .cra_module = THIS_MODULE,
1429                         .cra_ablkcipher = {
1430                                 .setkey = spacc_des_setkey,
1431                                 .encrypt = spacc_ablk_encrypt,
1432                                 .decrypt = spacc_ablk_decrypt,
1433                                 .min_keysize = DES3_EDE_KEY_SIZE,
1434                                 .max_keysize = DES3_EDE_KEY_SIZE,
1435                                 .ivsize = DES3_EDE_BLOCK_SIZE,
1436                         },
1437                         .cra_init = spacc_ablk_cra_init,
1438                         .cra_exit = spacc_ablk_cra_exit,
1439                 },
1440         },
1441         {
1442                 .key_offs = DES_BLOCK_SIZE,
1443                 .iv_offs = 0,
1444                 .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_ECB,
1445                 .alg = {
1446                         .cra_name = "ecb(des3_ede)",
1447                         .cra_driver_name = "ecb-des3-ede-picoxcell",
1448                         .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1449                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1450                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1451                         .cra_ctxsize = sizeof(struct spacc_ablk_ctx),
1452                         .cra_type = &crypto_ablkcipher_type,
1453                         .cra_module = THIS_MODULE,
1454                         .cra_ablkcipher = {
1455                                 .setkey = spacc_des_setkey,
1456                                 .encrypt = spacc_ablk_encrypt,
1457                                 .decrypt = spacc_ablk_decrypt,
1458                                 .min_keysize = DES3_EDE_KEY_SIZE,
1459                                 .max_keysize = DES3_EDE_KEY_SIZE,
1460                         },
1461                         .cra_init = spacc_ablk_cra_init,
1462                         .cra_exit = spacc_ablk_cra_exit,
1463                 },
1464         },
1465         {
1466                 .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_CBC |
1467                                 SPA_CTRL_HASH_ALG_SHA | SPA_CTRL_HASH_MODE_HMAC,
1468                 .key_offs = 0,
1469                 .iv_offs = AES_MAX_KEY_SIZE,
1470                 .alg = {
1471                         .cra_name = "authenc(hmac(sha1),cbc(aes))",
1472                         .cra_driver_name = "authenc-hmac-sha1-cbc-aes-picoxcell",
1473                         .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1474                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1475                         .cra_blocksize = AES_BLOCK_SIZE,
1476                         .cra_ctxsize = sizeof(struct spacc_aead_ctx),
1477                         .cra_type = &crypto_aead_type,
1478                         .cra_module = THIS_MODULE,
1479                         .cra_aead = {
1480                                 .setkey = spacc_aead_setkey,
1481                                 .setauthsize = spacc_aead_setauthsize,
1482                                 .encrypt = spacc_aead_encrypt,
1483                                 .decrypt = spacc_aead_decrypt,
1484                                 .givencrypt = spacc_aead_givencrypt,
1485                                 .ivsize = AES_BLOCK_SIZE,
1486                                 .maxauthsize = SHA1_DIGEST_SIZE,
1487                         },
1488                         .cra_init = spacc_aead_cra_init,
1489                         .cra_exit = spacc_aead_cra_exit,
1490                 },
1491         },
1492         {
1493                 .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_CBC |
1494                                 SPA_CTRL_HASH_ALG_SHA256 |
1495                                 SPA_CTRL_HASH_MODE_HMAC,
1496                 .key_offs = 0,
1497                 .iv_offs = AES_MAX_KEY_SIZE,
1498                 .alg = {
1499                         .cra_name = "authenc(hmac(sha256),cbc(aes))",
1500                         .cra_driver_name = "authenc-hmac-sha256-cbc-aes-picoxcell",
1501                         .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1502                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1503                         .cra_blocksize = AES_BLOCK_SIZE,
1504                         .cra_ctxsize = sizeof(struct spacc_aead_ctx),
1505                         .cra_type = &crypto_aead_type,
1506                         .cra_module = THIS_MODULE,
1507                         .cra_aead = {
1508                                 .setkey = spacc_aead_setkey,
1509                                 .setauthsize = spacc_aead_setauthsize,
1510                                 .encrypt = spacc_aead_encrypt,
1511                                 .decrypt = spacc_aead_decrypt,
1512                                 .givencrypt = spacc_aead_givencrypt,
1513                                 .ivsize = AES_BLOCK_SIZE,
1514                                 .maxauthsize = SHA256_DIGEST_SIZE,
1515                         },
1516                         .cra_init = spacc_aead_cra_init,
1517                         .cra_exit = spacc_aead_cra_exit,
1518                 },
1519         },
1520         {
1521                 .key_offs = 0,
1522                 .iv_offs = AES_MAX_KEY_SIZE,
1523                 .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_CBC |
1524                                 SPA_CTRL_HASH_ALG_MD5 | SPA_CTRL_HASH_MODE_HMAC,
1525                 .alg = {
1526                         .cra_name = "authenc(hmac(md5),cbc(aes))",
1527                         .cra_driver_name = "authenc-hmac-md5-cbc-aes-picoxcell",
1528                         .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1529                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1530                         .cra_blocksize = AES_BLOCK_SIZE,
1531                         .cra_ctxsize = sizeof(struct spacc_aead_ctx),
1532                         .cra_type = &crypto_aead_type,
1533                         .cra_module = THIS_MODULE,
1534                         .cra_aead = {
1535                                 .setkey = spacc_aead_setkey,
1536                                 .setauthsize = spacc_aead_setauthsize,
1537                                 .encrypt = spacc_aead_encrypt,
1538                                 .decrypt = spacc_aead_decrypt,
1539                                 .givencrypt = spacc_aead_givencrypt,
1540                                 .ivsize = AES_BLOCK_SIZE,
1541                                 .maxauthsize = MD5_DIGEST_SIZE,
1542                         },
1543                         .cra_init = spacc_aead_cra_init,
1544                         .cra_exit = spacc_aead_cra_exit,
1545                 },
1546         },
1547         {
1548                 .key_offs = DES_BLOCK_SIZE,
1549                 .iv_offs = 0,
1550                 .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_CBC |
1551                                 SPA_CTRL_HASH_ALG_SHA | SPA_CTRL_HASH_MODE_HMAC,
1552                 .alg = {
1553                         .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
1554                         .cra_driver_name = "authenc-hmac-sha1-cbc-3des-picoxcell",
1555                         .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1556                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1557                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1558                         .cra_ctxsize = sizeof(struct spacc_aead_ctx),
1559                         .cra_type = &crypto_aead_type,
1560                         .cra_module = THIS_MODULE,
1561                         .cra_aead = {
1562                                 .setkey = spacc_aead_setkey,
1563                                 .setauthsize = spacc_aead_setauthsize,
1564                                 .encrypt = spacc_aead_encrypt,
1565                                 .decrypt = spacc_aead_decrypt,
1566                                 .givencrypt = spacc_aead_givencrypt,
1567                                 .ivsize = DES3_EDE_BLOCK_SIZE,
1568                                 .maxauthsize = SHA1_DIGEST_SIZE,
1569                         },
1570                         .cra_init = spacc_aead_cra_init,
1571                         .cra_exit = spacc_aead_cra_exit,
1572                 },
1573         },
1574         {
1575                 .key_offs = DES_BLOCK_SIZE,
1576                 .iv_offs = 0,
1577                 .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_CBC |
1578                                 SPA_CTRL_HASH_ALG_SHA256 |
1579                                 SPA_CTRL_HASH_MODE_HMAC,
1580                 .alg = {
1581                         .cra_name = "authenc(hmac(sha256),cbc(des3_ede))",
1582                         .cra_driver_name = "authenc-hmac-sha256-cbc-3des-picoxcell",
1583                         .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1584                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1585                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1586                         .cra_ctxsize = sizeof(struct spacc_aead_ctx),
1587                         .cra_type = &crypto_aead_type,
1588                         .cra_module = THIS_MODULE,
1589                         .cra_aead = {
1590                                 .setkey = spacc_aead_setkey,
1591                                 .setauthsize = spacc_aead_setauthsize,
1592                                 .encrypt = spacc_aead_encrypt,
1593                                 .decrypt = spacc_aead_decrypt,
1594                                 .givencrypt = spacc_aead_givencrypt,
1595                                 .ivsize = DES3_EDE_BLOCK_SIZE,
1596                                 .maxauthsize = SHA256_DIGEST_SIZE,
1597                         },
1598                         .cra_init = spacc_aead_cra_init,
1599                         .cra_exit = spacc_aead_cra_exit,
1600                 },
1601         },
1602         {
1603                 .key_offs = DES_BLOCK_SIZE,
1604                 .iv_offs = 0,
1605                 .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_CBC |
1606                                 SPA_CTRL_HASH_ALG_MD5 | SPA_CTRL_HASH_MODE_HMAC,
1607                 .alg = {
1608                         .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
1609                         .cra_driver_name = "authenc-hmac-md5-cbc-3des-picoxcell",
1610                         .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1611                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1612                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1613                         .cra_ctxsize = sizeof(struct spacc_aead_ctx),
1614                         .cra_type = &crypto_aead_type,
1615                         .cra_module = THIS_MODULE,
1616                         .cra_aead = {
1617                                 .setkey = spacc_aead_setkey,
1618                                 .setauthsize = spacc_aead_setauthsize,
1619                                 .encrypt = spacc_aead_encrypt,
1620                                 .decrypt = spacc_aead_decrypt,
1621                                 .givencrypt = spacc_aead_givencrypt,
1622                                 .ivsize = DES3_EDE_BLOCK_SIZE,
1623                                 .maxauthsize = MD5_DIGEST_SIZE,
1624                         },
1625                         .cra_init = spacc_aead_cra_init,
1626                         .cra_exit = spacc_aead_cra_exit,
1627                 },
1628         },
1629 };
1630
1631 static struct spacc_alg l2_engine_algs[] = {
1632         {
1633                 .key_offs = 0,
1634                 .iv_offs = SPACC_CRYPTO_KASUMI_F8_KEY_LEN,
1635                 .ctrl_default = SPA_CTRL_CIPH_ALG_KASUMI |
1636                                 SPA_CTRL_CIPH_MODE_F8,
1637                 .alg = {
1638                         .cra_name = "f8(kasumi)",
1639                         .cra_driver_name = "f8-kasumi-picoxcell",
1640                         .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1641                         .cra_flags = CRYPTO_ALG_TYPE_GIVCIPHER | CRYPTO_ALG_ASYNC,
1642                         .cra_blocksize = 8,
1643                         .cra_ctxsize = sizeof(struct spacc_ablk_ctx),
1644                         .cra_type = &crypto_ablkcipher_type,
1645                         .cra_module = THIS_MODULE,
1646                         .cra_ablkcipher = {
1647                                 .setkey = spacc_kasumi_f8_setkey,
1648                                 .encrypt = spacc_ablk_encrypt,
1649                                 .decrypt = spacc_ablk_decrypt,
1650                                 .min_keysize = 16,
1651                                 .max_keysize = 16,
1652                                 .ivsize = 8,
1653                         },
1654                         .cra_init = spacc_ablk_cra_init,
1655                         .cra_exit = spacc_ablk_cra_exit,
1656                 },
1657         },
1658 };
1659
1660 static int __devinit spacc_probe(struct platform_device *pdev,
1661                                  unsigned max_ctxs, size_t cipher_pg_sz,
1662                                  size_t hash_pg_sz, size_t fifo_sz,
1663                                  struct spacc_alg *algs, size_t num_algs)
1664 {
1665         int i, err, ret = -EINVAL;
1666         struct resource *mem, *irq;
1667         struct spacc_engine *engine = devm_kzalloc(&pdev->dev, sizeof(*engine),
1668                                                    GFP_KERNEL);
1669         if (!engine)
1670                 return -ENOMEM;
1671
1672         engine->max_ctxs        = max_ctxs;
1673         engine->cipher_pg_sz    = cipher_pg_sz;
1674         engine->hash_pg_sz      = hash_pg_sz;
1675         engine->fifo_sz         = fifo_sz;
1676         engine->algs            = algs;
1677         engine->num_algs        = num_algs;
1678         engine->name            = dev_name(&pdev->dev);
1679
1680         mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1681         irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
1682         if (!mem || !irq) {
1683                 dev_err(&pdev->dev, "no memory/irq resource for engine\n");
1684                 return -ENXIO;
1685         }
1686
1687         if (!devm_request_mem_region(&pdev->dev, mem->start, resource_size(mem),
1688                                      engine->name))
1689                 return -ENOMEM;
1690
1691         engine->regs = devm_ioremap(&pdev->dev, mem->start, resource_size(mem));
1692         if (!engine->regs) {
1693                 dev_err(&pdev->dev, "memory map failed\n");
1694                 return -ENOMEM;
1695         }
1696
1697         if (devm_request_irq(&pdev->dev, irq->start, spacc_spacc_irq, 0,
1698                              engine->name, engine)) {
1699                 dev_err(engine->dev, "failed to request IRQ\n");
1700                 return -EBUSY;
1701         }
1702
1703         engine->dev             = &pdev->dev;
1704         engine->cipher_ctx_base = engine->regs + SPA_CIPH_KEY_BASE_REG_OFFSET;
1705         engine->hash_key_base   = engine->regs + SPA_HASH_KEY_BASE_REG_OFFSET;
1706
1707         engine->req_pool = dmam_pool_create(engine->name, engine->dev,
1708                 MAX_DDT_LEN * sizeof(struct spacc_ddt), 8, SZ_64K);
1709         if (!engine->req_pool)
1710                 return -ENOMEM;
1711
1712         spin_lock_init(&engine->hw_lock);
1713
1714         engine->clk = clk_get(&pdev->dev, NULL);
1715         if (IS_ERR(engine->clk)) {
1716                 dev_info(&pdev->dev, "clk unavailable\n");
1717                 device_remove_file(&pdev->dev, &dev_attr_stat_irq_thresh);
1718                 return PTR_ERR(engine->clk);
1719         }
1720
1721         if (clk_enable(engine->clk)) {
1722                 dev_info(&pdev->dev, "unable to enable clk\n");
1723                 clk_put(engine->clk);
1724                 return -EIO;
1725         }
1726
1727         err = device_create_file(&pdev->dev, &dev_attr_stat_irq_thresh);
1728         if (err) {
1729                 clk_disable(engine->clk);
1730                 clk_put(engine->clk);
1731                 return err;
1732         }
1733
1734
1735         /*
1736          * Use an IRQ threshold of 50% as a default. This seems to be a
1737          * reasonable trade off of latency against throughput but can be
1738          * changed at runtime.
1739          */
1740         engine->stat_irq_thresh = (engine->fifo_sz / 2);
1741
1742         /*
1743          * Configure the interrupts. We only use the STAT_CNT interrupt as we
1744          * only submit a new packet for processing when we complete another in
1745          * the queue. This minimizes time spent in the interrupt handler.
1746          */
1747         writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET,
1748                engine->regs + SPA_IRQ_CTRL_REG_OFFSET);
1749         writel(SPA_IRQ_EN_STAT_EN | SPA_IRQ_EN_GLBL_EN,
1750                engine->regs + SPA_IRQ_EN_REG_OFFSET);
1751
1752         setup_timer(&engine->packet_timeout, spacc_packet_timeout,
1753                     (unsigned long)engine);
1754
1755         INIT_LIST_HEAD(&engine->pending);
1756         INIT_LIST_HEAD(&engine->completed);
1757         INIT_LIST_HEAD(&engine->in_progress);
1758         engine->in_flight = 0;
1759         tasklet_init(&engine->complete, spacc_spacc_complete,
1760                      (unsigned long)engine);
1761
1762         platform_set_drvdata(pdev, engine);
1763
1764         INIT_LIST_HEAD(&engine->registered_algs);
1765         for (i = 0; i < engine->num_algs; ++i) {
1766                 engine->algs[i].engine = engine;
1767                 err = crypto_register_alg(&engine->algs[i].alg);
1768                 if (!err) {
1769                         list_add_tail(&engine->algs[i].entry,
1770                                       &engine->registered_algs);
1771                         ret = 0;
1772                 }
1773                 if (err)
1774                         dev_err(engine->dev, "failed to register alg \"%s\"\n",
1775                                 engine->algs[i].alg.cra_name);
1776                 else
1777                         dev_dbg(engine->dev, "registered alg \"%s\"\n",
1778                                 engine->algs[i].alg.cra_name);
1779         }
1780
1781         return ret;
1782 }
1783
1784 static int __devexit spacc_remove(struct platform_device *pdev)
1785 {
1786         struct spacc_alg *alg, *next;
1787         struct spacc_engine *engine = platform_get_drvdata(pdev);
1788
1789         del_timer_sync(&engine->packet_timeout);
1790         device_remove_file(&pdev->dev, &dev_attr_stat_irq_thresh);
1791
1792         list_for_each_entry_safe(alg, next, &engine->registered_algs, entry) {
1793                 list_del(&alg->entry);
1794                 crypto_unregister_alg(&alg->alg);
1795         }
1796
1797         clk_disable(engine->clk);
1798         clk_put(engine->clk);
1799
1800         return 0;
1801 }
1802
1803 static int __devinit ipsec_probe(struct platform_device *pdev)
1804 {
1805         return spacc_probe(pdev, SPACC_CRYPTO_IPSEC_MAX_CTXS,
1806                            SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ,
1807                            SPACC_CRYPTO_IPSEC_HASH_PG_SZ,
1808                            SPACC_CRYPTO_IPSEC_FIFO_SZ, ipsec_engine_algs,
1809                            ARRAY_SIZE(ipsec_engine_algs));
1810 }
1811
1812 static struct platform_driver ipsec_driver = {
1813         .probe          = ipsec_probe,
1814         .remove         = __devexit_p(spacc_remove),
1815         .driver         = {
1816                 .name   = "picoxcell-ipsec",
1817 #ifdef CONFIG_PM
1818                 .pm     = &spacc_pm_ops,
1819 #endif /* CONFIG_PM */
1820         },
1821 };
1822
1823 static int __devinit l2_probe(struct platform_device *pdev)
1824 {
1825         return spacc_probe(pdev, SPACC_CRYPTO_L2_MAX_CTXS,
1826                            SPACC_CRYPTO_L2_CIPHER_PG_SZ,
1827                            SPACC_CRYPTO_L2_HASH_PG_SZ, SPACC_CRYPTO_L2_FIFO_SZ,
1828                            l2_engine_algs, ARRAY_SIZE(l2_engine_algs));
1829 }
1830
1831 static struct platform_driver l2_driver = {
1832         .probe          = l2_probe,
1833         .remove         = __devexit_p(spacc_remove),
1834         .driver         = {
1835                 .name   = "picoxcell-l2",
1836 #ifdef CONFIG_PM
1837                 .pm     = &spacc_pm_ops,
1838 #endif /* CONFIG_PM */
1839         },
1840 };
1841
1842 static int __init spacc_init(void)
1843 {
1844         int ret = platform_driver_register(&ipsec_driver);
1845         if (ret) {
1846                 pr_err("failed to register ipsec spacc driver");
1847                 goto out;
1848         }
1849
1850         ret = platform_driver_register(&l2_driver);
1851         if (ret) {
1852                 pr_err("failed to register l2 spacc driver");
1853                 goto l2_failed;
1854         }
1855
1856         return 0;
1857
1858 l2_failed:
1859         platform_driver_unregister(&ipsec_driver);
1860 out:
1861         return ret;
1862 }
1863 module_init(spacc_init);
1864
1865 static void __exit spacc_exit(void)
1866 {
1867         platform_driver_unregister(&ipsec_driver);
1868         platform_driver_unregister(&l2_driver);
1869 }
1870 module_exit(spacc_exit);
1871
1872 MODULE_LICENSE("GPL");
1873 MODULE_AUTHOR("Jamie Iles");