drivers/crypto/n2_core.c

   1 /* n2_core.c: Niagara2 Stream Processing Unit (SPU) crypto support.
   2  *
   3  * Copyright (C) 2010 David S. Miller <davem@davemloft.net>
   4  */
   5
   6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   7
   8 #include <linux/kernel.h>
   9 #include <linux/module.h>
  10 #include <linux/of.h>
  11 #include <linux/of_device.h>
  12 #include <linux/cpumask.h>
  13 #include <linux/slab.h>
  14 #include <linux/interrupt.h>
  15 #include <linux/crypto.h>
  16 #include <crypto/md5.h>
  17 #include <crypto/sha.h>
  18 #include <crypto/aes.h>
  19 #include <crypto/des.h>
  20 #include <linux/mutex.h>
  21 #include <linux/delay.h>
  22 #include <linux/sched.h>
  23
  24 #include <crypto/internal/hash.h>
  25 #include <crypto/scatterwalk.h>
  26 #include <crypto/algapi.h>
  27
  28 #include <asm/hypervisor.h>
  29 #include <asm/mdesc.h>
  30
  31 #include "n2_core.h"
  32
  33 #define DRV_MODULE_NAME         "n2_crypto"
  34 #define DRV_MODULE_VERSION      "0.1"
  35 #define DRV_MODULE_RELDATE      "April 29, 2010"
  36
  37 static char version[] __devinitdata =
  38         DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
  39
  40 MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");
  41 MODULE_DESCRIPTION("Niagara2 Crypto driver");
  42 MODULE_LICENSE("GPL");
  43 MODULE_VERSION(DRV_MODULE_VERSION);
  44
  45 #define N2_CRA_PRIORITY         300
  46
  47 static DEFINE_MUTEX(spu_lock);
  48
  49 struct spu_queue {
  50         cpumask_t               sharing;
  51         unsigned long           qhandle;
  52
  53         spinlock_t              lock;
  54         u8                      q_type;
  55         void                    *q;
  56         unsigned long           head;
  57         unsigned long           tail;
  58         struct list_head        jobs;
  59
  60         unsigned long           devino;
  61
  62         char                    irq_name[32];
  63         unsigned int            irq;
  64
  65         struct list_head        list;
  66 };
  67
  68 static struct spu_queue **cpu_to_cwq;
  69 static struct spu_queue **cpu_to_mau;
  70
  71 static unsigned long spu_next_offset(struct spu_queue *q, unsigned long off)
  72 {
  73         if (q->q_type == HV_NCS_QTYPE_MAU) {
  74                 off += MAU_ENTRY_SIZE;
  75                 if (off == (MAU_ENTRY_SIZE * MAU_NUM_ENTRIES))
  76                         off = 0;
  77         } else {
  78                 off += CWQ_ENTRY_SIZE;
  79                 if (off == (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES))
  80                         off = 0;
  81         }
  82         return off;
  83 }
  84
  85 struct n2_request_common {
  86         struct list_head        entry;
  87         unsigned int            offset;
  88 };
  89 #define OFFSET_NOT_RUNNING      (~(unsigned int)0)
  90
  91 /* An async job request records the final tail value it used in
  92  * n2_request_common->offset, test to see if that offset is in
  93  * the range old_head, new_head, inclusive.
  94  */
  95 static inline bool job_finished(struct spu_queue *q, unsigned int offset,
  96                                 unsigned long old_head, unsigned long new_head)
  97 {
  98         if (old_head <= new_head) {
  99                 if (offset > old_head && offset <= new_head)
 100                         return true;
 101         } else {
 102                 if (offset > old_head || offset <= new_head)
 103                         return true;
 104         }
 105         return false;
 106 }
 107
 108 /* When the HEAD marker is unequal to the actual HEAD, we get
 109  * a virtual device INO interrupt.  We should process the
 110  * completed CWQ entries and adjust the HEAD marker to clear
 111  * the IRQ.
 112  */
 113 static irqreturn_t cwq_intr(int irq, void *dev_id)
 114 {
 115         unsigned long off, new_head, hv_ret;
 116         struct spu_queue *q = dev_id;
 117
 118         pr_err("CPU[%d]: Got CWQ interrupt for qhdl[%lx]\n",
 119                smp_processor_id(), q->qhandle);
 120
 121         spin_lock(&q->lock);
 122
 123         hv_ret = sun4v_ncs_gethead(q->qhandle, &new_head);
 124
 125         pr_err("CPU[%d]: CWQ gethead[%lx] hv_ret[%lu]\n",
 126                smp_processor_id(), new_head, hv_ret);
 127
 128         for (off = q->head; off != new_head; off = spu_next_offset(q, off)) {
 129                 /* XXX ... XXX */
 130         }
 131
 132         hv_ret = sun4v_ncs_sethead_marker(q->qhandle, new_head);
 133         if (hv_ret == HV_EOK)
 134                 q->head = new_head;
 135
 136         spin_unlock(&q->lock);
 137
 138         return IRQ_HANDLED;
 139 }
 140
 141 static irqreturn_t mau_intr(int irq, void *dev_id)
 142 {
 143         struct spu_queue *q = dev_id;
 144         unsigned long head, hv_ret;
 145
 146         spin_lock(&q->lock);
 147
 148         pr_err("CPU[%d]: Got MAU interrupt for qhdl[%lx]\n",
 149                smp_processor_id(), q->qhandle);
 150
 151         hv_ret = sun4v_ncs_gethead(q->qhandle, &head);
 152
 153         pr_err("CPU[%d]: MAU gethead[%lx] hv_ret[%lu]\n",
 154                smp_processor_id(), head, hv_ret);
 155
 156         sun4v_ncs_sethead_marker(q->qhandle, head);
 157
 158         spin_unlock(&q->lock);
 159
 160         return IRQ_HANDLED;
 161 }
 162
 163 static void *spu_queue_next(struct spu_queue *q, void *cur)
 164 {
 165         return q->q + spu_next_offset(q, cur - q->q);
 166 }
 167
 168 static int spu_queue_num_free(struct spu_queue *q)
 169 {
 170         unsigned long head = q->head;
 171         unsigned long tail = q->tail;
 172         unsigned long end = (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES);
 173         unsigned long diff;
 174
 175         if (head > tail)
 176                 diff = head - tail;
 177         else
 178                 diff = (end - tail) + head;
 179
 180         return (diff / CWQ_ENTRY_SIZE) - 1;
 181 }
 182
 183 static void *spu_queue_alloc(struct spu_queue *q, int num_entries)
 184 {
 185         int avail = spu_queue_num_free(q);
 186
 187         if (avail >= num_entries)
 188                 return q->q + q->tail;
 189
 190         return NULL;
 191 }
 192
 193 static unsigned long spu_queue_submit(struct spu_queue *q, void *last)
 194 {
 195         unsigned long hv_ret, new_tail;
 196
 197         new_tail = spu_next_offset(q, last - q->q);
 198
 199         hv_ret = sun4v_ncs_settail(q->qhandle, new_tail);
 200         if (hv_ret == HV_EOK)
 201                 q->tail = new_tail;
 202         return hv_ret;
 203 }
 204
 205 static u64 control_word_base(unsigned int len, unsigned int hmac_key_len,
 206                              int enc_type, int auth_type,
 207                              unsigned int hash_len,
 208                              bool sfas, bool sob, bool eob, bool encrypt,
 209                              int opcode)
 210 {
 211         u64 word = (len - 1) & CONTROL_LEN;
 212
 213         word |= ((u64) opcode << CONTROL_OPCODE_SHIFT);
 214         word |= ((u64) enc_type << CONTROL_ENC_TYPE_SHIFT);
 215         word |= ((u64) auth_type << CONTROL_AUTH_TYPE_SHIFT);
 216         if (sfas)
 217                 word |= CONTROL_STORE_FINAL_AUTH_STATE;
 218         if (sob)
 219                 word |= CONTROL_START_OF_BLOCK;
 220         if (eob)
 221                 word |= CONTROL_END_OF_BLOCK;
 222         if (encrypt)
 223                 word |= CONTROL_ENCRYPT;
 224         if (hmac_key_len)
 225                 word |= ((u64) (hmac_key_len - 1)) << CONTROL_HMAC_KEY_LEN_SHIFT;
 226         if (hash_len)
 227                 word |= ((u64) (hash_len - 1)) << CONTROL_HASH_LEN_SHIFT;
 228
 229         return word;
 230 }
 231
 232 #if 0
 233 static inline bool n2_should_run_async(struct spu_queue *qp, int this_len)
 234 {
 235         if (this_len >= 64 ||
 236             qp->head != qp->tail)
 237                 return true;
 238         return false;
 239 }
 240 #endif
 241
 242 struct n2_base_ctx {
 243         struct list_head                list;
 244 };
 245
 246 static void n2_base_ctx_init(struct n2_base_ctx *ctx)
 247 {
 248         INIT_LIST_HEAD(&ctx->list);
 249 }
 250
 251 struct n2_hash_ctx {
 252         struct n2_base_ctx              base;
 253
 254         struct crypto_ahash             *fallback_tfm;
 255 };
 256
 257 struct n2_hash_req_ctx {
 258         union {
 259                 struct md5_state        md5;
 260                 struct sha1_state       sha1;
 261                 struct sha256_state     sha256;
 262         } u;
 263
 264         unsigned char                   hash_key[64];
 265         unsigned char                   keyed_zero_hash[32];
 266
 267         struct ahash_request            fallback_req;
 268 };
 269
 270 static int n2_hash_async_init(struct ahash_request *req)
 271 {
 272         struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 273         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 274         struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 275
 276         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 277         rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 278
 279         return crypto_ahash_init(&rctx->fallback_req);
 280 }
 281
 282 static int n2_hash_async_update(struct ahash_request *req)
 283 {
 284         struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 285         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 286         struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 287
 288         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 289         rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 290         rctx->fallback_req.nbytes = req->nbytes;
 291         rctx->fallback_req.src = req->src;
 292
 293         return crypto_ahash_update(&rctx->fallback_req);
 294 }
 295
 296 static int n2_hash_async_final(struct ahash_request *req)
 297 {
 298         struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 299         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 300         struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 301
 302         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 303         rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 304         rctx->fallback_req.result = req->result;
 305
 306         return crypto_ahash_final(&rctx->fallback_req);
 307 }
 308
 309 static int n2_hash_async_finup(struct ahash_request *req)
 310 {
 311         struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 312         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 313         struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 314
 315         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 316         rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 317         rctx->fallback_req.nbytes = req->nbytes;
 318         rctx->fallback_req.src = req->src;
 319         rctx->fallback_req.result = req->result;
 320
 321         return crypto_ahash_finup(&rctx->fallback_req);
 322 }
 323
 324 static int n2_hash_cra_init(struct crypto_tfm *tfm)
 325 {
 326         const char *fallback_driver_name = tfm->__crt_alg->cra_name;
 327         struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
 328         struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 329         struct crypto_ahash *fallback_tfm;
 330         int err;
 331
 332         fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,
 333                                           CRYPTO_ALG_NEED_FALLBACK);
 334         if (IS_ERR(fallback_tfm)) {
 335                 pr_warning("Fallback driver '%s' could not be loaded!\n",
 336                            fallback_driver_name);
 337                 err = PTR_ERR(fallback_tfm);
 338                 goto out;
 339         }
 340
 341         crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) +
 342                                          crypto_ahash_reqsize(fallback_tfm)));
 343
 344         ctx->fallback_tfm = fallback_tfm;
 345         return 0;
 346
 347 out:
 348         return err;
 349 }
 350
 351 static void n2_hash_cra_exit(struct crypto_tfm *tfm)
 352 {
 353         struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
 354         struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 355
 356         crypto_free_ahash(ctx->fallback_tfm);
 357 }
 358
 359 static unsigned long wait_for_tail(struct spu_queue *qp)
 360 {
 361         unsigned long head, hv_ret;
 362
 363         do {
 364                 hv_ret = sun4v_ncs_gethead(qp->qhandle, &head);
 365                 if (hv_ret != HV_EOK) {
 366                         pr_err("Hypervisor error on gethead\n");
 367                         break;
 368                 }
 369                 if (head == qp->tail) {
 370                         qp->head = head;
 371                         break;
 372                 }
 373         } while (1);
 374         return hv_ret;
 375 }
 376
 377 static unsigned long submit_and_wait_for_tail(struct spu_queue *qp,
 378                                               struct cwq_initial_entry *ent)
 379 {
 380         unsigned long hv_ret = spu_queue_submit(qp, ent);
 381
 382         if (hv_ret == HV_EOK)
 383                 hv_ret = wait_for_tail(qp);
 384
 385         return hv_ret;
 386 }
 387
 388 static int n2_hash_async_digest(struct ahash_request *req,
 389                                 unsigned int auth_type, unsigned int digest_size,
 390                                 unsigned int result_size, void *hash_loc)
 391 {
 392         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 393         struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 394         struct cwq_initial_entry *ent;
 395         struct crypto_hash_walk walk;
 396         struct spu_queue *qp;
 397         unsigned long flags;
 398         int err = -ENODEV;
 399         int nbytes, cpu;
 400
 401         /* The total effective length of the operation may not
 402          * exceed 2^16.
 403          */
 404         if (unlikely(req->nbytes > (1 << 16))) {
 405                 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 406
 407                 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 408                 rctx->fallback_req.base.flags =
 409                         req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 410                 rctx->fallback_req.nbytes = req->nbytes;
 411                 rctx->fallback_req.src = req->src;
 412                 rctx->fallback_req.result = req->result;
 413
 414                 return crypto_ahash_digest(&rctx->fallback_req);
 415         }
 416
 417         n2_base_ctx_init(&ctx->base);
 418
 419         nbytes = crypto_hash_walk_first(req, &walk);
 420
 421         cpu = get_cpu();
 422         qp = cpu_to_cwq[cpu];
 423         if (!qp)
 424                 goto out;
 425
 426         spin_lock_irqsave(&qp->lock, flags);
 427
 428         /* XXX can do better, improve this later by doing a by-hand scatterlist
 429          * XXX walk, etc.
 430          */
 431         ent = qp->q + qp->tail;
 432
 433         ent->control = control_word_base(nbytes, 0, 0,
 434                                          auth_type, digest_size,
 435                                          false, true, false, false,
 436                                          OPCODE_INPLACE_BIT |
 437                                          OPCODE_AUTH_MAC);
 438         ent->src_addr = __pa(walk.data);
 439         ent->auth_key_addr = 0UL;
 440         ent->auth_iv_addr = __pa(hash_loc);
 441         ent->final_auth_state_addr = 0UL;
 442         ent->enc_key_addr = 0UL;
 443         ent->enc_iv_addr = 0UL;
 444         ent->dest_addr = __pa(hash_loc);
 445
 446         nbytes = crypto_hash_walk_done(&walk, 0);
 447         while (nbytes > 0) {
 448                 ent = spu_queue_next(qp, ent);
 449
 450                 ent->control = (nbytes - 1);
 451                 ent->src_addr = __pa(walk.data);
 452                 ent->auth_key_addr = 0UL;
 453                 ent->auth_iv_addr = 0UL;
 454                 ent->final_auth_state_addr = 0UL;
 455                 ent->enc_key_addr = 0UL;
 456                 ent->enc_iv_addr = 0UL;
 457                 ent->dest_addr = 0UL;
 458
 459                 nbytes = crypto_hash_walk_done(&walk, 0);
 460         }
 461         ent->control |= CONTROL_END_OF_BLOCK;
 462
 463         if (submit_and_wait_for_tail(qp, ent) != HV_EOK)
 464                 err = -EINVAL;
 465         else
 466                 err = 0;
 467
 468         spin_unlock_irqrestore(&qp->lock, flags);
 469
 470         if (!err)
 471                 memcpy(req->result, hash_loc, result_size);
 472 out:
 473         put_cpu();
 474
 475         return err;
 476 }
 477
 478 static int n2_md5_async_digest(struct ahash_request *req)
 479 {
 480         struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 481         struct md5_state *m = &rctx->u.md5;
 482
 483         if (unlikely(req->nbytes == 0)) {
 484                 static const char md5_zero[MD5_DIGEST_SIZE] = {
 485                         0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04,
 486                         0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e,
 487                 };
 488
 489                 memcpy(req->result, md5_zero, MD5_DIGEST_SIZE);
 490                 return 0;
 491         }
 492         m->hash[0] = cpu_to_le32(0x67452301);
 493         m->hash[1] = cpu_to_le32(0xefcdab89);
 494         m->hash[2] = cpu_to_le32(0x98badcfe);
 495         m->hash[3] = cpu_to_le32(0x10325476);
 496
 497         return n2_hash_async_digest(req, AUTH_TYPE_MD5,
 498                                     MD5_DIGEST_SIZE, MD5_DIGEST_SIZE,
 499                                     m->hash);
 500 }
 501
 502 static int n2_sha1_async_digest(struct ahash_request *req)
 503 {
 504         struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 505         struct sha1_state *s = &rctx->u.sha1;
 506
 507         if (unlikely(req->nbytes == 0)) {
 508                 static const char sha1_zero[SHA1_DIGEST_SIZE] = {
 509                         0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32,
 510                         0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8,
 511                         0x07, 0x09
 512                 };
 513
 514                 memcpy(req->result, sha1_zero, SHA1_DIGEST_SIZE);
 515                 return 0;
 516         }
 517         s->state[0] = SHA1_H0;
 518         s->state[1] = SHA1_H1;
 519         s->state[2] = SHA1_H2;
 520         s->state[3] = SHA1_H3;
 521         s->state[4] = SHA1_H4;
 522
 523         return n2_hash_async_digest(req, AUTH_TYPE_SHA1,
 524                                     SHA1_DIGEST_SIZE, SHA1_DIGEST_SIZE,
 525                                     s->state);
 526 }
 527
 528 static int n2_sha256_async_digest(struct ahash_request *req)
 529 {
 530         struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 531         struct sha256_state *s = &rctx->u.sha256;
 532
 533         if (req->nbytes == 0) {
 534                 static const char sha256_zero[SHA256_DIGEST_SIZE] = {
 535                         0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a,
 536                         0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae,
 537                         0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99,
 538                         0x1b, 0x78, 0x52, 0xb8, 0x55
 539                 };
 540
 541                 memcpy(req->result, sha256_zero, SHA256_DIGEST_SIZE);
 542                 return 0;
 543         }
 544         s->state[0] = SHA256_H0;
 545         s->state[1] = SHA256_H1;
 546         s->state[2] = SHA256_H2;
 547         s->state[3] = SHA256_H3;
 548         s->state[4] = SHA256_H4;
 549         s->state[5] = SHA256_H5;
 550         s->state[6] = SHA256_H6;
 551         s->state[7] = SHA256_H7;
 552
 553         return n2_hash_async_digest(req, AUTH_TYPE_SHA256,
 554                                     SHA256_DIGEST_SIZE, SHA256_DIGEST_SIZE,
 555                                     s->state);
 556 }
 557
 558 static int n2_sha224_async_digest(struct ahash_request *req)
 559 {
 560         struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 561         struct sha256_state *s = &rctx->u.sha256;
 562
 563         if (req->nbytes == 0) {
 564                 static const char sha224_zero[SHA224_DIGEST_SIZE] = {
 565                         0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47,
 566                         0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2,
 567                         0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 0xc5, 0xb3, 0xe4,
 568                         0x2f
 569                 };
 570
 571                 memcpy(req->result, sha224_zero, SHA224_DIGEST_SIZE);
 572                 return 0;
 573         }
 574         s->state[0] = SHA224_H0;
 575         s->state[1] = SHA224_H1;
 576         s->state[2] = SHA224_H2;
 577         s->state[3] = SHA224_H3;
 578         s->state[4] = SHA224_H4;
 579         s->state[5] = SHA224_H5;
 580         s->state[6] = SHA224_H6;
 581         s->state[7] = SHA224_H7;
 582
 583         return n2_hash_async_digest(req, AUTH_TYPE_SHA256,
 584                                     SHA256_DIGEST_SIZE, SHA224_DIGEST_SIZE,
 585                                     s->state);
 586 }
 587
 588 struct n2_cipher_context {
 589         int                     key_len;
 590         int                     enc_type;
 591         union {
 592                 u8              aes[AES_MAX_KEY_SIZE];
 593                 u8              des[DES_KEY_SIZE];
 594                 u8              des3[3 * DES_KEY_SIZE];
 595                 u8              arc4[258]; /* S-box, X, Y */
 596         } key;
 597 };
 598
 599 #define N2_CHUNK_ARR_LEN        16
 600
 601 struct n2_crypto_chunk {
 602         struct list_head        entry;
 603         unsigned long           iv_paddr : 44;
 604         unsigned long           arr_len : 20;
 605         unsigned long           dest_paddr;
 606         unsigned long           dest_final;
 607         struct {
 608                 unsigned long   src_paddr : 44;
 609                 unsigned long   src_len : 20;
 610         } arr[N2_CHUNK_ARR_LEN];
 611 };
 612
 613 struct n2_request_context {
 614         struct ablkcipher_walk  walk;
 615         struct list_head        chunk_list;
 616         struct n2_crypto_chunk  chunk;
 617         u8                      temp_iv[16];
 618 };
 619
 620 /* The SPU allows some level of flexibility for partial cipher blocks
 621  * being specified in a descriptor.
 622  *
 623  * It merely requires that every descriptor's length field is at least
 624  * as large as the cipher block size.  This means that a cipher block
 625  * can span at most 2 descriptors.  However, this does not allow a
 626  * partial block to span into the final descriptor as that would
 627  * violate the rule (since every descriptor's length must be at lest
 628  * the block size).  So, for example, assuming an 8 byte block size:
 629  *
 630  *      0xe --> 0xa --> 0x8
 631  *
 632  * is a valid length sequence, whereas:
 633  *
 634  *      0xe --> 0xb --> 0x7
 635  *
 636  * is not a valid sequence.
 637  */
 638
 639 struct n2_cipher_alg {
 640         struct list_head        entry;
 641         u8                      enc_type;
 642         struct crypto_alg       alg;
 643 };
 644
 645 static inline struct n2_cipher_alg *n2_cipher_alg(struct crypto_tfm *tfm)
 646 {
 647         struct crypto_alg *alg = tfm->__crt_alg;
 648
 649         return container_of(alg, struct n2_cipher_alg, alg);
 650 }
 651
 652 struct n2_cipher_request_context {
 653         struct ablkcipher_walk  walk;
 654 };
 655
 656 static int n2_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
 657                          unsigned int keylen)
 658 {
 659         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
 660         struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
 661         struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
 662
 663         ctx->enc_type = (n2alg->enc_type & ENC_TYPE_CHAINING_MASK);
 664
 665         switch (keylen) {
 666         case AES_KEYSIZE_128:
 667                 ctx->enc_type |= ENC_TYPE_ALG_AES128;
 668                 break;
 669         case AES_KEYSIZE_192:
 670                 ctx->enc_type |= ENC_TYPE_ALG_AES192;
 671                 break;
 672         case AES_KEYSIZE_256:
 673                 ctx->enc_type |= ENC_TYPE_ALG_AES256;
 674                 break;
 675         default:
 676                 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
 677                 return -EINVAL;
 678         }
 679
 680         ctx->key_len = keylen;
 681         memcpy(ctx->key.aes, key, keylen);
 682         return 0;
 683 }
 684
 685 static int n2_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
 686                          unsigned int keylen)
 687 {
 688         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
 689         struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
 690         struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
 691         u32 tmp[DES_EXPKEY_WORDS];
 692         int err;
 693
 694         ctx->enc_type = n2alg->enc_type;
 695
 696         if (keylen != DES_KEY_SIZE) {
 697                 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
 698                 return -EINVAL;
 699         }
 700
 701         err = des_ekey(tmp, key);
 702         if (err == 0 && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
 703                 tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
 704                 return -EINVAL;
 705         }
 706
 707         ctx->key_len = keylen;
 708         memcpy(ctx->key.des, key, keylen);
 709         return 0;
 710 }
 711
 712 static int n2_3des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
 713                           unsigned int keylen)
 714 {
 715         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
 716         struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
 717         struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
 718
 719         ctx->enc_type = n2alg->enc_type;
 720
 721         if (keylen != (3 * DES_KEY_SIZE)) {
 722                 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
 723                 return -EINVAL;
 724         }
 725         ctx->key_len = keylen;
 726         memcpy(ctx->key.des3, key, keylen);
 727         return 0;
 728 }
 729
 730 static int n2_arc4_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
 731                           unsigned int keylen)
 732 {
 733         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
 734         struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
 735         struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
 736         u8 *s = ctx->key.arc4;
 737         u8 *x = s + 256;
 738         u8 *y = x + 1;
 739         int i, j, k;
 740
 741         ctx->enc_type = n2alg->enc_type;
 742
 743         j = k = 0;
 744         *x = 0;
 745         *y = 0;
 746         for (i = 0; i < 256; i++)
 747                 s[i] = i;
 748         for (i = 0; i < 256; i++) {
 749                 u8 a = s[i];
 750                 j = (j + key[k] + a) & 0xff;
 751                 s[i] = s[j];
 752                 s[j] = a;
 753                 if (++k >= keylen)
 754                         k = 0;
 755         }
 756
 757         return 0;
 758 }
 759
 760 static inline int cipher_descriptor_len(int nbytes, unsigned int block_size)
 761 {
 762         int this_len = nbytes;
 763
 764         this_len -= (nbytes & (block_size - 1));
 765         return this_len > (1 << 16) ? (1 << 16) : this_len;
 766 }
 767
 768 static int __n2_crypt_chunk(struct crypto_tfm *tfm, struct n2_crypto_chunk *cp,
 769                             struct spu_queue *qp, bool encrypt)
 770 {
 771         struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
 772         struct cwq_initial_entry *ent;
 773         bool in_place;
 774         int i;
 775
 776         ent = spu_queue_alloc(qp, cp->arr_len);
 777         if (!ent) {
 778                 pr_info("queue_alloc() of %d fails\n",
 779                         cp->arr_len);
 780                 return -EBUSY;
 781         }
 782
 783         in_place = (cp->dest_paddr == cp->arr[0].src_paddr);
 784
 785         ent->control = control_word_base(cp->arr[0].src_len,
 786                                          0, ctx->enc_type, 0, 0,
 787                                          false, true, false, encrypt,
 788                                          OPCODE_ENCRYPT |
 789                                          (in_place ? OPCODE_INPLACE_BIT : 0));
 790         ent->src_addr = cp->arr[0].src_paddr;
 791         ent->auth_key_addr = 0UL;
 792         ent->auth_iv_addr = 0UL;
 793         ent->final_auth_state_addr = 0UL;
 794         ent->enc_key_addr = __pa(&ctx->key);
 795         ent->enc_iv_addr = cp->iv_paddr;
 796         ent->dest_addr = (in_place ? 0UL : cp->dest_paddr);
 797
 798         for (i = 1; i < cp->arr_len; i++) {
 799                 ent = spu_queue_next(qp, ent);
 800
 801                 ent->control = cp->arr[i].src_len - 1;
 802                 ent->src_addr = cp->arr[i].src_paddr;
 803                 ent->auth_key_addr = 0UL;
 804                 ent->auth_iv_addr = 0UL;
 805                 ent->final_auth_state_addr = 0UL;
 806                 ent->enc_key_addr = 0UL;
 807                 ent->enc_iv_addr = 0UL;
 808                 ent->dest_addr = 0UL;
 809         }
 810         ent->control |= CONTROL_END_OF_BLOCK;
 811
 812         return (spu_queue_submit(qp, ent) != HV_EOK) ? -EINVAL : 0;
 813 }
 814
 815 static int n2_compute_chunks(struct ablkcipher_request *req)
 816 {
 817         struct n2_request_context *rctx = ablkcipher_request_ctx(req);
 818         struct ablkcipher_walk *walk = &rctx->walk;
 819         struct n2_crypto_chunk *chunk;
 820         unsigned long dest_prev;
 821         unsigned int tot_len;
 822         bool prev_in_place;
 823         int err, nbytes;
 824
 825         ablkcipher_walk_init(walk, req->dst, req->src, req->nbytes);
 826         err = ablkcipher_walk_phys(req, walk);
 827         if (err)
 828                 return err;
 829
 830         INIT_LIST_HEAD(&rctx->chunk_list);
 831
 832         chunk = &rctx->chunk;
 833         INIT_LIST_HEAD(&chunk->entry);
 834
 835         chunk->iv_paddr = 0UL;
 836         chunk->arr_len = 0;
 837         chunk->dest_paddr = 0UL;
 838
 839         prev_in_place = false;
 840         dest_prev = ~0UL;
 841         tot_len = 0;
 842
 843         while ((nbytes = walk->nbytes) != 0) {
 844                 unsigned long dest_paddr, src_paddr;
 845                 bool in_place;
 846                 int this_len;
 847
 848                 src_paddr = (page_to_phys(walk->src.page) +
 849                              walk->src.offset);
 850                 dest_paddr = (page_to_phys(walk->dst.page) +
 851                               walk->dst.offset);
 852                 in_place = (src_paddr == dest_paddr);
 853                 this_len = cipher_descriptor_len(nbytes, walk->blocksize);
 854
 855                 if (chunk->arr_len != 0) {
 856                         if (in_place != prev_in_place ||
 857                             (!prev_in_place &&
 858                              dest_paddr != dest_prev) ||
 859                             chunk->arr_len == N2_CHUNK_ARR_LEN ||
 860                             tot_len + this_len > (1 << 16)) {
 861                                 chunk->dest_final = dest_prev;
 862                                 list_add_tail(&chunk->entry,
 863                                               &rctx->chunk_list);
 864                                 chunk = kzalloc(sizeof(*chunk), GFP_ATOMIC);
 865                                 if (!chunk) {
 866                                         err = -ENOMEM;
 867                                         break;
 868                                 }
 869                                 INIT_LIST_HEAD(&chunk->entry);
 870                         }
 871                 }
 872                 if (chunk->arr_len == 0) {
 873                         chunk->dest_paddr = dest_paddr;
 874                         tot_len = 0;
 875                 }
 876                 chunk->arr[chunk->arr_len].src_paddr = src_paddr;
 877                 chunk->arr[chunk->arr_len].src_len = this_len;
 878                 chunk->arr_len++;
 879
 880                 dest_prev = dest_paddr + this_len;
 881                 prev_in_place = in_place;
 882                 tot_len += this_len;
 883
 884                 err = ablkcipher_walk_done(req, walk, nbytes - this_len);
 885                 if (err)
 886                         break;
 887         }
 888         if (!err && chunk->arr_len != 0) {
 889                 chunk->dest_final = dest_prev;
 890                 list_add_tail(&chunk->entry, &rctx->chunk_list);
 891         }
 892
 893         return err;
 894 }
 895
 896 static void n2_chunk_complete(struct ablkcipher_request *req, void *final_iv)
 897 {
 898         struct n2_request_context *rctx = ablkcipher_request_ctx(req);
 899         struct n2_crypto_chunk *c, *tmp;
 900
 901         if (final_iv)
 902                 memcpy(rctx->walk.iv, final_iv, rctx->walk.blocksize);
 903
 904         ablkcipher_walk_complete(&rctx->walk);
 905         list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {
 906                 list_del(&c->entry);
 907                 if (unlikely(c != &rctx->chunk))
 908                         kfree(c);
 909         }
 910
 911 }
 912
 913 static int n2_do_ecb(struct ablkcipher_request *req, bool encrypt)
 914 {
 915         struct n2_request_context *rctx = ablkcipher_request_ctx(req);
 916         struct crypto_tfm *tfm = req->base.tfm;
 917         int err = n2_compute_chunks(req);
 918         struct n2_crypto_chunk *c, *tmp;
 919         unsigned long flags, hv_ret;
 920         struct spu_queue *qp;
 921
 922         if (err)
 923                 return err;
 924
 925         qp = cpu_to_cwq[get_cpu()];
 926         err = -ENODEV;
 927         if (!qp)
 928                 goto out;
 929
 930         spin_lock_irqsave(&qp->lock, flags);
 931
 932         list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {
 933                 err = __n2_crypt_chunk(tfm, c, qp, encrypt);
 934                 if (err)
 935                         break;
 936                 list_del(&c->entry);
 937                 if (unlikely(c != &rctx->chunk))
 938                         kfree(c);
 939         }
 940         if (!err) {
 941                 hv_ret = wait_for_tail(qp);
 942                 if (hv_ret != HV_EOK)
 943                         err = -EINVAL;
 944         }
 945
 946         spin_unlock_irqrestore(&qp->lock, flags);
 947
 948         put_cpu();
 949
 950 out:
 951         n2_chunk_complete(req, NULL);
 952         return err;
 953 }
 954
 955 static int n2_encrypt_ecb(struct ablkcipher_request *req)
 956 {
 957         return n2_do_ecb(req, true);
 958 }
 959
 960 static int n2_decrypt_ecb(struct ablkcipher_request *req)
 961 {
 962         return n2_do_ecb(req, false);
 963 }
 964
 965 static int n2_do_chaining(struct ablkcipher_request *req, bool encrypt)
 966 {
 967         struct n2_request_context *rctx = ablkcipher_request_ctx(req);
 968         struct crypto_tfm *tfm = req->base.tfm;
 969         unsigned long flags, hv_ret, iv_paddr;
 970         int err = n2_compute_chunks(req);
 971         struct n2_crypto_chunk *c, *tmp;
 972         struct spu_queue *qp;
 973         void *final_iv_addr;
 974
 975         final_iv_addr = NULL;
 976
 977         if (err)
 978                 return err;
 979
 980         qp = cpu_to_cwq[get_cpu()];
 981         err = -ENODEV;
 982         if (!qp)
 983                 goto out;
 984
 985         spin_lock_irqsave(&qp->lock, flags);
 986
 987         if (encrypt) {
 988                 iv_paddr = __pa(rctx->walk.iv);
 989                 list_for_each_entry_safe(c, tmp, &rctx->chunk_list,
 990                                          entry) {
 991                         c->iv_paddr = iv_paddr;
 992                         err = __n2_crypt_chunk(tfm, c, qp, true);
 993                         if (err)
 994                                 break;
 995                         iv_paddr = c->dest_final - rctx->walk.blocksize;
 996                         list_del(&c->entry);
 997                         if (unlikely(c != &rctx->chunk))
 998                                 kfree(c);
 999                 }
1000                 final_iv_addr = __va(iv_paddr);
1001         } else {
1002                 list_for_each_entry_safe_reverse(c, tmp, &rctx->chunk_list,
1003                                                  entry) {
1004                         if (c == &rctx->chunk) {
1005                                 iv_paddr = __pa(rctx->walk.iv);
1006                         } else {
1007                                 iv_paddr = (tmp->arr[tmp->arr_len-1].src_paddr +
1008                                             tmp->arr[tmp->arr_len-1].src_len -
1009                                             rctx->walk.blocksize);
1010                         }
1011                         if (!final_iv_addr) {
1012                                 unsigned long pa;
1013
1014                                 pa = (c->arr[c->arr_len-1].src_paddr +
1015                                       c->arr[c->arr_len-1].src_len -
1016                                       rctx->walk.blocksize);
1017                                 final_iv_addr = rctx->temp_iv;
1018                                 memcpy(rctx->temp_iv, __va(pa),
1019                                        rctx->walk.blocksize);
1020                         }
1021                         c->iv_paddr = iv_paddr;
1022                         err = __n2_crypt_chunk(tfm, c, qp, false);
1023                         if (err)
1024                                 break;
1025                         list_del(&c->entry);
1026                         if (unlikely(c != &rctx->chunk))
1027                                 kfree(c);
1028                 }
1029         }
1030         if (!err) {
1031                 hv_ret = wait_for_tail(qp);
1032                 if (hv_ret != HV_EOK)
1033                         err = -EINVAL;
1034         }
1035
1036         spin_unlock_irqrestore(&qp->lock, flags);
1037
1038         put_cpu();
1039
1040 out:
1041         n2_chunk_complete(req, err ? NULL : final_iv_addr);
1042         return err;
1043 }
1044
1045 static int n2_encrypt_chaining(struct ablkcipher_request *req)
1046 {
1047         return n2_do_chaining(req, true);
1048 }
1049
1050 static int n2_decrypt_chaining(struct ablkcipher_request *req)
1051 {
1052         return n2_do_chaining(req, false);
1053 }
1054
1055 struct n2_cipher_tmpl {
1056         const char              *name;
1057         const char              *drv_name;
1058         u8                      block_size;
1059         u8                      enc_type;
1060         struct ablkcipher_alg   ablkcipher;
1061 };
1062
1063 static const struct n2_cipher_tmpl cipher_tmpls[] = {
1064         /* ARC4: only ECB is supported (chaining bits ignored) */
1065         {       .name           = "ecb(arc4)",
1066                 .drv_name       = "ecb-arc4",
1067                 .block_size     = 1,
1068                 .enc_type       = (ENC_TYPE_ALG_RC4_STREAM |
1069                                    ENC_TYPE_CHAINING_ECB),
1070                 .ablkcipher     = {
1071                         .min_keysize    = 1,
1072                         .max_keysize    = 256,
1073                         .setkey         = n2_arc4_setkey,
1074                         .encrypt        = n2_encrypt_ecb,
1075                         .decrypt        = n2_decrypt_ecb,
1076                 },
1077         },
1078
1079         /* DES: ECB CBC and CFB are supported */
1080         {       .name           = "ecb(des)",
1081                 .drv_name       = "ecb-des",
1082                 .block_size     = DES_BLOCK_SIZE,
1083                 .enc_type       = (ENC_TYPE_ALG_DES |
1084                                    ENC_TYPE_CHAINING_ECB),
1085                 .ablkcipher     = {
1086                         .min_keysize    = DES_KEY_SIZE,
1087                         .max_keysize    = DES_KEY_SIZE,
1088                         .setkey         = n2_des_setkey,
1089                         .encrypt        = n2_encrypt_ecb,
1090                         .decrypt        = n2_decrypt_ecb,
1091                 },
1092         },
1093         {       .name           = "cbc(des)",
1094                 .drv_name       = "cbc-des",
1095                 .block_size     = DES_BLOCK_SIZE,
1096                 .enc_type       = (ENC_TYPE_ALG_DES |
1097                                    ENC_TYPE_CHAINING_CBC),
1098                 .ablkcipher     = {
1099                         .ivsize         = DES_BLOCK_SIZE,
1100                         .min_keysize    = DES_KEY_SIZE,
1101                         .max_keysize    = DES_KEY_SIZE,
1102                         .setkey         = n2_des_setkey,
1103                         .encrypt        = n2_encrypt_chaining,
1104                         .decrypt        = n2_decrypt_chaining,
1105                 },
1106         },
1107         {       .name           = "cfb(des)",
1108                 .drv_name       = "cfb-des",
1109                 .block_size     = DES_BLOCK_SIZE,
1110                 .enc_type       = (ENC_TYPE_ALG_DES |
1111                                    ENC_TYPE_CHAINING_CFB),
1112                 .ablkcipher     = {
1113                         .min_keysize    = DES_KEY_SIZE,
1114                         .max_keysize    = DES_KEY_SIZE,
1115                         .setkey         = n2_des_setkey,
1116                         .encrypt        = n2_encrypt_chaining,
1117                         .decrypt        = n2_decrypt_chaining,
1118                 },
1119         },
1120
1121         /* 3DES: ECB CBC and CFB are supported */
1122         {       .name           = "ecb(des3_ede)",
1123                 .drv_name       = "ecb-3des",
1124                 .block_size     = DES_BLOCK_SIZE,
1125                 .enc_type       = (ENC_TYPE_ALG_3DES |
1126                                    ENC_TYPE_CHAINING_ECB),
1127                 .ablkcipher     = {
1128                         .min_keysize    = 3 * DES_KEY_SIZE,
1129                         .max_keysize    = 3 * DES_KEY_SIZE,
1130                         .setkey         = n2_3des_setkey,
1131                         .encrypt        = n2_encrypt_ecb,
1132                         .decrypt        = n2_decrypt_ecb,
1133                 },
1134         },
1135         {       .name           = "cbc(des3_ede)",
1136                 .drv_name       = "cbc-3des",
1137                 .block_size     = DES_BLOCK_SIZE,
1138                 .enc_type       = (ENC_TYPE_ALG_3DES |
1139                                    ENC_TYPE_CHAINING_CBC),
1140                 .ablkcipher     = {
1141                         .ivsize         = DES_BLOCK_SIZE,
1142                         .min_keysize    = 3 * DES_KEY_SIZE,
1143                         .max_keysize    = 3 * DES_KEY_SIZE,
1144                         .setkey         = n2_3des_setkey,
1145                         .encrypt        = n2_encrypt_chaining,
1146                         .decrypt        = n2_decrypt_chaining,
1147                 },
1148         },
1149         {       .name           = "cfb(des3_ede)",
1150                 .drv_name       = "cfb-3des",
1151                 .block_size     = DES_BLOCK_SIZE,
1152                 .enc_type       = (ENC_TYPE_ALG_3DES |
1153                                    ENC_TYPE_CHAINING_CFB),
1154                 .ablkcipher     = {
1155                         .min_keysize    = 3 * DES_KEY_SIZE,
1156                         .max_keysize    = 3 * DES_KEY_SIZE,
1157                         .setkey         = n2_3des_setkey,
1158                         .encrypt        = n2_encrypt_chaining,
1159                         .decrypt        = n2_decrypt_chaining,
1160                 },
1161         },
1162         /* AES: ECB CBC and CTR are supported */
1163         {       .name           = "ecb(aes)",
1164                 .drv_name       = "ecb-aes",
1165                 .block_size     = AES_BLOCK_SIZE,
1166                 .enc_type       = (ENC_TYPE_ALG_AES128 |
1167                                    ENC_TYPE_CHAINING_ECB),
1168                 .ablkcipher     = {
1169                         .min_keysize    = AES_MIN_KEY_SIZE,
1170                         .max_keysize    = AES_MAX_KEY_SIZE,
1171                         .setkey         = n2_aes_setkey,
1172                         .encrypt        = n2_encrypt_ecb,
1173                         .decrypt        = n2_decrypt_ecb,
1174                 },
1175         },
1176         {       .name           = "cbc(aes)",
1177                 .drv_name       = "cbc-aes",
1178                 .block_size     = AES_BLOCK_SIZE,
1179                 .enc_type       = (ENC_TYPE_ALG_AES128 |
1180                                    ENC_TYPE_CHAINING_CBC),
1181                 .ablkcipher     = {
1182                         .ivsize         = AES_BLOCK_SIZE,
1183                         .min_keysize    = AES_MIN_KEY_SIZE,
1184                         .max_keysize    = AES_MAX_KEY_SIZE,
1185                         .setkey         = n2_aes_setkey,
1186                         .encrypt        = n2_encrypt_chaining,
1187                         .decrypt        = n2_decrypt_chaining,
1188                 },
1189         },
1190         {       .name           = "ctr(aes)",
1191                 .drv_name       = "ctr-aes",
1192                 .block_size     = AES_BLOCK_SIZE,
1193                 .enc_type       = (ENC_TYPE_ALG_AES128 |
1194                                    ENC_TYPE_CHAINING_COUNTER),
1195                 .ablkcipher     = {
1196                         .ivsize         = AES_BLOCK_SIZE,
1197                         .min_keysize    = AES_MIN_KEY_SIZE,
1198                         .max_keysize    = AES_MAX_KEY_SIZE,
1199                         .setkey         = n2_aes_setkey,
1200                         .encrypt        = n2_encrypt_chaining,
1201                         .decrypt        = n2_encrypt_chaining,
1202                 },
1203         },
1204
1205 };
1206 #define NUM_CIPHER_TMPLS ARRAY_SIZE(cipher_tmpls)
1207
1208 static LIST_HEAD(cipher_algs);
1209
1210 struct n2_hash_tmpl {
1211         const char      *name;
1212         int             (*digest)(struct ahash_request *req);
1213         u8              digest_size;
1214         u8              block_size;
1215 };
1216 static const struct n2_hash_tmpl hash_tmpls[] = {
1217         { .name         = "md5",
1218           .digest       = n2_md5_async_digest,
1219           .digest_size  = MD5_DIGEST_SIZE,
1220           .block_size   = MD5_HMAC_BLOCK_SIZE },
1221         { .name         = "sha1",
1222           .digest       = n2_sha1_async_digest,
1223           .digest_size  = SHA1_DIGEST_SIZE,
1224           .block_size   = SHA1_BLOCK_SIZE },
1225         { .name         = "sha256",
1226           .digest       = n2_sha256_async_digest,
1227           .digest_size  = SHA256_DIGEST_SIZE,
1228           .block_size   = SHA256_BLOCK_SIZE },
1229         { .name         = "sha224",
1230           .digest       = n2_sha224_async_digest,
1231           .digest_size  = SHA224_DIGEST_SIZE,
1232           .block_size   = SHA224_BLOCK_SIZE },
1233 };
1234 #define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls)
1235
1236 struct n2_ahash_alg {
1237         struct list_head        entry;
1238         struct ahash_alg        alg;
1239 };
1240 static LIST_HEAD(ahash_algs);
1241
1242 static int algs_registered;
1243
1244 static void __n2_unregister_algs(void)
1245 {
1246         struct n2_cipher_alg *cipher, *cipher_tmp;
1247         struct n2_ahash_alg *alg, *alg_tmp;
1248
1249         list_for_each_entry_safe(cipher, cipher_tmp, &cipher_algs, entry) {
1250                 crypto_unregister_alg(&cipher->alg);
1251                 list_del(&cipher->entry);
1252                 kfree(cipher);
1253         }
1254         list_for_each_entry_safe(alg, alg_tmp, &ahash_algs, entry) {
1255                 crypto_unregister_ahash(&alg->alg);
1256                 list_del(&alg->entry);
1257                 kfree(alg);
1258         }
1259 }
1260
1261 static int n2_cipher_cra_init(struct crypto_tfm *tfm)
1262 {
1263         tfm->crt_ablkcipher.reqsize = sizeof(struct n2_request_context);
1264         return 0;
1265 }
1266
1267 static int __devinit __n2_register_one_cipher(const struct n2_cipher_tmpl *tmpl)
1268 {
1269         struct n2_cipher_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
1270         struct crypto_alg *alg;
1271         int err;
1272
1273         if (!p)
1274                 return -ENOMEM;
1275
1276         alg = &p->alg;
1277
1278         snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
1279         snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->drv_name);
1280         alg->cra_priority = N2_CRA_PRIORITY;
1281         alg->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
1282         alg->cra_blocksize = tmpl->block_size;
1283         p->enc_type = tmpl->enc_type;
1284         alg->cra_ctxsize = sizeof(struct n2_cipher_context);
1285         alg->cra_type = &crypto_ablkcipher_type;
1286         alg->cra_u.ablkcipher = tmpl->ablkcipher;
1287         alg->cra_init = n2_cipher_cra_init;
1288         alg->cra_module = THIS_MODULE;
1289
1290         list_add(&p->entry, &cipher_algs);
1291         err = crypto_register_alg(alg);
1292         if (err) {
1293                 list_del(&p->entry);
1294                 kfree(p);
1295         }
1296         return err;
1297 }
1298
1299 static int __devinit __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl)
1300 {
1301         struct n2_ahash_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
1302         struct hash_alg_common *halg;
1303         struct crypto_alg *base;
1304         struct ahash_alg *ahash;
1305         int err;
1306
1307         if (!p)
1308                 return -ENOMEM;
1309
1310         ahash = &p->alg;
1311         ahash->init = n2_hash_async_init;
1312         ahash->update = n2_hash_async_update;
1313         ahash->final = n2_hash_async_final;
1314         ahash->finup = n2_hash_async_finup;
1315         ahash->digest = tmpl->digest;
1316
1317         halg = &ahash->halg;
1318         halg->digestsize = tmpl->digest_size;
1319
1320         base = &halg->base;
1321         snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
1322         snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->name);
1323         base->cra_priority = N2_CRA_PRIORITY;
1324         base->cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_NEED_FALLBACK;
1325         base->cra_blocksize = tmpl->block_size;
1326         base->cra_ctxsize = sizeof(struct n2_hash_ctx);
1327         base->cra_module = THIS_MODULE;
1328         base->cra_init = n2_hash_cra_init;
1329         base->cra_exit = n2_hash_cra_exit;
1330
1331         list_add(&p->entry, &ahash_algs);
1332         err = crypto_register_ahash(ahash);
1333         if (err) {
1334                 list_del(&p->entry);
1335                 kfree(p);
1336         }
1337         return err;
1338 }
1339
1340 static int __devinit n2_register_algs(void)
1341 {
1342         int i, err = 0;
1343
1344         mutex_lock(&spu_lock);
1345         if (algs_registered++)
1346                 goto out;
1347
1348         for (i = 0; i < NUM_HASH_TMPLS; i++) {
1349                 err = __n2_register_one_ahash(&hash_tmpls[i]);
1350                 if (err) {
1351                         __n2_unregister_algs();
1352                         goto out;
1353                 }
1354         }
1355         for (i = 0; i < NUM_CIPHER_TMPLS; i++) {
1356                 err = __n2_register_one_cipher(&cipher_tmpls[i]);
1357                 if (err) {
1358                         __n2_unregister_algs();
1359                         goto out;
1360                 }
1361         }
1362
1363 out:
1364         mutex_unlock(&spu_lock);
1365         return err;
1366 }
1367
1368 static void __exit n2_unregister_algs(void)
1369 {
1370         mutex_lock(&spu_lock);
1371         if (!--algs_registered)
1372                 __n2_unregister_algs();
1373         mutex_unlock(&spu_lock);
1374 }
1375
1376 /* To map CWQ queues to interrupt sources, the hypervisor API provides
1377  * a devino.  This isn't very useful to us because all of the
1378  * interrupts listed in the of_device node have been translated to
1379  * Linux virtual IRQ cookie numbers.
1380  *
1381  * So we have to back-translate, going through the 'intr' and 'ino'
1382  * property tables of the n2cp MDESC node, matching it with the OF
1383  * 'interrupts' property entries, in order to to figure out which
1384  * devino goes to which already-translated IRQ.
1385  */
1386 static int find_devino_index(struct of_device *dev, struct spu_mdesc_info *ip,
1387                              unsigned long dev_ino)
1388 {
1389         const unsigned int *dev_intrs;
1390         unsigned int intr;
1391         int i;
1392
1393         for (i = 0; i < ip->num_intrs; i++) {
1394                 if (ip->ino_table[i].ino == dev_ino)
1395                         break;
1396         }
1397         if (i == ip->num_intrs)
1398                 return -ENODEV;
1399
1400         intr = ip->ino_table[i].intr;
1401
1402         dev_intrs = of_get_property(dev->dev.of_node, "interrupts", NULL);
1403         if (!dev_intrs)
1404                 return -ENODEV;
1405
1406         for (i = 0; i < dev->num_irqs; i++) {
1407                 if (dev_intrs[i] == intr)
1408                         return i;
1409         }
1410
1411         return -ENODEV;
1412 }
1413
1414 static int spu_map_ino(struct of_device *dev, struct spu_mdesc_info *ip,
1415                        const char *irq_name, struct spu_queue *p,
1416                        irq_handler_t handler)
1417 {
1418         unsigned long herr;
1419         int index;
1420
1421         herr = sun4v_ncs_qhandle_to_devino(p->qhandle, &p->devino);
1422         if (herr)
1423                 return -EINVAL;
1424
1425         index = find_devino_index(dev, ip, p->devino);
1426         if (index < 0)
1427                 return index;
1428
1429         p->irq = dev->irqs[index];
1430
1431         sprintf(p->irq_name, "%s-%d", irq_name, index);
1432
1433         return request_irq(p->irq, handler, IRQF_SAMPLE_RANDOM,
1434                            p->irq_name, p);
1435 }
1436
1437 static struct kmem_cache *queue_cache[2];
1438
1439 static void *new_queue(unsigned long q_type)
1440 {
1441         return kmem_cache_zalloc(queue_cache[q_type - 1], GFP_KERNEL);
1442 }
1443
1444 static void free_queue(void *p, unsigned long q_type)
1445 {
1446         return kmem_cache_free(queue_cache[q_type - 1], p);
1447 }
1448
1449 static int queue_cache_init(void)
1450 {
1451         if (!queue_cache[HV_NCS_QTYPE_MAU - 1])
1452                 queue_cache[HV_NCS_QTYPE_MAU - 1] =
1453                         kmem_cache_create("mau_queue",
1454                                           (MAU_NUM_ENTRIES *
1455                                            MAU_ENTRY_SIZE),
1456                                           MAU_ENTRY_SIZE, 0, NULL);
1457         if (!queue_cache[HV_NCS_QTYPE_MAU - 1])
1458                 return -ENOMEM;
1459
1460         if (!queue_cache[HV_NCS_QTYPE_CWQ - 1])
1461                 queue_cache[HV_NCS_QTYPE_CWQ - 1] =
1462                         kmem_cache_create("cwq_queue",
1463                                           (CWQ_NUM_ENTRIES *
1464                                            CWQ_ENTRY_SIZE),
1465                                           CWQ_ENTRY_SIZE, 0, NULL);
1466         if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) {
1467                 kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);
1468                 return -ENOMEM;
1469         }
1470         return 0;
1471 }
1472
1473 static void queue_cache_destroy(void)
1474 {
1475         kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);
1476         kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_CWQ - 1]);
1477 }
1478
1479 static int spu_queue_register(struct spu_queue *p, unsigned long q_type)
1480 {
1481         cpumask_var_t old_allowed;
1482         unsigned long hv_ret;
1483
1484         if (cpumask_empty(&p->sharing))
1485                 return -EINVAL;
1486
1487         if (!alloc_cpumask_var(&old_allowed, GFP_KERNEL))
1488                 return -ENOMEM;
1489
1490         cpumask_copy(old_allowed, &current->cpus_allowed);
1491
1492         set_cpus_allowed_ptr(current, &p->sharing);
1493
1494         hv_ret = sun4v_ncs_qconf(q_type, __pa(p->q),
1495                                  CWQ_NUM_ENTRIES, &p->qhandle);
1496         if (!hv_ret)
1497                 sun4v_ncs_sethead_marker(p->qhandle, 0);
1498
1499         set_cpus_allowed_ptr(current, old_allowed);
1500
1501         free_cpumask_var(old_allowed);
1502
1503         return (hv_ret ? -EINVAL : 0);
1504 }
1505
1506 static int spu_queue_setup(struct spu_queue *p)
1507 {
1508         int err;
1509
1510         p->q = new_queue(p->q_type);
1511         if (!p->q)
1512                 return -ENOMEM;
1513
1514         err = spu_queue_register(p, p->q_type);
1515         if (err) {
1516                 free_queue(p->q, p->q_type);
1517                 p->q = NULL;
1518         }
1519
1520         return err;
1521 }
1522
1523 static void spu_queue_destroy(struct spu_queue *p)
1524 {
1525         unsigned long hv_ret;
1526
1527         if (!p->q)
1528                 return;
1529
1530         hv_ret = sun4v_ncs_qconf(p->q_type, p->qhandle, 0, &p->qhandle);
1531
1532         if (!hv_ret)
1533                 free_queue(p->q, p->q_type);
1534 }
1535
1536 static void spu_list_destroy(struct list_head *list)
1537 {
1538         struct spu_queue *p, *n;
1539
1540         list_for_each_entry_safe(p, n, list, list) {
1541                 int i;
1542
1543                 for (i = 0; i < NR_CPUS; i++) {
1544                         if (cpu_to_cwq[i] == p)
1545                                 cpu_to_cwq[i] = NULL;
1546                 }
1547
1548                 if (p->irq) {
1549                         free_irq(p->irq, p);
1550                         p->irq = 0;
1551                 }
1552                 spu_queue_destroy(p);
1553                 list_del(&p->list);
1554                 kfree(p);
1555         }
1556 }
1557
1558 /* Walk the backward arcs of a CWQ 'exec-unit' node,
1559  * gathering cpu membership information.
1560  */
1561 static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc,
1562                                struct of_device *dev,
1563                                u64 node, struct spu_queue *p,
1564                                struct spu_queue **table)
1565 {
1566         u64 arc;
1567
1568         mdesc_for_each_arc(arc, mdesc, node, MDESC_ARC_TYPE_BACK) {
1569                 u64 tgt = mdesc_arc_target(mdesc, arc);
1570                 const char *name = mdesc_node_name(mdesc, tgt);
1571                 const u64 *id;
1572
1573                 if (strcmp(name, "cpu"))
1574                         continue;
1575                 id = mdesc_get_property(mdesc, tgt, "id", NULL);
1576                 if (table[*id] != NULL) {
1577                         dev_err(&dev->dev, "%s: SPU cpu slot already set.\n",
1578                                 dev->dev.of_node->full_name);
1579                         return -EINVAL;
1580                 }
1581                 cpu_set(*id, p->sharing);
1582                 table[*id] = p;
1583         }
1584         return 0;
1585 }
1586
1587 /* Process an 'exec-unit' MDESC node of type 'cwq'.  */
1588 static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list,
1589                             struct of_device *dev, struct mdesc_handle *mdesc,
1590                             u64 node, const char *iname, unsigned long q_type,
1591                             irq_handler_t handler, struct spu_queue **table)
1592 {
1593         struct spu_queue *p;
1594         int err;
1595
1596         p = kzalloc(sizeof(struct spu_queue), GFP_KERNEL);
1597         if (!p) {
1598                 dev_err(&dev->dev, "%s: Could not allocate SPU queue.\n",
1599                         dev->dev.of_node->full_name);
1600                 return -ENOMEM;
1601         }
1602
1603         cpus_clear(p->sharing);
1604         spin_lock_init(&p->lock);
1605         p->q_type = q_type;
1606         INIT_LIST_HEAD(&p->jobs);
1607         list_add(&p->list, list);
1608
1609         err = spu_mdesc_walk_arcs(mdesc, dev, node, p, table);
1610         if (err)
1611                 return err;
1612
1613         err = spu_queue_setup(p);
1614         if (err)
1615                 return err;
1616
1617         return spu_map_ino(dev, ip, iname, p, handler);
1618 }
1619
1620 static int spu_mdesc_scan(struct mdesc_handle *mdesc, struct of_device *dev,
1621                           struct spu_mdesc_info *ip, struct list_head *list,
1622                           const char *exec_name, unsigned long q_type,
1623                           irq_handler_t handler, struct spu_queue **table)
1624 {
1625         int err = 0;
1626         u64 node;
1627
1628         mdesc_for_each_node_by_name(mdesc, node, "exec-unit") {
1629                 const char *type;
1630
1631                 type = mdesc_get_property(mdesc, node, "type", NULL);
1632                 if (!type || strcmp(type, exec_name))
1633                         continue;
1634
1635                 err = handle_exec_unit(ip, list, dev, mdesc, node,
1636                                        exec_name, q_type, handler, table);
1637                 if (err) {
1638                         spu_list_destroy(list);
1639                         break;
1640                 }
1641         }
1642
1643         return err;
1644 }
1645
1646 static int __devinit get_irq_props(struct mdesc_handle *mdesc, u64 node,
1647                                    struct spu_mdesc_info *ip)
1648 {
1649         const u64 *intr, *ino;
1650         int intr_len, ino_len;
1651         int i;
1652
1653         intr = mdesc_get_property(mdesc, node, "intr", &intr_len);
1654         if (!intr)
1655                 return -ENODEV;
1656
1657         ino = mdesc_get_property(mdesc, node, "ino", &ino_len);
1658         if (!intr)
1659                 return -ENODEV;
1660
1661         if (intr_len != ino_len)
1662                 return -EINVAL;
1663
1664         ip->num_intrs = intr_len / sizeof(u64);
1665         ip->ino_table = kzalloc((sizeof(struct ino_blob) *
1666                                  ip->num_intrs),
1667                                 GFP_KERNEL);
1668         if (!ip->ino_table)
1669                 return -ENOMEM;
1670
1671         for (i = 0; i < ip->num_intrs; i++) {
1672                 struct ino_blob *b = &ip->ino_table[i];
1673                 b->intr = intr[i];
1674                 b->ino = ino[i];
1675         }
1676
1677         return 0;
1678 }
1679
1680 static int __devinit grab_mdesc_irq_props(struct mdesc_handle *mdesc,
1681                                           struct of_device *dev,
1682                                           struct spu_mdesc_info *ip,
1683                                           const char *node_name)
1684 {
1685         const unsigned int *reg;
1686         u64 node;
1687
1688         reg = of_get_property(dev->dev.of_node, "reg", NULL);
1689         if (!reg)
1690                 return -ENODEV;
1691
1692         mdesc_for_each_node_by_name(mdesc, node, "virtual-device") {
1693                 const char *name;
1694                 const u64 *chdl;
1695
1696                 name = mdesc_get_property(mdesc, node, "name", NULL);
1697                 if (!name || strcmp(name, node_name))
1698                         continue;
1699                 chdl = mdesc_get_property(mdesc, node, "cfg-handle", NULL);
1700                 if (!chdl || (*chdl != *reg))
1701                         continue;
1702                 ip->cfg_handle = *chdl;
1703                 return get_irq_props(mdesc, node, ip);
1704         }
1705
1706         return -ENODEV;
1707 }
1708
1709 static unsigned long n2_spu_hvapi_major;
1710 static unsigned long n2_spu_hvapi_minor;
1711
1712 static int __devinit n2_spu_hvapi_register(void)
1713 {
1714         int err;
1715
1716         n2_spu_hvapi_major = 2;
1717         n2_spu_hvapi_minor = 0;
1718
1719         err = sun4v_hvapi_register(HV_GRP_NCS,
1720                                    n2_spu_hvapi_major,
1721                                    &n2_spu_hvapi_minor);
1722
1723         if (!err)
1724                 pr_info("Registered NCS HVAPI version %lu.%lu\n",
1725                         n2_spu_hvapi_major,
1726                         n2_spu_hvapi_minor);
1727
1728         return err;
1729 }
1730
1731 static void n2_spu_hvapi_unregister(void)
1732 {
1733         sun4v_hvapi_unregister(HV_GRP_NCS);
1734 }
1735
1736 static int global_ref;
1737
1738 static int __devinit grab_global_resources(void)
1739 {
1740         int err = 0;
1741
1742         mutex_lock(&spu_lock);
1743
1744         if (global_ref++)
1745                 goto out;
1746
1747         err = n2_spu_hvapi_register();
1748         if (err)
1749                 goto out;
1750
1751         err = queue_cache_init();
1752         if (err)
1753                 goto out_hvapi_release;
1754
1755         err = -ENOMEM;
1756         cpu_to_cwq = kzalloc(sizeof(struct spu_queue *) * NR_CPUS,
1757                              GFP_KERNEL);
1758         if (!cpu_to_cwq)
1759                 goto out_queue_cache_destroy;
1760
1761         cpu_to_mau = kzalloc(sizeof(struct spu_queue *) * NR_CPUS,
1762                              GFP_KERNEL);
1763         if (!cpu_to_mau)
1764                 goto out_free_cwq_table;
1765
1766         err = 0;
1767
1768 out:
1769         if (err)
1770                 global_ref--;
1771         mutex_unlock(&spu_lock);
1772         return err;
1773
1774 out_free_cwq_table:
1775         kfree(cpu_to_cwq);
1776         cpu_to_cwq = NULL;
1777
1778 out_queue_cache_destroy:
1779         queue_cache_destroy();
1780
1781 out_hvapi_release:
1782         n2_spu_hvapi_unregister();
1783         goto out;
1784 }
1785
1786 static void release_global_resources(void)
1787 {
1788         mutex_lock(&spu_lock);
1789         if (!--global_ref) {
1790                 kfree(cpu_to_cwq);
1791                 cpu_to_cwq = NULL;
1792
1793                 kfree(cpu_to_mau);
1794                 cpu_to_mau = NULL;
1795
1796                 queue_cache_destroy();
1797                 n2_spu_hvapi_unregister();
1798         }
1799         mutex_unlock(&spu_lock);
1800 }
1801
1802 static struct n2_crypto * __devinit alloc_n2cp(void)
1803 {
1804         struct n2_crypto *np = kzalloc(sizeof(struct n2_crypto), GFP_KERNEL);
1805
1806         if (np)
1807                 INIT_LIST_HEAD(&np->cwq_list);
1808
1809         return np;
1810 }
1811
1812 static void free_n2cp(struct n2_crypto *np)
1813 {
1814         if (np->cwq_info.ino_table) {
1815                 kfree(np->cwq_info.ino_table);
1816                 np->cwq_info.ino_table = NULL;
1817         }
1818
1819         kfree(np);
1820 }
1821
1822 static void __devinit n2_spu_driver_version(void)
1823 {
1824         static int n2_spu_version_printed;
1825
1826         if (n2_spu_version_printed++ == 0)
1827                 pr_info("%s", version);
1828 }
1829
1830 static int __devinit n2_crypto_probe(struct of_device *dev,
1831                                      const struct of_device_id *match)
1832 {
1833         struct mdesc_handle *mdesc;
1834         const char *full_name;
1835         struct n2_crypto *np;
1836         int err;
1837
1838         n2_spu_driver_version();
1839
1840         full_name = dev->dev.of_node->full_name;
1841         pr_info("Found N2CP at %s\n", full_name);
1842
1843         np = alloc_n2cp();
1844         if (!np) {
1845                 dev_err(&dev->dev, "%s: Unable to allocate n2cp.\n",
1846                         full_name);
1847                 return -ENOMEM;
1848         }
1849
1850         err = grab_global_resources();
1851         if (err) {
1852                 dev_err(&dev->dev, "%s: Unable to grab "
1853                         "global resources.\n", full_name);
1854                 goto out_free_n2cp;
1855         }
1856
1857         mdesc = mdesc_grab();
1858
1859         if (!mdesc) {
1860                 dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",
1861                         full_name);
1862                 err = -ENODEV;
1863                 goto out_free_global;
1864         }
1865         err = grab_mdesc_irq_props(mdesc, dev, &np->cwq_info, "n2cp");
1866         if (err) {
1867                 dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",
1868                         full_name);
1869                 mdesc_release(mdesc);
1870                 goto out_free_global;
1871         }
1872
1873         err = spu_mdesc_scan(mdesc, dev, &np->cwq_info, &np->cwq_list,
1874                              "cwq", HV_NCS_QTYPE_CWQ, cwq_intr,
1875                              cpu_to_cwq);
1876         mdesc_release(mdesc);
1877
1878         if (err) {
1879                 dev_err(&dev->dev, "%s: CWQ MDESC scan failed.\n",
1880                         full_name);
1881                 goto out_free_global;
1882         }
1883
1884         err = n2_register_algs();
1885         if (err) {
1886                 dev_err(&dev->dev, "%s: Unable to register algorithms.\n",
1887                         full_name);
1888                 goto out_free_spu_list;
1889         }
1890
1891         dev_set_drvdata(&dev->dev, np);
1892
1893         return 0;
1894
1895 out_free_spu_list:
1896         spu_list_destroy(&np->cwq_list);
1897
1898 out_free_global:
1899         release_global_resources();
1900
1901 out_free_n2cp:
1902         free_n2cp(np);
1903
1904         return err;
1905 }
1906
1907 static int __devexit n2_crypto_remove(struct of_device *dev)
1908 {
1909         struct n2_crypto *np = dev_get_drvdata(&dev->dev);
1910
1911         n2_unregister_algs();
1912
1913         spu_list_destroy(&np->cwq_list);
1914
1915         release_global_resources();
1916
1917         free_n2cp(np);
1918
1919         return 0;
1920 }
1921
1922 static struct n2_mau * __devinit alloc_ncp(void)
1923 {
1924         struct n2_mau *mp = kzalloc(sizeof(struct n2_mau), GFP_KERNEL);
1925
1926         if (mp)
1927                 INIT_LIST_HEAD(&mp->mau_list);
1928
1929         return mp;
1930 }
1931
1932 static void free_ncp(struct n2_mau *mp)
1933 {
1934         if (mp->mau_info.ino_table) {
1935                 kfree(mp->mau_info.ino_table);
1936                 mp->mau_info.ino_table = NULL;
1937         }
1938
1939         kfree(mp);
1940 }
1941
1942 static int __devinit n2_mau_probe(struct of_device *dev,
1943                                      const struct of_device_id *match)
1944 {
1945         struct mdesc_handle *mdesc;
1946         const char *full_name;
1947         struct n2_mau *mp;
1948         int err;
1949
1950         n2_spu_driver_version();
1951
1952         full_name = dev->dev.of_node->full_name;
1953         pr_info("Found NCP at %s\n", full_name);
1954
1955         mp = alloc_ncp();
1956         if (!mp) {
1957                 dev_err(&dev->dev, "%s: Unable to allocate ncp.\n",
1958                         full_name);
1959                 return -ENOMEM;
1960         }
1961
1962         err = grab_global_resources();
1963         if (err) {
1964                 dev_err(&dev->dev, "%s: Unable to grab "
1965                         "global resources.\n", full_name);
1966                 goto out_free_ncp;
1967         }
1968
1969         mdesc = mdesc_grab();
1970
1971         if (!mdesc) {
1972                 dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",
1973                         full_name);
1974                 err = -ENODEV;
1975                 goto out_free_global;
1976         }
1977
1978         err = grab_mdesc_irq_props(mdesc, dev, &mp->mau_info, "ncp");
1979         if (err) {
1980                 dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",
1981                         full_name);
1982                 mdesc_release(mdesc);
1983                 goto out_free_global;
1984         }
1985
1986         err = spu_mdesc_scan(mdesc, dev, &mp->mau_info, &mp->mau_list,
1987                              "mau", HV_NCS_QTYPE_MAU, mau_intr,
1988                              cpu_to_mau);
1989         mdesc_release(mdesc);
1990
1991         if (err) {
1992                 dev_err(&dev->dev, "%s: MAU MDESC scan failed.\n",
1993                         full_name);
1994                 goto out_free_global;
1995         }
1996
1997         dev_set_drvdata(&dev->dev, mp);
1998
1999         return 0;
2000
2001 out_free_global:
2002         release_global_resources();
2003
2004 out_free_ncp:
2005         free_ncp(mp);
2006
2007         return err;
2008 }
2009
2010 static int __devexit n2_mau_remove(struct of_device *dev)
2011 {
2012         struct n2_mau *mp = dev_get_drvdata(&dev->dev);
2013
2014         spu_list_destroy(&mp->mau_list);
2015
2016         release_global_resources();
2017
2018         free_ncp(mp);
2019
2020         return 0;
2021 }
2022
2023 static struct of_device_id n2_crypto_match[] = {
2024         {
2025                 .name = "n2cp",
2026                 .compatible = "SUNW,n2-cwq",
2027         },
2028         {
2029                 .name = "n2cp",
2030                 .compatible = "SUNW,vf-cwq",
2031         },
2032         {},
2033 };
2034
2035 MODULE_DEVICE_TABLE(of, n2_crypto_match);
2036
2037 static struct of_platform_driver n2_crypto_driver = {
2038         .driver = {
2039                 .name           =       "n2cp",
2040                 .owner          =       THIS_MODULE,
2041                 .of_match_table =       n2_crypto_match,
2042         },
2043         .probe          =       n2_crypto_probe,
2044         .remove         =       __devexit_p(n2_crypto_remove),
2045 };
2046
2047 static struct of_device_id n2_mau_match[] = {
2048         {
2049                 .name = "ncp",
2050                 .compatible = "SUNW,n2-mau",
2051         },
2052         {
2053                 .name = "ncp",
2054                 .compatible = "SUNW,vf-mau",
2055         },
2056         {},
2057 };
2058
2059 MODULE_DEVICE_TABLE(of, n2_mau_match);
2060
2061 static struct of_platform_driver n2_mau_driver = {
2062         .driver = {
2063                 .name           =       "ncp",
2064                 .owner          =       THIS_MODULE,
2065                 .of_match_table =       n2_mau_match,
2066         },
2067         .probe          =       n2_mau_probe,
2068         .remove         =       __devexit_p(n2_mau_remove),
2069 };
2070
2071 static int __init n2_init(void)
2072 {
2073         int err = of_register_driver(&n2_crypto_driver, &of_bus_type);
2074
2075         if (!err) {
2076                 err = of_register_driver(&n2_mau_driver, &of_bus_type);
2077                 if (err)
2078                         of_unregister_driver(&n2_crypto_driver);
2079         }
2080         return err;
2081 }
2082
2083 static void __exit n2_exit(void)
2084 {
2085         of_unregister_driver(&n2_mau_driver);
2086         of_unregister_driver(&n2_crypto_driver);
2087 }
2088
2089 module_init(n2_init);
2090 module_exit(n2_exit);