arch/cris/arch-v32/drivers/cryptocop.c

   1 /*
   2  * Stream co-processor driver for the ETRAX FS
   3  *
   4  *    Copyright (C) 2003-2007  Axis Communications AB
   5  */
   6
   7 #include <linux/init.h>
   8 #include <linux/sched.h>
   9 #include <linux/module.h>
  10 #include <linux/slab.h>
  11 #include <linux/string.h>
  12 #include <linux/fs.h>
  13 #include <linux/mm.h>
  14 #include <linux/spinlock.h>
  15 #include <linux/stddef.h>
  16
  17 #include <asm/uaccess.h>
  18 #include <asm/io.h>
  19 #include <asm/atomic.h>
  20
  21 #include <linux/list.h>
  22 #include <linux/interrupt.h>
  23
  24 #include <asm/signal.h>
  25 #include <asm/irq.h>
  26
  27 #include <dma.h>
  28 #include <hwregs/dma.h>
  29 #include <hwregs/reg_map.h>
  30 #include <hwregs/reg_rdwr.h>
  31 #include <hwregs/intr_vect_defs.h>
  32
  33 #include <hwregs/strcop.h>
  34 #include <hwregs/strcop_defs.h>
  35 #include <cryptocop.h>
  36
  37 #ifdef CONFIG_ETRAXFS
  38 #define IN_DMA 9
  39 #define OUT_DMA 8
  40 #define IN_DMA_INST regi_dma9
  41 #define OUT_DMA_INST regi_dma8
  42 #define DMA_IRQ DMA9_INTR_VECT
  43 #else
  44 #define IN_DMA 3
  45 #define OUT_DMA 2
  46 #define IN_DMA_INST regi_dma3
  47 #define OUT_DMA_INST regi_dma2
  48 #define DMA_IRQ DMA3_INTR_VECT
  49 #endif
  50
  51 #define DESCR_ALLOC_PAD  (31)
  52
  53 struct cryptocop_dma_desc {
  54         char *free_buf; /* If non-null will be kfreed in free_cdesc() */
  55         dma_descr_data *dma_descr;
  56
  57         unsigned char dma_descr_buf[sizeof(dma_descr_data) + DESCR_ALLOC_PAD];
  58
  59         unsigned int from_pool:1; /* If 1 'allocated' from the descriptor pool. */
  60         struct cryptocop_dma_desc *next;
  61 };
  62
  63
  64 struct cryptocop_int_operation{
  65         void                        *alloc_ptr;
  66         cryptocop_session_id        sid;
  67
  68         dma_descr_context           ctx_out;
  69         dma_descr_context           ctx_in;
  70
  71         /* DMA descriptors allocated by driver. */
  72         struct cryptocop_dma_desc   *cdesc_out;
  73         struct cryptocop_dma_desc   *cdesc_in;
  74
  75         /* Strcop config to use. */
  76         cryptocop_3des_mode         tdes_mode;
  77         cryptocop_csum_type         csum_mode;
  78
  79         /* DMA descrs provided by consumer. */
  80         dma_descr_data              *ddesc_out;
  81         dma_descr_data              *ddesc_in;
  82 };
  83
  84
  85 struct cryptocop_tfrm_ctx {
  86         cryptocop_tfrm_id tid;
  87         unsigned int blocklength;
  88
  89         unsigned int start_ix;
  90
  91         struct cryptocop_tfrm_cfg *tcfg;
  92         struct cryptocop_transform_ctx *tctx;
  93
  94         unsigned char previous_src;
  95         unsigned char current_src;
  96
  97         /* Values to use in metadata out. */
  98         unsigned char hash_conf;
  99         unsigned char hash_mode;
 100         unsigned char ciph_conf;
 101         unsigned char cbcmode;
 102         unsigned char decrypt;
 103
 104         unsigned int requires_padding:1;
 105         unsigned int strict_block_length:1;
 106         unsigned int active:1;
 107         unsigned int done:1;
 108         size_t consumed;
 109         size_t produced;
 110
 111         /* Pad (input) descriptors to put in the DMA out list when the transform
 112          * output is put on the DMA in list. */
 113         struct cryptocop_dma_desc *pad_descs;
 114
 115         struct cryptocop_tfrm_ctx *prev_src;
 116         struct cryptocop_tfrm_ctx *curr_src;
 117
 118         /* Mapping to HW. */
 119         unsigned char unit_no;
 120 };
 121
 122
 123 struct cryptocop_private{
 124         cryptocop_session_id sid;
 125         struct cryptocop_private *next;
 126 };
 127
 128 /* Session list. */
 129
 130 struct cryptocop_transform_ctx{
 131         struct cryptocop_transform_init init;
 132         unsigned char dec_key[CRYPTOCOP_MAX_KEY_LENGTH];
 133         unsigned int dec_key_set:1;
 134
 135         struct cryptocop_transform_ctx *next;
 136 };
 137
 138
 139 struct cryptocop_session{
 140         cryptocop_session_id sid;
 141
 142         struct cryptocop_transform_ctx *tfrm_ctx;
 143
 144         struct cryptocop_session *next;
 145 };
 146
 147 /* Priority levels for jobs sent to the cryptocop.  Checksum operations from
 148    kernel have highest priority since TCPIP stack processing must not
 149    be a bottleneck. */
 150 typedef enum {
 151         cryptocop_prio_kernel_csum = 0,
 152         cryptocop_prio_kernel = 1,
 153         cryptocop_prio_user = 2,
 154         cryptocop_prio_no_prios = 3
 155 } cryptocop_queue_priority;
 156
 157 struct cryptocop_prio_queue{
 158         struct list_head jobs;
 159         cryptocop_queue_priority prio;
 160 };
 161
 162 struct cryptocop_prio_job{
 163         struct list_head node;
 164         cryptocop_queue_priority prio;
 165
 166         struct cryptocop_operation *oper;
 167         struct cryptocop_int_operation *iop;
 168 };
 169
 170 struct ioctl_job_cb_ctx {
 171         unsigned int processed:1;
 172 };
 173
 174
 175 static struct cryptocop_session *cryptocop_sessions = NULL;
 176 spinlock_t cryptocop_sessions_lock;
 177
 178 /* Next Session ID to assign. */
 179 static cryptocop_session_id next_sid = 1;
 180
 181 /* Pad for checksum. */
 182 static const char csum_zero_pad[1] = {0x00};
 183
 184 /* Trash buffer for mem2mem operations. */
 185 #define MEM2MEM_DISCARD_BUF_LENGTH  (512)
 186 static unsigned char mem2mem_discard_buf[MEM2MEM_DISCARD_BUF_LENGTH];
 187
 188 /* Descriptor pool. */
 189 /* FIXME Tweak this value. */
 190 #define CRYPTOCOP_DESCRIPTOR_POOL_SIZE   (100)
 191 static struct cryptocop_dma_desc descr_pool[CRYPTOCOP_DESCRIPTOR_POOL_SIZE];
 192 static struct cryptocop_dma_desc *descr_pool_free_list;
 193 static int descr_pool_no_free;
 194 static spinlock_t descr_pool_lock;
 195
 196 /* Lock to stop cryptocop to start processing of a new operation. The holder
 197    of this lock MUST call cryptocop_start_job() after it is unlocked. */
 198 spinlock_t cryptocop_process_lock;
 199
 200 static struct cryptocop_prio_queue cryptocop_job_queues[cryptocop_prio_no_prios];
 201 static spinlock_t cryptocop_job_queue_lock;
 202 static struct cryptocop_prio_job *cryptocop_running_job = NULL;
 203 static spinlock_t running_job_lock;
 204
 205 /* The interrupt handler appends completed jobs to this list. The scehduled
 206  * tasklet removes them upon sending the response to the crypto consumer. */
 207 static struct list_head cryptocop_completed_jobs;
 208 static spinlock_t cryptocop_completed_jobs_lock;
 209
 210 DECLARE_WAIT_QUEUE_HEAD(cryptocop_ioc_process_wq);
 211
 212
 213 /** Local functions. **/
 214
 215 static int cryptocop_open(struct inode *, struct file *);
 216
 217 static int cryptocop_release(struct inode *, struct file *);
 218
 219 static int cryptocop_ioctl(struct inode *inode, struct file *file,
 220                            unsigned int cmd, unsigned long arg);
 221
 222 static void cryptocop_start_job(void);
 223
 224 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation);
 225 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation);
 226
 227 static int cryptocop_job_queue_init(void);
 228 static void cryptocop_job_queue_close(void);
 229
 230 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
 231
 232 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
 233
 234 static int transform_ok(struct cryptocop_transform_init *tinit);
 235
 236 static struct cryptocop_session *get_session(cryptocop_session_id sid);
 237
 238 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid);
 239
 240 static void delete_internal_operation(struct cryptocop_int_operation *iop);
 241
 242 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned  char *key, unsigned int keylength);
 243
 244 static int init_stream_coprocessor(void);
 245
 246 static void __exit exit_stream_coprocessor(void);
 247
 248 /*#define LDEBUG*/
 249 #ifdef LDEBUG
 250 #define DEBUG(s) s
 251 #define DEBUG_API(s) s
 252 static void print_cryptocop_operation(struct cryptocop_operation *cop);
 253 static void print_dma_descriptors(struct cryptocop_int_operation *iop);
 254 static void print_strcop_crypto_op(struct strcop_crypto_op *cop);
 255 static void print_lock_status(void);
 256 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op);
 257 #define assert(s) do{if (!(s)) panic(#s);} while(0);
 258 #else
 259 #define DEBUG(s)
 260 #define DEBUG_API(s)
 261 #define assert(s)
 262 #endif
 263
 264
 265 /* Transform constants. */
 266 #define DES_BLOCK_LENGTH   (8)
 267 #define AES_BLOCK_LENGTH   (16)
 268 #define MD5_BLOCK_LENGTH   (64)
 269 #define SHA1_BLOCK_LENGTH  (64)
 270 #define CSUM_BLOCK_LENGTH  (2)
 271 #define MD5_STATE_LENGTH   (16)
 272 #define SHA1_STATE_LENGTH  (20)
 273
 274 /* The device number. */
 275 #define CRYPTOCOP_MAJOR    (254)
 276 #define CRYPTOCOP_MINOR    (0)
 277
 278
 279
 280 const struct file_operations cryptocop_fops = {
 281         .owner =        THIS_MODULE,
 282         .open =         cryptocop_open,
 283         .release =      cryptocop_release,
 284         .ioctl =        cryptocop_ioctl
 285 };
 286
 287
 288 static void free_cdesc(struct cryptocop_dma_desc *cdesc)
 289 {
 290         DEBUG(printk("free_cdesc: cdesc 0x%p, from_pool=%d\n", cdesc, cdesc->from_pool));
 291         kfree(cdesc->free_buf);
 292
 293         if (cdesc->from_pool) {
 294                 unsigned long int flags;
 295                 spin_lock_irqsave(&descr_pool_lock, flags);
 296                 cdesc->next = descr_pool_free_list;
 297                 descr_pool_free_list = cdesc;
 298                 ++descr_pool_no_free;
 299                 spin_unlock_irqrestore(&descr_pool_lock, flags);
 300         } else {
 301                 kfree(cdesc);
 302         }
 303 }
 304
 305
 306 static struct cryptocop_dma_desc *alloc_cdesc(int alloc_flag)
 307 {
 308         int use_pool = (alloc_flag & GFP_ATOMIC) ? 1 : 0;
 309         struct cryptocop_dma_desc *cdesc;
 310
 311         if (use_pool) {
 312                 unsigned long int flags;
 313                 spin_lock_irqsave(&descr_pool_lock, flags);
 314                 if (!descr_pool_free_list) {
 315                         spin_unlock_irqrestore(&descr_pool_lock, flags);
 316                         DEBUG_API(printk("alloc_cdesc: pool is empty\n"));
 317                         return NULL;
 318                 }
 319                 cdesc = descr_pool_free_list;
 320                 descr_pool_free_list = descr_pool_free_list->next;
 321                 --descr_pool_no_free;
 322                 spin_unlock_irqrestore(&descr_pool_lock, flags);
 323                 cdesc->from_pool = 1;
 324         } else {
 325                 cdesc = kmalloc(sizeof(struct cryptocop_dma_desc), alloc_flag);
 326                 if (!cdesc) {
 327                         DEBUG_API(printk("alloc_cdesc: kmalloc\n"));
 328                         return NULL;
 329                 }
 330                 cdesc->from_pool = 0;
 331         }
 332         cdesc->dma_descr = (dma_descr_data*)(((unsigned long int)cdesc + offsetof(struct cryptocop_dma_desc, dma_descr_buf) + DESCR_ALLOC_PAD) & ~0x0000001F);
 333
 334         cdesc->next = NULL;
 335
 336         cdesc->free_buf = NULL;
 337         cdesc->dma_descr->out_eop = 0;
 338         cdesc->dma_descr->in_eop = 0;
 339         cdesc->dma_descr->intr = 0;
 340         cdesc->dma_descr->eol = 0;
 341         cdesc->dma_descr->wait = 0;
 342         cdesc->dma_descr->buf = NULL;
 343         cdesc->dma_descr->after = NULL;
 344
 345         DEBUG_API(printk("alloc_cdesc: return 0x%p, cdesc->dma_descr=0x%p, from_pool=%d\n", cdesc, cdesc->dma_descr, cdesc->from_pool));
 346         return cdesc;
 347 }
 348
 349
 350 static void setup_descr_chain(struct cryptocop_dma_desc *cd)
 351 {
 352         DEBUG(printk("setup_descr_chain: entering\n"));
 353         while (cd) {
 354                 if (cd->next) {
 355                         cd->dma_descr->next = (dma_descr_data*)virt_to_phys(cd->next->dma_descr);
 356                 } else {
 357                         cd->dma_descr->next = NULL;
 358                 }
 359                 cd = cd->next;
 360         }
 361         DEBUG(printk("setup_descr_chain: exit\n"));
 362 }
 363
 364
 365 /* Create a pad descriptor for the transform.
 366  * Return -1 for error, 0 if pad created. */
 367 static int create_pad_descriptor(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **pad_desc, int alloc_flag)
 368 {
 369         struct cryptocop_dma_desc        *cdesc = NULL;
 370         int                              error = 0;
 371         struct strcop_meta_out           mo = {
 372                 .ciphsel = src_none,
 373                 .hashsel = src_none,
 374                 .csumsel = src_none
 375         };
 376         char                             *pad;
 377         size_t                           plen;
 378
 379         DEBUG(printk("create_pad_descriptor: start.\n"));
 380         /* Setup pad descriptor. */
 381
 382         DEBUG(printk("create_pad_descriptor: setting up padding.\n"));
 383         cdesc = alloc_cdesc(alloc_flag);
 384         if (!cdesc){
 385                 DEBUG_API(printk("create_pad_descriptor: alloc pad desc\n"));
 386                 goto error_cleanup;
 387         }
 388         switch (tc->unit_no) {
 389         case src_md5:
 390                 error = create_md5_pad(alloc_flag, tc->consumed, &pad, &plen);
 391                 if (error){
 392                         DEBUG_API(printk("create_pad_descriptor: create_md5_pad_failed\n"));
 393                         goto error_cleanup;
 394                 }
 395                 cdesc->free_buf = pad;
 396                 mo.hashsel = src_dma;
 397                 mo.hashconf = tc->hash_conf;
 398                 mo.hashmode = tc->hash_mode;
 399                 break;
 400         case src_sha1:
 401                 error = create_sha1_pad(alloc_flag, tc->consumed, &pad, &plen);
 402                 if (error){
 403                         DEBUG_API(printk("create_pad_descriptor: create_sha1_pad_failed\n"));
 404                         goto error_cleanup;
 405                 }
 406                 cdesc->free_buf = pad;
 407                 mo.hashsel = src_dma;
 408                 mo.hashconf = tc->hash_conf;
 409                 mo.hashmode = tc->hash_mode;
 410                 break;
 411         case src_csum:
 412                 if (tc->consumed % tc->blocklength){
 413                         pad = (char*)csum_zero_pad;
 414                         plen = 1;
 415                 } else {
 416                         pad = (char*)cdesc; /* Use any pointer. */
 417                         plen = 0;
 418                 }
 419                 mo.csumsel = src_dma;
 420                 break;
 421         }
 422         cdesc->dma_descr->wait = 1;
 423         cdesc->dma_descr->out_eop = 1; /* Since this is a pad output is pushed.  EOP is ok here since the padded unit is the only one active. */
 424         cdesc->dma_descr->buf = (char*)virt_to_phys((char*)pad);
 425         cdesc->dma_descr->after = cdesc->dma_descr->buf + plen;
 426
 427         cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
 428         *pad_desc = cdesc;
 429
 430         return 0;
 431
 432  error_cleanup:
 433         if (cdesc) free_cdesc(cdesc);
 434         return -1;
 435 }
 436
 437
 438 static int setup_key_dl_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **kd, int alloc_flag)
 439 {
 440         struct cryptocop_dma_desc  *key_desc = alloc_cdesc(alloc_flag);
 441         struct strcop_meta_out     mo = {0};
 442
 443         DEBUG(printk("setup_key_dl_desc\n"));
 444
 445         if (!key_desc) {
 446                 DEBUG_API(printk("setup_key_dl_desc: failed descriptor allocation.\n"));
 447                 return -ENOMEM;
 448         }
 449
 450         /* Download key. */
 451         if ((tc->tctx->init.alg == cryptocop_alg_aes) && (tc->tcfg->flags & CRYPTOCOP_DECRYPT)) {
 452                 /* Precook the AES decrypt key. */
 453                 if (!tc->tctx->dec_key_set){
 454                         get_aes_decrypt_key(tc->tctx->dec_key, tc->tctx->init.key, tc->tctx->init.keylen);
 455                         tc->tctx->dec_key_set = 1;
 456                 }
 457                 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->dec_key);
 458                 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
 459         } else {
 460                 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->init.key);
 461                 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
 462         }
 463         /* Setup metadata. */
 464         mo.dlkey = 1;
 465         switch (tc->tctx->init.keylen) {
 466         case 64:
 467                 mo.decrypt = 0;
 468                 mo.hashmode = 0;
 469                 break;
 470         case 128:
 471                 mo.decrypt = 0;
 472                 mo.hashmode = 1;
 473                 break;
 474         case 192:
 475                 mo.decrypt = 1;
 476                 mo.hashmode = 0;
 477                 break;
 478         case 256:
 479                 mo.decrypt = 1;
 480                 mo.hashmode = 1;
 481                 break;
 482         default:
 483                 break;
 484         }
 485         mo.ciphsel = mo.hashsel = mo.csumsel = src_none;
 486         key_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
 487
 488         key_desc->dma_descr->out_eop = 1;
 489         key_desc->dma_descr->wait = 1;
 490         key_desc->dma_descr->intr = 0;
 491
 492         *kd = key_desc;
 493         return 0;
 494 }
 495
 496 static int setup_cipher_iv_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
 497 {
 498         struct cryptocop_dma_desc  *iv_desc = alloc_cdesc(alloc_flag);
 499         struct strcop_meta_out     mo = {0};
 500
 501         DEBUG(printk("setup_cipher_iv_desc\n"));
 502
 503         if (!iv_desc) {
 504                 DEBUG_API(printk("setup_cipher_iv_desc: failed CBC IV descriptor allocation.\n"));
 505                 return -ENOMEM;
 506         }
 507         /* Download IV. */
 508         iv_desc->dma_descr->buf = (char*)virt_to_phys(tc->tcfg->iv);
 509         iv_desc->dma_descr->after = iv_desc->dma_descr->buf + tc->blocklength;
 510
 511         /* Setup metadata. */
 512         mo.hashsel = mo.csumsel = src_none;
 513         mo.ciphsel = src_dma;
 514         mo.ciphconf = tc->ciph_conf;
 515         mo.cbcmode = tc->cbcmode;
 516
 517         iv_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
 518
 519         iv_desc->dma_descr->out_eop = 0;
 520         iv_desc->dma_descr->wait = 1;
 521         iv_desc->dma_descr->intr = 0;
 522
 523         *id = iv_desc;
 524         return 0;
 525 }
 526
 527 /* Map the ouput length of the transform to operation output starting on the inject index. */
 528 static int create_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
 529 {
 530         int                        err = 0;
 531         struct cryptocop_dma_desc  head = {0};
 532         struct cryptocop_dma_desc  *outdesc = &head;
 533         size_t                     iov_offset = 0;
 534         size_t                     out_ix = 0;
 535         int                        outiov_ix = 0;
 536         struct strcop_meta_in      mi = {0};
 537
 538         size_t                     out_length = tc->produced;
 539         int                        rem_length;
 540         int                        dlength;
 541
 542         assert(out_length != 0);
 543         if (((tc->produced + tc->tcfg->inject_ix) > operation->tfrm_op.outlen) || (tc->produced && (operation->tfrm_op.outlen == 0))) {
 544                 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
 545                 return -EINVAL;
 546         }
 547         /* Traverse the out iovec until the result inject index is reached. */
 548         while ((outiov_ix < operation->tfrm_op.outcount) && ((out_ix + operation->tfrm_op.outdata[outiov_ix].iov_len) <= tc->tcfg->inject_ix)){
 549                 out_ix += operation->tfrm_op.outdata[outiov_ix].iov_len;
 550                 outiov_ix++;
 551         }
 552         if (outiov_ix >= operation->tfrm_op.outcount){
 553                 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
 554                 return -EINVAL;
 555         }
 556         iov_offset = tc->tcfg->inject_ix - out_ix;
 557         mi.dmasel = tc->unit_no;
 558
 559         /* Setup the output descriptors. */
 560         while ((out_length > 0) && (outiov_ix < operation->tfrm_op.outcount)) {
 561                 outdesc->next = alloc_cdesc(alloc_flag);
 562                 if (!outdesc->next) {
 563                         DEBUG_API(printk("create_input_descriptors: alloc_cdesc\n"));
 564                         err = -ENOMEM;
 565                         goto error_cleanup;
 566                 }
 567                 outdesc = outdesc->next;
 568                 rem_length = operation->tfrm_op.outdata[outiov_ix].iov_len - iov_offset;
 569                 dlength = (out_length < rem_length) ? out_length : rem_length;
 570
 571                 DEBUG(printk("create_input_descriptors:\n"
 572                              "outiov_ix=%d, rem_length=%d, dlength=%d\n"
 573                              "iov_offset=%d, outdata[outiov_ix].iov_len=%d\n"
 574                              "outcount=%d, outiov_ix=%d\n",
 575                              outiov_ix, rem_length, dlength, iov_offset, operation->tfrm_op.outdata[outiov_ix].iov_len, operation->tfrm_op.outcount, outiov_ix));
 576
 577                 outdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.outdata[outiov_ix].iov_base + iov_offset);
 578                 outdesc->dma_descr->after = outdesc->dma_descr->buf + dlength;
 579                 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 580
 581                 out_length -= dlength;
 582                 iov_offset += dlength;
 583                 if (iov_offset >= operation->tfrm_op.outdata[outiov_ix].iov_len) {
 584                         iov_offset = 0;
 585                         ++outiov_ix;
 586                 }
 587         }
 588         if (out_length > 0){
 589                 DEBUG_API(printk("create_input_descriptors: not enough room for output, %d remained\n", out_length));
 590                 err = -EINVAL;
 591                 goto error_cleanup;
 592         }
 593         /* Set sync in last descriptor. */
 594         mi.sync = 1;
 595         outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 596
 597         *id = head.next;
 598         return 0;
 599
 600  error_cleanup:
 601         while (head.next) {
 602                 outdesc = head.next->next;
 603                 free_cdesc(head.next);
 604                 head.next = outdesc;
 605         }
 606         return err;
 607 }
 608
 609
 610 static int create_output_descriptors(struct cryptocop_operation *operation, int *iniov_ix, int *iniov_offset, size_t desc_len, struct cryptocop_dma_desc **current_out_cdesc, struct strcop_meta_out *meta_out, int alloc_flag)
 611 {
 612         while (desc_len != 0) {
 613                 struct cryptocop_dma_desc  *cdesc;
 614                 int                        rem_length = operation->tfrm_op.indata[*iniov_ix].iov_len - *iniov_offset;
 615                 int                        dlength = (desc_len < rem_length) ? desc_len : rem_length;
 616
 617                 cdesc = alloc_cdesc(alloc_flag);
 618                 if (!cdesc) {
 619                         DEBUG_API(printk("create_output_descriptors: alloc_cdesc\n"));
 620                         return -ENOMEM;
 621                 }
 622                 (*current_out_cdesc)->next = cdesc;
 623                 (*current_out_cdesc) = cdesc;
 624
 625                 cdesc->free_buf = NULL;
 626
 627                 cdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.indata[*iniov_ix].iov_base + *iniov_offset);
 628                 cdesc->dma_descr->after = cdesc->dma_descr->buf + dlength;
 629
 630                 desc_len -= dlength;
 631                 *iniov_offset += dlength;
 632                 assert(desc_len >= 0);
 633                 if (*iniov_offset >= operation->tfrm_op.indata[*iniov_ix].iov_len) {
 634                         *iniov_offset = 0;
 635                         ++(*iniov_ix);
 636                         if (*iniov_ix > operation->tfrm_op.incount) {
 637                                 DEBUG_API(printk("create_output_descriptors: not enough indata in operation."));
 638                                 return  -EINVAL;
 639                         }
 640                 }
 641                 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, (*meta_out));
 642         } /* while (desc_len != 0) */
 643         /* Last DMA descriptor gets a 'wait' bit to signal expected change in metadata. */
 644         (*current_out_cdesc)->dma_descr->wait = 1; /* This will set extraneous WAIT in some situations, e.g. when padding hashes and checksums. */
 645
 646         return 0;
 647 }
 648
 649
 650 static int append_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_dma_desc **current_in_cdesc, struct cryptocop_dma_desc **current_out_cdesc, struct cryptocop_tfrm_ctx *tc, int alloc_flag)
 651 {
 652         DEBUG(printk("append_input_descriptors, tc=0x%p, unit_no=%d\n", tc, tc->unit_no));
 653         if (tc->tcfg) {
 654                 int                        failed = 0;
 655                 struct cryptocop_dma_desc  *idescs = NULL;
 656                 DEBUG(printk("append_input_descriptors: pushing output, consumed %d produced %d bytes.\n", tc->consumed, tc->produced));
 657                 if (tc->pad_descs) {
 658                         DEBUG(printk("append_input_descriptors: append pad descriptors to DMA out list.\n"));
 659                         while (tc->pad_descs) {
 660                                 DEBUG(printk("append descriptor 0x%p\n", tc->pad_descs));
 661                                 (*current_out_cdesc)->next = tc->pad_descs;
 662                                 tc->pad_descs = tc->pad_descs->next;
 663                                 (*current_out_cdesc) = (*current_out_cdesc)->next;
 664                         }
 665                 }
 666
 667                 /* Setup and append output descriptors to DMA in list. */
 668                 if (tc->unit_no == src_dma){
 669                         /* mem2mem.  Setup DMA in descriptors to discard all input prior to the requested mem2mem data. */
 670                         struct strcop_meta_in mi = {.sync = 0, .dmasel = src_dma};
 671                         unsigned int start_ix = tc->start_ix;
 672                         while (start_ix){
 673                                 unsigned int desclen = start_ix < MEM2MEM_DISCARD_BUF_LENGTH ? start_ix : MEM2MEM_DISCARD_BUF_LENGTH;
 674                                 (*current_in_cdesc)->next = alloc_cdesc(alloc_flag);
 675                                 if (!(*current_in_cdesc)->next){
 676                                         DEBUG_API(printk("append_input_descriptors: alloc_cdesc mem2mem discard failed\n"));
 677                                         return -ENOMEM;
 678                                 }
 679                                 (*current_in_cdesc) = (*current_in_cdesc)->next;
 680                                 (*current_in_cdesc)->dma_descr->buf = (char*)virt_to_phys(mem2mem_discard_buf);
 681                                 (*current_in_cdesc)->dma_descr->after = (*current_in_cdesc)->dma_descr->buf + desclen;
 682                                 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 683                                 start_ix -= desclen;
 684                         }
 685                         mi.sync = 1;
 686                         (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 687                 }
 688
 689                 failed = create_input_descriptors(operation, tc, &idescs, alloc_flag);
 690                 if (failed){
 691                         DEBUG_API(printk("append_input_descriptors: output descriptor setup failed\n"));
 692                         return failed;
 693                 }
 694                 DEBUG(printk("append_input_descriptors: append output descriptors to DMA in list.\n"));
 695                 while (idescs) {
 696                         DEBUG(printk("append descriptor 0x%p\n", idescs));
 697                         (*current_in_cdesc)->next = idescs;
 698                         idescs = idescs->next;
 699                         (*current_in_cdesc) = (*current_in_cdesc)->next;
 700                 }
 701         }
 702         return 0;
 703 }
 704
 705
 706
 707 static int cryptocop_setup_dma_list(struct cryptocop_operation *operation, struct cryptocop_int_operation **int_op, int alloc_flag)
 708 {
 709         struct cryptocop_session *sess;
 710         struct cryptocop_transform_ctx *tctx;
 711
 712         struct cryptocop_tfrm_ctx digest_ctx = {
 713                 .previous_src = src_none,
 714                 .current_src = src_none,
 715                 .start_ix = 0,
 716                 .requires_padding = 1,
 717                 .strict_block_length = 0,
 718                 .hash_conf = 0,
 719                 .hash_mode = 0,
 720                 .ciph_conf = 0,
 721                 .cbcmode = 0,
 722                 .decrypt = 0,
 723                 .consumed = 0,
 724                 .produced = 0,
 725                 .pad_descs = NULL,
 726                 .active = 0,
 727                 .done = 0,
 728                 .prev_src = NULL,
 729                 .curr_src = NULL,
 730                 .tcfg = NULL};
 731         struct cryptocop_tfrm_ctx cipher_ctx = {
 732                 .previous_src = src_none,
 733                 .current_src = src_none,
 734                 .start_ix = 0,
 735                 .requires_padding = 0,
 736                 .strict_block_length = 1,
 737                 .hash_conf = 0,
 738                 .hash_mode = 0,
 739                 .ciph_conf = 0,
 740                 .cbcmode = 0,
 741                 .decrypt = 0,
 742                 .consumed = 0,
 743                 .produced = 0,
 744                 .pad_descs = NULL,
 745                 .active = 0,
 746                 .done = 0,
 747                 .prev_src = NULL,
 748                 .curr_src = NULL,
 749                 .tcfg = NULL};
 750         struct cryptocop_tfrm_ctx csum_ctx = {
 751                 .previous_src = src_none,
 752                 .current_src = src_none,
 753                 .start_ix = 0,
 754                 .blocklength = 2,
 755                 .requires_padding = 1,
 756                 .strict_block_length = 0,
 757                 .hash_conf = 0,
 758                 .hash_mode = 0,
 759                 .ciph_conf = 0,
 760                 .cbcmode = 0,
 761                 .decrypt = 0,
 762                 .consumed = 0,
 763                 .produced = 0,
 764                 .pad_descs = NULL,
 765                 .active = 0,
 766                 .done = 0,
 767                 .tcfg = NULL,
 768                 .prev_src = NULL,
 769                 .curr_src = NULL,
 770                 .unit_no = src_csum};
 771         struct cryptocop_tfrm_cfg *tcfg = operation->tfrm_op.tfrm_cfg;
 772
 773         unsigned int indata_ix = 0;
 774
 775         /* iovec accounting. */
 776         int iniov_ix = 0;
 777         int iniov_offset = 0;
 778
 779         /* Operation descriptor cfg traversal pointer. */
 780         struct cryptocop_desc *odsc;
 781
 782         int failed = 0;
 783         /* List heads for allocated descriptors. */
 784         struct cryptocop_dma_desc out_cdesc_head = {0};
 785         struct cryptocop_dma_desc in_cdesc_head = {0};
 786
 787         struct cryptocop_dma_desc *current_out_cdesc = &out_cdesc_head;
 788         struct cryptocop_dma_desc *current_in_cdesc = &in_cdesc_head;
 789
 790         struct cryptocop_tfrm_ctx *output_tc = NULL;
 791         void                      *iop_alloc_ptr;
 792
 793         assert(operation != NULL);
 794         assert(int_op != NULL);
 795
 796         DEBUG(printk("cryptocop_setup_dma_list: start\n"));
 797         DEBUG(print_cryptocop_operation(operation));
 798
 799         sess = get_session(operation->sid);
 800         if (!sess) {
 801                 DEBUG_API(printk("cryptocop_setup_dma_list: no session found for operation.\n"));
 802                 failed = -EINVAL;
 803                 goto error_cleanup;
 804         }
 805         iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
 806         if (!iop_alloc_ptr) {
 807                 DEBUG_API(printk("cryptocop_setup_dma_list:  kmalloc cryptocop_int_operation\n"));
 808                 failed = -ENOMEM;
 809                 goto error_cleanup;
 810         }
 811         (*int_op) = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
 812         DEBUG(memset((*int_op), 0xff, sizeof(struct cryptocop_int_operation)));
 813         (*int_op)->alloc_ptr = iop_alloc_ptr;
 814         DEBUG(printk("cryptocop_setup_dma_list: *int_op=0x%p, alloc_ptr=0x%p\n", *int_op, (*int_op)->alloc_ptr));
 815
 816         (*int_op)->sid = operation->sid;
 817         (*int_op)->cdesc_out = NULL;
 818         (*int_op)->cdesc_in = NULL;
 819         (*int_op)->tdes_mode = cryptocop_3des_ede;
 820         (*int_op)->csum_mode = cryptocop_csum_le;
 821         (*int_op)->ddesc_out = NULL;
 822         (*int_op)->ddesc_in = NULL;
 823
 824         /* Scan operation->tfrm_op.tfrm_cfg for bad configuration and set up the local contexts. */
 825         if (!tcfg) {
 826                 DEBUG_API(printk("cryptocop_setup_dma_list: no configured transforms in operation.\n"));
 827                 failed = -EINVAL;
 828                 goto error_cleanup;
 829         }
 830         while (tcfg) {
 831                 tctx = get_transform_ctx(sess, tcfg->tid);
 832                 if (!tctx) {
 833                         DEBUG_API(printk("cryptocop_setup_dma_list: no transform id %d in session.\n", tcfg->tid));
 834                         failed = -EINVAL;
 835                         goto error_cleanup;
 836                 }
 837                 if (tcfg->inject_ix > operation->tfrm_op.outlen){
 838                         DEBUG_API(printk("cryptocop_setup_dma_list: transform id %d inject_ix (%d) > operation->tfrm_op.outlen(%d)", tcfg->tid, tcfg->inject_ix, operation->tfrm_op.outlen));
 839                         failed = -EINVAL;
 840                         goto error_cleanup;
 841                 }
 842                 switch (tctx->init.alg){
 843                 case cryptocop_alg_mem2mem:
 844                         if (cipher_ctx.tcfg != NULL){
 845                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
 846                                 failed = -EINVAL;
 847                                 goto error_cleanup;
 848                         }
 849                         /* mem2mem is handled as a NULL cipher. */
 850                         cipher_ctx.cbcmode = 0;
 851                         cipher_ctx.decrypt = 0;
 852                         cipher_ctx.blocklength = 1;
 853                         cipher_ctx.ciph_conf = 0;
 854                         cipher_ctx.unit_no = src_dma;
 855                         cipher_ctx.tcfg = tcfg;
 856                         cipher_ctx.tctx = tctx;
 857                         break;
 858                 case cryptocop_alg_des:
 859                 case cryptocop_alg_3des:
 860                 case cryptocop_alg_aes:
 861                         /* cipher */
 862                         if (cipher_ctx.tcfg != NULL){
 863                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
 864                                 failed = -EINVAL;
 865                                 goto error_cleanup;
 866                         }
 867                         cipher_ctx.tcfg = tcfg;
 868                         cipher_ctx.tctx = tctx;
 869                         if (cipher_ctx.tcfg->flags & CRYPTOCOP_DECRYPT){
 870                                 cipher_ctx.decrypt = 1;
 871                         }
 872                         switch (tctx->init.cipher_mode) {
 873                         case cryptocop_cipher_mode_ecb:
 874                                 cipher_ctx.cbcmode = 0;
 875                                 break;
 876                         case cryptocop_cipher_mode_cbc:
 877                                 cipher_ctx.cbcmode = 1;
 878                                 break;
 879                         default:
 880                                 DEBUG_API(printk("cryptocop_setup_dma_list: cipher_ctx, bad cipher mode==%d\n", tctx->init.cipher_mode));
 881                                 failed = -EINVAL;
 882                                 goto error_cleanup;
 883                         }
 884                         DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx, set CBC mode==%d\n", cipher_ctx.cbcmode));
 885                         switch (tctx->init.alg){
 886                         case cryptocop_alg_des:
 887                                 cipher_ctx.ciph_conf = 0;
 888                                 cipher_ctx.unit_no = src_des;
 889                                 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
 890                                 break;
 891                         case cryptocop_alg_3des:
 892                                 cipher_ctx.ciph_conf = 1;
 893                                 cipher_ctx.unit_no = src_des;
 894                                 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
 895                                 break;
 896                         case cryptocop_alg_aes:
 897                                 cipher_ctx.ciph_conf = 2;
 898                                 cipher_ctx.unit_no = src_aes;
 899                                 cipher_ctx.blocklength = AES_BLOCK_LENGTH;
 900                                 break;
 901                         default:
 902                                 panic("cryptocop_setup_dma_list: impossible algorithm %d\n", tctx->init.alg);
 903                         }
 904                         (*int_op)->tdes_mode = tctx->init.tdes_mode;
 905                         break;
 906                 case cryptocop_alg_md5:
 907                 case cryptocop_alg_sha1:
 908                         /* digest */
 909                         if (digest_ctx.tcfg != NULL){
 910                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple digests in operation.\n"));
 911                                 failed = -EINVAL;
 912                                 goto error_cleanup;
 913                         }
 914                         digest_ctx.tcfg = tcfg;
 915                         digest_ctx.tctx = tctx;
 916                         digest_ctx.hash_mode = 0; /* Don't use explicit IV in this API. */
 917                         switch (tctx->init.alg){
 918                         case cryptocop_alg_md5:
 919                                 digest_ctx.blocklength = MD5_BLOCK_LENGTH;
 920                                 digest_ctx.unit_no = src_md5;
 921                                 digest_ctx.hash_conf = 1; /* 1 => MD-5 */
 922                                 break;
 923                         case cryptocop_alg_sha1:
 924                                 digest_ctx.blocklength = SHA1_BLOCK_LENGTH;
 925                                 digest_ctx.unit_no = src_sha1;
 926                                 digest_ctx.hash_conf = 0; /* 0 => SHA-1 */
 927                                 break;
 928                         default:
 929                                 panic("cryptocop_setup_dma_list: impossible digest algorithm\n");
 930                         }
 931                         break;
 932                 case cryptocop_alg_csum:
 933                         /* digest */
 934                         if (csum_ctx.tcfg != NULL){
 935                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple checksums in operation.\n"));
 936                                 failed = -EINVAL;
 937                                 goto error_cleanup;
 938                         }
 939                         (*int_op)->csum_mode = tctx->init.csum_mode;
 940                         csum_ctx.tcfg = tcfg;
 941                         csum_ctx.tctx = tctx;
 942                         break;
 943                 default:
 944                         /* no algorithm. */
 945                         DEBUG_API(printk("cryptocop_setup_dma_list: invalid algorithm %d specified in tfrm %d.\n", tctx->init.alg, tcfg->tid));
 946                         failed = -EINVAL;
 947                         goto error_cleanup;
 948                 }
 949                 tcfg = tcfg->next;
 950         }
 951         /* Download key if a cipher is used. */
 952         if (cipher_ctx.tcfg && (cipher_ctx.tctx->init.alg != cryptocop_alg_mem2mem)){
 953                 struct cryptocop_dma_desc  *key_desc = NULL;
 954
 955                 failed = setup_key_dl_desc(&cipher_ctx, &key_desc, alloc_flag);
 956                 if (failed) {
 957                         DEBUG_API(printk("cryptocop_setup_dma_list: setup key dl\n"));
 958                         goto error_cleanup;
 959                 }
 960                 current_out_cdesc->next = key_desc;
 961                 current_out_cdesc = key_desc;
 962                 indata_ix += (unsigned int)(key_desc->dma_descr->after - key_desc->dma_descr->buf);
 963
 964                 /* Download explicit IV if a cipher is used and CBC mode and explicit IV selected. */
 965                 if ((cipher_ctx.tctx->init.cipher_mode == cryptocop_cipher_mode_cbc) && (cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV)) {
 966                         struct cryptocop_dma_desc  *iv_desc = NULL;
 967
 968                         DEBUG(printk("cryptocop_setup_dma_list: setup cipher CBC IV descriptor.\n"));
 969
 970                         failed = setup_cipher_iv_desc(&cipher_ctx, &iv_desc, alloc_flag);
 971                         if (failed) {
 972                                 DEBUG_API(printk("cryptocop_setup_dma_list: CBC IV descriptor.\n"));
 973                                 goto error_cleanup;
 974                         }
 975                         current_out_cdesc->next = iv_desc;
 976                         current_out_cdesc = iv_desc;
 977                         indata_ix += (unsigned int)(iv_desc->dma_descr->after - iv_desc->dma_descr->buf);
 978                 }
 979         }
 980
 981         /* Process descriptors. */
 982         odsc = operation->tfrm_op.desc;
 983         while (odsc) {
 984                 struct cryptocop_desc_cfg   *dcfg = odsc->cfg;
 985                 struct strcop_meta_out      meta_out = {0};
 986                 size_t                      desc_len = odsc->length;
 987                 int                         active_count, eop_needed_count;
 988
 989                 output_tc = NULL;
 990
 991                 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor\n"));
 992
 993                 while (dcfg) {
 994                         struct cryptocop_tfrm_ctx  *tc = NULL;
 995
 996                         DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor configuration.\n"));
 997                         /* Get the local context for the transform and mark it as the output unit if it produces output. */
 998                         if (digest_ctx.tcfg && (digest_ctx.tcfg->tid == dcfg->tid)){
 999                                 tc = &digest_ctx;
1000                         } else if (cipher_ctx.tcfg && (cipher_ctx.tcfg->tid == dcfg->tid)){
1001                                 tc = &cipher_ctx;
1002                         } else if (csum_ctx.tcfg && (csum_ctx.tcfg->tid == dcfg->tid)){
1003                                 tc = &csum_ctx;
1004                         }
1005                         if (!tc) {
1006                                 DEBUG_API(printk("cryptocop_setup_dma_list: invalid transform %d specified in descriptor.\n", dcfg->tid));
1007                                 failed = -EINVAL;
1008                                 goto error_cleanup;
1009                         }
1010                         if (tc->done) {
1011                                 DEBUG_API(printk("cryptocop_setup_dma_list: completed transform %d reused.\n", dcfg->tid));
1012                                 failed = -EINVAL;
1013                                 goto error_cleanup;
1014                         }
1015                         if (!tc->active) {
1016                                 tc->start_ix = indata_ix;
1017                                 tc->active = 1;
1018                         }
1019
1020                         tc->previous_src = tc->current_src;
1021                         tc->prev_src = tc->curr_src;
1022                         /* Map source unit id to DMA source config. */
1023                         switch (dcfg->src){
1024                         case cryptocop_source_dma:
1025                                 tc->current_src = src_dma;
1026                                 break;
1027                         case cryptocop_source_des:
1028                                 tc->current_src = src_des;
1029                                 break;
1030                         case cryptocop_source_3des:
1031                                 tc->current_src = src_des;
1032                                 break;
1033                         case cryptocop_source_aes:
1034                                 tc->current_src = src_aes;
1035                                 break;
1036                         case cryptocop_source_md5:
1037                         case cryptocop_source_sha1:
1038                         case cryptocop_source_csum:
1039                         case cryptocop_source_none:
1040                         default:
1041                                 /* We do not allow using accumulating style units (SHA-1, MD5, checksum) as sources to other units.
1042                                  */
1043                                 DEBUG_API(printk("cryptocop_setup_dma_list: bad unit source configured %d.\n", dcfg->src));
1044                                 failed = -EINVAL;
1045                                 goto error_cleanup;
1046                         }
1047                         if (tc->current_src != src_dma) {
1048                                 /* Find the unit we are sourcing from. */
1049                                 if (digest_ctx.unit_no == tc->current_src){
1050                                         tc->curr_src = &digest_ctx;
1051                                 } else if (cipher_ctx.unit_no == tc->current_src){
1052                                         tc->curr_src = &cipher_ctx;
1053                                 } else if (csum_ctx.unit_no == tc->current_src){
1054                                         tc->curr_src = &csum_ctx;
1055                                 }
1056                                 if ((tc->curr_src == tc) && (tc->unit_no != src_dma)){
1057                                         DEBUG_API(printk("cryptocop_setup_dma_list: unit %d configured to source from itself.\n", tc->unit_no));
1058                                         failed = -EINVAL;
1059                                         goto error_cleanup;
1060                                 }
1061                         } else {
1062                                 tc->curr_src = NULL;
1063                         }
1064
1065                         /* Detect source switch. */
1066                         DEBUG(printk("cryptocop_setup_dma_list: tc->active=%d tc->unit_no=%d tc->current_src=%d tc->previous_src=%d, tc->curr_src=0x%p, tc->prev_srv=0x%p\n", tc->active, tc->unit_no, tc->current_src, tc->previous_src, tc->curr_src, tc->prev_src));
1067                         if (tc->active && (tc->current_src != tc->previous_src)) {
1068                                 /* Only allow source switch when both the old source unit and the new one have
1069                                  * no pending data to process (i.e. the consumed length must be a multiple of the
1070                                  * transform blocklength). */
1071                                 /* Note: if the src == NULL we are actually sourcing from DMA out. */
1072                                 if (((tc->prev_src != NULL) && (tc->prev_src->consumed % tc->prev_src->blocklength)) ||
1073                                     ((tc->curr_src != NULL) && (tc->curr_src->consumed % tc->curr_src->blocklength)))
1074                                 {
1075                                         DEBUG_API(printk("cryptocop_setup_dma_list: can only disconnect from or connect to a unit on a multiple of the blocklength, old: cons=%d, prod=%d, block=%d, new: cons=%d prod=%d, block=%d.\n", tc->prev_src ? tc->prev_src->consumed : INT_MIN, tc->prev_src ? tc->prev_src->produced : INT_MIN, tc->prev_src ? tc->prev_src->blocklength : INT_MIN, tc->curr_src ? tc->curr_src->consumed : INT_MIN, tc->curr_src ? tc->curr_src->produced : INT_MIN, tc->curr_src ? tc->curr_src->blocklength : INT_MIN));
1076                                         failed = -EINVAL;
1077                                         goto error_cleanup;
1078                                 }
1079                         }
1080                         /* Detect unit deactivation. */
1081                         if (dcfg->last) {
1082                                 /* Length check of this is handled below. */
1083                                 tc->done = 1;
1084                         }
1085                         dcfg = dcfg->next;
1086                 } /* while (dcfg) */
1087                 DEBUG(printk("cryptocop_setup_dma_list: parsing operation descriptor configuration complete.\n"));
1088
1089                 if (cipher_ctx.active && (cipher_ctx.curr_src != NULL) && !cipher_ctx.curr_src->active){
1090                         DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", cipher_ctx.curr_src->unit_no));
1091                         failed = -EINVAL;
1092                         goto error_cleanup;
1093                 }
1094                 if (digest_ctx.active && (digest_ctx.curr_src != NULL) && !digest_ctx.curr_src->active){
1095                         DEBUG_API(printk("cryptocop_setup_dma_list: digest source from inactive unit %d\n", digest_ctx.curr_src->unit_no));
1096                         failed = -EINVAL;
1097                         goto error_cleanup;
1098                 }
1099                 if (csum_ctx.active && (csum_ctx.curr_src != NULL) && !csum_ctx.curr_src->active){
1100                         DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", csum_ctx.curr_src->unit_no));
1101                         failed = -EINVAL;
1102                         goto error_cleanup;
1103                 }
1104
1105                 /* Update consumed and produced lengths.
1106
1107                    The consumed length accounting here is actually cheating.  If a unit source from DMA (or any
1108                    other unit that process data in blocks of one octet) it is correct, but if it source from a
1109                    block processing unit, i.e. a cipher, it will be temporarily incorrect at some times.  However
1110                    since it is only allowed--by the HW--to change source to or from a block processing unit at times where that
1111                    unit has processed an exact multiple of its block length the end result will be correct.
1112                    Beware that if the source change restriction change this code will need to be (much) reworked.
1113                 */
1114                 DEBUG(printk("cryptocop_setup_dma_list: desc->length=%d, desc_len=%d.\n", odsc->length, desc_len));
1115
1116                 if (csum_ctx.active) {
1117                         csum_ctx.consumed += desc_len;
1118                         if (csum_ctx.done) {
1119                                 csum_ctx.produced = 2;
1120                         }
1121                         DEBUG(printk("cryptocop_setup_dma_list: csum_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", csum_ctx.consumed, csum_ctx.produced, csum_ctx.blocklength));
1122                 }
1123                 if (digest_ctx.active) {
1124                         digest_ctx.consumed += desc_len;
1125                         if (digest_ctx.done) {
1126                                 if (digest_ctx.unit_no == src_md5) {
1127                                         digest_ctx.produced = MD5_STATE_LENGTH;
1128                                 } else {
1129                                         digest_ctx.produced = SHA1_STATE_LENGTH;
1130                                 }
1131                         }
1132                         DEBUG(printk("cryptocop_setup_dma_list: digest_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", digest_ctx.consumed, digest_ctx.produced, digest_ctx.blocklength));
1133                 }
1134                 if (cipher_ctx.active) {
1135                         /* Ciphers are allowed only to source from DMA out.  That is filtered above. */
1136                         assert(cipher_ctx.current_src == src_dma);
1137                         cipher_ctx.consumed += desc_len;
1138                         cipher_ctx.produced = cipher_ctx.blocklength * (cipher_ctx.consumed / cipher_ctx.blocklength);
1139                         if (cipher_ctx.cbcmode && !(cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV) && cipher_ctx.produced){
1140                                 cipher_ctx.produced -= cipher_ctx.blocklength; /* Compensate for CBC iv. */
1141                         }
1142                         DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", cipher_ctx.consumed, cipher_ctx.produced, cipher_ctx.blocklength));
1143                 }
1144
1145                 /* Setup the DMA out descriptors. */
1146                 /* Configure the metadata. */
1147                 active_count = 0;
1148                 eop_needed_count = 0;
1149                 if (cipher_ctx.active) {
1150                         ++active_count;
1151                         if (cipher_ctx.unit_no == src_dma){
1152                                 /* mem2mem */
1153                                 meta_out.ciphsel = src_none;
1154                         } else {
1155                                 meta_out.ciphsel = cipher_ctx.current_src;
1156                         }
1157                         meta_out.ciphconf = cipher_ctx.ciph_conf;
1158                         meta_out.cbcmode = cipher_ctx.cbcmode;
1159                         meta_out.decrypt = cipher_ctx.decrypt;
1160                         DEBUG(printk("set ciphsel=%d ciphconf=%d cbcmode=%d decrypt=%d\n", meta_out.ciphsel, meta_out.ciphconf, meta_out.cbcmode, meta_out.decrypt));
1161                         if (cipher_ctx.done) ++eop_needed_count;
1162                 } else {
1163                         meta_out.ciphsel = src_none;
1164                 }
1165
1166                 if (digest_ctx.active) {
1167                         ++active_count;
1168                         meta_out.hashsel = digest_ctx.current_src;
1169                         meta_out.hashconf = digest_ctx.hash_conf;
1170                         meta_out.hashmode = 0; /* Explicit mode is not used here. */
1171                         DEBUG(printk("set hashsel=%d hashconf=%d hashmode=%d\n", meta_out.hashsel, meta_out.hashconf, meta_out.hashmode));
1172                         if (digest_ctx.done) {
1173                                 assert(digest_ctx.pad_descs == NULL);
1174                                 failed = create_pad_descriptor(&digest_ctx, &digest_ctx.pad_descs, alloc_flag);
1175                                 if (failed) {
1176                                         DEBUG_API(printk("cryptocop_setup_dma_list: failed digest pad creation.\n"));
1177                                         goto error_cleanup;
1178                                 }
1179                         }
1180                 } else {
1181                         meta_out.hashsel = src_none;
1182                 }
1183
1184                 if (csum_ctx.active) {
1185                         ++active_count;
1186                         meta_out.csumsel = csum_ctx.current_src;
1187                         if (csum_ctx.done) {
1188                                 assert(csum_ctx.pad_descs == NULL);
1189                                 failed = create_pad_descriptor(&csum_ctx, &csum_ctx.pad_descs, alloc_flag);
1190                                 if (failed) {
1191                                         DEBUG_API(printk("cryptocop_setup_dma_list: failed csum pad creation.\n"));
1192                                         goto error_cleanup;
1193                                 }
1194                         }
1195                 } else {
1196                         meta_out.csumsel = src_none;
1197                 }
1198                 DEBUG(printk("cryptocop_setup_dma_list: %d eop needed, %d active units\n", eop_needed_count, active_count));
1199                 /* Setup DMA out descriptors for the indata. */
1200                 failed = create_output_descriptors(operation, &iniov_ix, &iniov_offset, desc_len, &current_out_cdesc, &meta_out, alloc_flag);
1201                 if (failed) {
1202                         DEBUG_API(printk("cryptocop_setup_dma_list: create_output_descriptors %d\n", failed));
1203                         goto error_cleanup;
1204                 }
1205                 /* Setup out EOP.  If there are active units that are not done here they cannot get an EOP
1206                  * so we ust setup a zero length descriptor to DMA to signal EOP only to done units.
1207                  * If there is a pad descriptor EOP for the padded unit will be EOPed by it.
1208                  */
1209                 assert(active_count >= eop_needed_count);
1210                 assert((eop_needed_count == 0) || (eop_needed_count == 1));
1211                 if (eop_needed_count) {
1212                         /* This means that the bulk operation (cipeher/m2m) is terminated. */
1213                         if (active_count > 1) {
1214                                 /* Use zero length EOP descriptor. */
1215                                 struct cryptocop_dma_desc *ed = alloc_cdesc(alloc_flag);
1216                                 struct strcop_meta_out    ed_mo = {0};
1217                                 if (!ed) {
1218                                         DEBUG_API(printk("cryptocop_setup_dma_list: alloc EOP descriptor for cipher\n"));
1219                                         failed = -ENOMEM;
1220                                         goto error_cleanup;
1221                                 }
1222
1223                                 assert(cipher_ctx.active && cipher_ctx.done);
1224
1225                                 if (cipher_ctx.unit_no == src_dma){
1226                                         /* mem2mem */
1227                                         ed_mo.ciphsel = src_none;
1228                                 } else {
1229                                         ed_mo.ciphsel = cipher_ctx.current_src;
1230                                 }
1231                                 ed_mo.ciphconf = cipher_ctx.ciph_conf;
1232                                 ed_mo.cbcmode = cipher_ctx.cbcmode;
1233                                 ed_mo.decrypt = cipher_ctx.decrypt;
1234
1235                                 ed->free_buf = NULL;
1236                                 ed->dma_descr->wait = 1;
1237                                 ed->dma_descr->out_eop = 1;
1238
1239                                 ed->dma_descr->buf = (char*)virt_to_phys(&ed); /* Use any valid physical address for zero length descriptor. */
1240                                 ed->dma_descr->after = ed->dma_descr->buf;
1241                                 ed->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, ed_mo);
1242                                 current_out_cdesc->next = ed;
1243                                 current_out_cdesc = ed;
1244                         } else {
1245                                 /* Set EOP in the current out descriptor since the only active module is
1246                                  * the one needing the EOP. */
1247
1248                                 current_out_cdesc->dma_descr->out_eop = 1;
1249                         }
1250                 }
1251
1252                 if (cipher_ctx.done && cipher_ctx.active) cipher_ctx.active = 0;
1253                 if (digest_ctx.done && digest_ctx.active) digest_ctx.active = 0;
1254                 if (csum_ctx.done && csum_ctx.active) csum_ctx.active = 0;
1255                 indata_ix += odsc->length;
1256                 odsc = odsc->next;
1257         } /* while (odsc) */ /* Process descriptors. */
1258         DEBUG(printk("cryptocop_setup_dma_list: done parsing operation descriptors\n"));
1259         if (cipher_ctx.tcfg && (cipher_ctx.active || !cipher_ctx.done)){
1260                 DEBUG_API(printk("cryptocop_setup_dma_list: cipher operation not terminated.\n"));
1261                 failed = -EINVAL;
1262                 goto error_cleanup;
1263         }
1264         if (digest_ctx.tcfg && (digest_ctx.active || !digest_ctx.done)){
1265                 DEBUG_API(printk("cryptocop_setup_dma_list: digest operation not terminated.\n"));
1266                 failed = -EINVAL;
1267                 goto error_cleanup;
1268         }
1269         if (csum_ctx.tcfg && (csum_ctx.active || !csum_ctx.done)){
1270                 DEBUG_API(printk("cryptocop_setup_dma_list: csum operation not terminated.\n"));
1271                 failed = -EINVAL;
1272                 goto error_cleanup;
1273         }
1274
1275         failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &cipher_ctx, alloc_flag);
1276         if (failed){
1277                 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1278                 goto error_cleanup;
1279         }
1280         failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &digest_ctx, alloc_flag);
1281         if (failed){
1282                 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1283                 goto error_cleanup;
1284         }
1285         failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &csum_ctx, alloc_flag);
1286         if (failed){
1287                 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1288                 goto error_cleanup;
1289         }
1290
1291         DEBUG(printk("cryptocop_setup_dma_list: int_op=0x%p, *int_op=0x%p\n", int_op, *int_op));
1292         (*int_op)->cdesc_out = out_cdesc_head.next;
1293         (*int_op)->cdesc_in = in_cdesc_head.next;
1294         DEBUG(printk("cryptocop_setup_dma_list: out_cdesc_head=0x%p in_cdesc_head=0x%p\n", (*int_op)->cdesc_out, (*int_op)->cdesc_in));
1295
1296         setup_descr_chain(out_cdesc_head.next);
1297         setup_descr_chain(in_cdesc_head.next);
1298
1299         /* Last but not least: mark the last DMA in descriptor for a INTR and EOL and the the
1300          * last DMA out descriptor for EOL.
1301          */
1302         current_in_cdesc->dma_descr->intr = 1;
1303         current_in_cdesc->dma_descr->eol = 1;
1304         current_out_cdesc->dma_descr->eol = 1;
1305
1306         /* Setup DMA contexts. */
1307         (*int_op)->ctx_out.next = NULL;
1308         (*int_op)->ctx_out.eol = 1;
1309         (*int_op)->ctx_out.intr = 0;
1310         (*int_op)->ctx_out.store_mode = 0;
1311         (*int_op)->ctx_out.en = 0;
1312         (*int_op)->ctx_out.dis = 0;
1313         (*int_op)->ctx_out.md0 = 0;
1314         (*int_op)->ctx_out.md1 = 0;
1315         (*int_op)->ctx_out.md2 = 0;
1316         (*int_op)->ctx_out.md3 = 0;
1317         (*int_op)->ctx_out.md4 = 0;
1318         (*int_op)->ctx_out.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_out->dma_descr);
1319         (*int_op)->ctx_out.saved_data_buf = (*int_op)->cdesc_out->dma_descr->buf; /* Already physical address. */
1320
1321         (*int_op)->ctx_in.next = NULL;
1322         (*int_op)->ctx_in.eol = 1;
1323         (*int_op)->ctx_in.intr = 0;
1324         (*int_op)->ctx_in.store_mode = 0;
1325         (*int_op)->ctx_in.en = 0;
1326         (*int_op)->ctx_in.dis = 0;
1327         (*int_op)->ctx_in.md0 = 0;
1328         (*int_op)->ctx_in.md1 = 0;
1329         (*int_op)->ctx_in.md2 = 0;
1330         (*int_op)->ctx_in.md3 = 0;
1331         (*int_op)->ctx_in.md4 = 0;
1332
1333         (*int_op)->ctx_in.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_in->dma_descr);
1334         (*int_op)->ctx_in.saved_data_buf = (*int_op)->cdesc_in->dma_descr->buf; /* Already physical address. */
1335
1336         DEBUG(printk("cryptocop_setup_dma_list: done\n"));
1337         return 0;
1338
1339 error_cleanup:
1340         {
1341                 /* Free all allocated resources. */
1342                 struct cryptocop_dma_desc *tmp_cdesc;
1343                 while (digest_ctx.pad_descs){
1344                         tmp_cdesc = digest_ctx.pad_descs->next;
1345                         free_cdesc(digest_ctx.pad_descs);
1346                         digest_ctx.pad_descs = tmp_cdesc;
1347                 }
1348                 while (csum_ctx.pad_descs){
1349                         tmp_cdesc = csum_ctx.pad_descs->next;
1350                         free_cdesc(csum_ctx.pad_descs);
1351                         csum_ctx.pad_descs = tmp_cdesc;
1352                 }
1353                 assert(cipher_ctx.pad_descs == NULL); /* The ciphers are never padded. */
1354
1355                 if (*int_op != NULL) delete_internal_operation(*int_op);
1356         }
1357         DEBUG_API(printk("cryptocop_setup_dma_list: done with error %d\n", failed));
1358         return failed;
1359 }
1360
1361
1362 static void delete_internal_operation(struct cryptocop_int_operation *iop)
1363 {
1364         void                      *ptr = iop->alloc_ptr;
1365         struct cryptocop_dma_desc *cd = iop->cdesc_out;
1366         struct cryptocop_dma_desc *next;
1367
1368         DEBUG(printk("delete_internal_operation: iop=0x%p, alloc_ptr=0x%p\n", iop, ptr));
1369
1370         while (cd) {
1371                 next = cd->next;
1372                 free_cdesc(cd);
1373                 cd = next;
1374         }
1375         cd = iop->cdesc_in;
1376         while (cd) {
1377                 next = cd->next;
1378                 free_cdesc(cd);
1379                 cd = next;
1380         }
1381         kfree(ptr);
1382 }
1383
1384 #define MD5_MIN_PAD_LENGTH (9)
1385 #define MD5_PAD_LENGTH_FIELD_LENGTH (8)
1386
1387 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1388 {
1389         size_t                  padlen = MD5_BLOCK_LENGTH - (hashed_length % MD5_BLOCK_LENGTH);
1390         unsigned char           *p;
1391         int                     i;
1392         unsigned long long int  bit_length = hashed_length << 3;
1393
1394         if (padlen < MD5_MIN_PAD_LENGTH) padlen += MD5_BLOCK_LENGTH;
1395
1396         p = kmalloc(padlen, alloc_flag);
1397         if (!pad) return -ENOMEM;
1398
1399         *p = 0x80;
1400         memset(p+1, 0, padlen - 1);
1401
1402         DEBUG(printk("create_md5_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1403
1404         i = padlen - MD5_PAD_LENGTH_FIELD_LENGTH;
1405         while (bit_length != 0){
1406                 p[i++] = bit_length % 0x100;
1407                 bit_length >>= 8;
1408         }
1409
1410         *pad = (char*)p;
1411         *pad_length = padlen;
1412
1413         return 0;
1414 }
1415
1416 #define SHA1_MIN_PAD_LENGTH (9)
1417 #define SHA1_PAD_LENGTH_FIELD_LENGTH (8)
1418
1419 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1420 {
1421         size_t                  padlen = SHA1_BLOCK_LENGTH - (hashed_length % SHA1_BLOCK_LENGTH);
1422         unsigned char           *p;
1423         int                     i;
1424         unsigned long long int  bit_length = hashed_length << 3;
1425
1426         if (padlen < SHA1_MIN_PAD_LENGTH) padlen += SHA1_BLOCK_LENGTH;
1427
1428         p = kmalloc(padlen, alloc_flag);
1429         if (!pad) return -ENOMEM;
1430
1431         *p = 0x80;
1432         memset(p+1, 0, padlen - 1);
1433
1434         DEBUG(printk("create_sha1_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1435
1436         i = padlen - 1;
1437         while (bit_length != 0){
1438                 p[i--] = bit_length % 0x100;
1439                 bit_length >>= 8;
1440         }
1441
1442         *pad = (char*)p;
1443         *pad_length = padlen;
1444
1445         return 0;
1446 }
1447
1448
1449 static int transform_ok(struct cryptocop_transform_init *tinit)
1450 {
1451         switch (tinit->alg){
1452         case cryptocop_alg_csum:
1453                 switch (tinit->csum_mode){
1454                 case cryptocop_csum_le:
1455                 case cryptocop_csum_be:
1456                         break;
1457                 default:
1458                         DEBUG_API(printk("transform_ok: Bad mode set for csum transform\n"));
1459                         return -EINVAL;
1460                 }
1461         case cryptocop_alg_mem2mem:
1462         case cryptocop_alg_md5:
1463         case cryptocop_alg_sha1:
1464                 if (tinit->keylen != 0) {
1465                         DEBUG_API(printk("transform_ok: non-zero keylength, %d, for a digest/csum algorithm\n", tinit->keylen));
1466                         return -EINVAL; /* This check is a bit strict. */
1467                 }
1468                 break;
1469         case cryptocop_alg_des:
1470                 if (tinit->keylen != 64) {
1471                         DEBUG_API(printk("transform_ok: keylen %d invalid for DES\n", tinit->keylen));
1472                         return -EINVAL;
1473                 }
1474                 break;
1475         case cryptocop_alg_3des:
1476                 if (tinit->keylen != 192) {
1477                         DEBUG_API(printk("transform_ok: keylen %d invalid for 3DES\n", tinit->keylen));
1478                         return -EINVAL;
1479                 }
1480                 break;
1481         case cryptocop_alg_aes:
1482                 if (tinit->keylen != 128 && tinit->keylen != 192 && tinit->keylen != 256) {
1483                         DEBUG_API(printk("transform_ok: keylen %d invalid for AES\n", tinit->keylen));
1484                         return -EINVAL;
1485                 }
1486                 break;
1487         case cryptocop_no_alg:
1488         default:
1489                 DEBUG_API(printk("transform_ok: no such algorithm %d\n", tinit->alg));
1490                 return -EINVAL;
1491         }
1492
1493         switch (tinit->alg){
1494         case cryptocop_alg_des:
1495         case cryptocop_alg_3des:
1496         case cryptocop_alg_aes:
1497                 if (tinit->cipher_mode != cryptocop_cipher_mode_ecb && tinit->cipher_mode != cryptocop_cipher_mode_cbc) return -EINVAL;
1498         default:
1499                  break;
1500         }
1501         return 0;
1502 }
1503
1504
1505 int cryptocop_new_session(cryptocop_session_id *sid, struct cryptocop_transform_init *tinit, int alloc_flag)
1506 {
1507         struct cryptocop_session         *sess;
1508         struct cryptocop_transform_init  *tfrm_in = tinit;
1509         struct cryptocop_transform_init  *tmp_in;
1510         int                              no_tfrms = 0;
1511         int                              i;
1512         unsigned long int                flags;
1513
1514         init_stream_coprocessor(); /* For safety if we are called early */
1515
1516         while (tfrm_in){
1517                 int err;
1518                 ++no_tfrms;
1519                 if ((err = transform_ok(tfrm_in))) {
1520                         DEBUG_API(printk("cryptocop_new_session, bad transform\n"));
1521                         return err;
1522                 }
1523                 tfrm_in = tfrm_in->next;
1524         }
1525         if (0 == no_tfrms) {
1526                 DEBUG_API(printk("cryptocop_new_session, no transforms specified\n"));
1527                 return -EINVAL;
1528         }
1529
1530         sess = kmalloc(sizeof(struct cryptocop_session), alloc_flag);
1531         if (!sess){
1532                 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_session\n"));
1533                 return -ENOMEM;
1534         }
1535
1536         sess->tfrm_ctx = kmalloc(no_tfrms * sizeof(struct cryptocop_transform_ctx), alloc_flag);
1537         if (!sess->tfrm_ctx) {
1538                 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_transform_ctx\n"));
1539                 kfree(sess);
1540                 return -ENOMEM;
1541         }
1542
1543         tfrm_in = tinit;
1544         for (i = 0; i < no_tfrms; i++){
1545                 tmp_in = tfrm_in->next;
1546                 while (tmp_in){
1547                         if (tmp_in->tid == tfrm_in->tid) {
1548                                 DEBUG_API(printk("cryptocop_new_session, duplicate transform ids\n"));
1549                                 kfree(sess->tfrm_ctx);
1550                                 kfree(sess);
1551                                 return -EINVAL;
1552                         }
1553                         tmp_in = tmp_in->next;
1554                 }
1555                 memcpy(&sess->tfrm_ctx[i].init, tfrm_in, sizeof(struct cryptocop_transform_init));
1556                 sess->tfrm_ctx[i].dec_key_set = 0;
1557                 sess->tfrm_ctx[i].next = &sess->tfrm_ctx[i] + 1;
1558
1559                 tfrm_in = tfrm_in->next;
1560         }
1561         sess->tfrm_ctx[i-1].next = NULL;
1562
1563         spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1564         sess->sid = next_sid;
1565         next_sid++;
1566         /* TODO If we are really paranoid we should do duplicate check to handle sid wraparound.
1567          *      OTOH 2^64 is a really large number of session. */
1568         if (next_sid == 0) next_sid = 1;
1569
1570         /* Prepend to session list. */
1571         sess->next = cryptocop_sessions;
1572         cryptocop_sessions = sess;
1573         spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1574         *sid = sess->sid;
1575         return 0;
1576 }
1577
1578
1579 int cryptocop_free_session(cryptocop_session_id sid)
1580 {
1581         struct cryptocop_transform_ctx    *tc;
1582         struct cryptocop_session          *sess = NULL;
1583         struct cryptocop_session          *psess = NULL;
1584         unsigned long int                 flags;
1585         int                               i;
1586         LIST_HEAD(remove_list);
1587         struct list_head                  *node, *tmp;
1588         struct cryptocop_prio_job         *pj;
1589
1590         DEBUG(printk("cryptocop_free_session: sid=%lld\n", sid));
1591
1592         spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1593         sess = cryptocop_sessions;
1594         while (sess && sess->sid != sid){
1595                 psess = sess;
1596                 sess = sess->next;
1597         }
1598         if (sess){
1599                 if (psess){
1600                         psess->next = sess->next;
1601                 } else {
1602                         cryptocop_sessions = sess->next;
1603                 }
1604         }
1605         spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1606
1607         if (!sess) return -EINVAL;
1608
1609         /* Remove queued jobs. */
1610         spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1611
1612         for (i = 0; i < cryptocop_prio_no_prios; i++){
1613                 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
1614                         list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
1615                                 pj = list_entry(node, struct cryptocop_prio_job, node);
1616                                 if (pj->oper->sid == sid) {
1617                                         list_move_tail(node, &remove_list);
1618                                 }
1619                         }
1620                 }
1621         }
1622         spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1623
1624         list_for_each_safe(node, tmp, &remove_list) {
1625                 list_del(node);
1626                 pj = list_entry(node, struct cryptocop_prio_job, node);
1627                 pj->oper->operation_status = -EAGAIN;  /* EAGAIN is not ideal for job/session terminated but it's the best choice I know of. */
1628                 DEBUG(printk("cryptocop_free_session: pj=0x%p, pj->oper=0x%p, pj->iop=0x%p\n", pj, pj->oper, pj->iop));
1629                 pj->oper->cb(pj->oper, pj->oper->cb_data);
1630                 delete_internal_operation(pj->iop);
1631                 kfree(pj);
1632         }
1633
1634         tc = sess->tfrm_ctx;
1635         /* Erase keying data. */
1636         while (tc){
1637                 DEBUG(printk("cryptocop_free_session: memset keys, tfrm id=%d\n", tc->init.tid));
1638                 memset(tc->init.key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1639                 memset(tc->dec_key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1640                 tc = tc->next;
1641         }
1642         kfree(sess->tfrm_ctx);
1643         kfree(sess);
1644
1645         return 0;
1646 }
1647
1648 static struct cryptocop_session *get_session(cryptocop_session_id sid)
1649 {
1650         struct cryptocop_session    *sess;
1651         unsigned long int           flags;
1652
1653         spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1654         sess = cryptocop_sessions;
1655         while (sess && (sess->sid != sid)){
1656                 sess = sess->next;
1657         }
1658         spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1659
1660         return sess;
1661 }
1662
1663 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid)
1664 {
1665         struct cryptocop_transform_ctx *tc = sess->tfrm_ctx;
1666
1667         DEBUG(printk("get_transform_ctx, sess=0x%p, tid=%d\n", sess, tid));
1668         assert(sess != NULL);
1669         while (tc && tc->init.tid != tid){
1670                 DEBUG(printk("tc=0x%p, tc->next=0x%p\n", tc, tc->next));
1671                 tc = tc->next;
1672         }
1673         DEBUG(printk("get_transform_ctx, returning tc=0x%p\n", tc));
1674         return tc;
1675 }
1676
1677
1678
1679 /* The AES s-transform matrix (s-box). */
1680 static const u8 aes_sbox[256] = {
1681         99,  124, 119, 123, 242, 107, 111, 197, 48,  1,   103, 43,  254, 215, 171, 118,
1682         202, 130, 201, 125, 250, 89,  71,  240, 173, 212, 162, 175, 156, 164, 114, 192,
1683         183, 253, 147, 38,  54,  63,  247, 204, 52,  165, 229, 241, 113, 216, 49,  21,
1684         4,   199, 35,  195, 24,  150, 5,   154, 7,   18,  128, 226, 235, 39,  178, 117,
1685         9,   131, 44,  26,  27,  110, 90,  160, 82,  59,  214, 179, 41,  227, 47,  132,
1686         83,  209, 0,   237, 32,  252, 177, 91,  106, 203, 190, 57,  74,  76,  88,  207,
1687         208, 239, 170, 251, 67,  77,  51,  133, 69,  249, 2,   127, 80,  60,  159, 168,
1688         81,  163, 64,  143, 146, 157, 56,  245, 188, 182, 218, 33,  16,  255, 243, 210,
1689         205, 12,  19,  236, 95,  151, 68,  23,  196, 167, 126, 61,  100, 93,  25,  115,
1690         96,  129, 79,  220, 34,  42,  144, 136, 70,  238, 184, 20,  222, 94,  11,  219,
1691         224, 50,  58,  10,  73,  6,   36,  92,  194, 211, 172, 98,  145, 149, 228, 121,
1692         231, 200, 55,  109, 141, 213, 78,  169, 108, 86,  244, 234, 101, 122, 174, 8,
1693         186, 120, 37,  46,  28,  166, 180, 198, 232, 221, 116, 31,  75,  189, 139, 138,
1694         112, 62,  181, 102, 72,  3,   246, 14,  97,  53,  87,  185, 134, 193, 29,  158,
1695         225, 248, 152, 17,  105, 217, 142, 148, 155, 30,  135, 233, 206, 85,  40,  223,
1696         140, 161, 137, 13,  191, 230, 66,  104, 65,  153, 45,  15,  176, 84,  187, 22
1697 };
1698
1699 /* AES has a 32 bit word round constants for each round in the
1700  * key schedule.  round_constant[i] is really Rcon[i+1] in FIPS187.
1701  */
1702 static u32 round_constant[11] = {
1703         0x01000000, 0x02000000, 0x04000000, 0x08000000,
1704         0x10000000, 0x20000000, 0x40000000, 0x80000000,
1705         0x1B000000, 0x36000000, 0x6C000000
1706 };
1707
1708 /* Apply the s-box to each of the four occtets in w. */
1709 static u32 aes_ks_subword(const u32 w)
1710 {
1711         u8 bytes[4];
1712
1713         *(u32*)(&bytes[0]) = w;
1714         bytes[0] = aes_sbox[bytes[0]];
1715         bytes[1] = aes_sbox[bytes[1]];
1716         bytes[2] = aes_sbox[bytes[2]];
1717         bytes[3] = aes_sbox[bytes[3]];
1718         return *(u32*)(&bytes[0]);
1719 }
1720
1721 /* The encrypt (forward) Rijndael key schedule algorithm pseudo code:
1722  * (Note that AES words are 32 bit long)
1723  *
1724  * KeyExpansion(byte key[4*Nk], word w[Nb*(Nr+1)], Nk){
1725  * word temp
1726  * i = 0
1727  * while (i < Nk) {
1728  *   w[i] = word(key[4*i, 4*i + 1, 4*i + 2, 4*i + 3])
1729  *   i = i + 1
1730  * }
1731  * i = Nk
1732  *
1733  * while (i < (Nb * (Nr + 1))) {
1734  *   temp = w[i - 1]
1735  *   if ((i mod Nk) == 0) {
1736  *     temp = SubWord(RotWord(temp)) xor Rcon[i/Nk]
1737  *   }
1738  *   else if ((Nk > 6) && ((i mod Nk) == 4)) {
1739  *     temp = SubWord(temp)
1740  *   }
1741  *   w[i] = w[i - Nk] xor temp
1742  * }
1743  * RotWord(t) does a 8 bit cyclic shift left on a 32 bit word.
1744  * SubWord(t) applies the AES s-box individually to each octet
1745  * in a 32 bit word.
1746  *
1747  * For AES Nk can have the values 4, 6, and 8 (corresponding to
1748  * values for Nr of 10, 12, and 14).  Nb is always 4.
1749  *
1750  * To construct w[i], w[i - 1] and w[i - Nk] must be
1751  * available.  Consequently we must keep a state of the last Nk words
1752  * to be able to create the last round keys.
1753  */
1754 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned  char *key, unsigned int keylength)
1755 {
1756         u32 temp;
1757         u32 w_ring[8]; /* nk is max 8, use elements 0..(nk - 1) as a ringbuffer */
1758         u8  w_last_ix;
1759         int i;
1760         u8  nr, nk;
1761
1762         switch (keylength){
1763         case 128:
1764                 nk = 4;
1765                 nr = 10;
1766                 break;
1767         case 192:
1768                 nk = 6;
1769                 nr = 12;
1770                 break;
1771         case 256:
1772                 nk = 8;
1773                 nr = 14;
1774                 break;
1775         default:
1776                 panic("stream co-processor: bad aes key length in get_aes_decrypt_key\n");
1777         };
1778
1779         /* Need to do host byte order correction here since key is byte oriented and the
1780          * kx algorithm is word (u32) oriented. */
1781         for (i = 0; i < nk; i+=1) {
1782                 w_ring[i] = be32_to_cpu(*(u32*)&key[4*i]);
1783         }
1784
1785         i = (int)nk;
1786         w_last_ix = i - 1;
1787         while (i < (4 * (nr + 2))) {
1788                 temp = w_ring[w_last_ix];
1789                 if (!(i % nk)) {
1790                         /* RotWord(temp) */
1791                         temp = (temp << 8) | (temp >> 24);
1792                         temp = aes_ks_subword(temp);
1793                         temp ^= round_constant[i/nk - 1];
1794                 } else if ((nk > 6) && ((i % nk) == 4)) {
1795                         temp = aes_ks_subword(temp);
1796                 }
1797                 w_last_ix = (w_last_ix + 1) % nk; /* This is the same as (i-Nk) mod Nk */
1798                 temp ^= w_ring[w_last_ix];
1799                 w_ring[w_last_ix] = temp;
1800
1801                 /* We need the round keys for round Nr+1 and Nr+2 (round key
1802                  * Nr+2 is the round key beyond the last one used when
1803                  * encrypting).  Rounds are numbered starting from 0, Nr=10
1804                  * implies 11 rounds are used in encryption/decryption.
1805                  */
1806                 if (i >= (4 * nr)) {
1807                         /* Need to do host byte order correction here, the key
1808                          * is byte oriented. */
1809                         *(u32*)dec_key = cpu_to_be32(temp);
1810                         dec_key += 4;
1811                 }
1812                 ++i;
1813         }
1814 }
1815
1816
1817 /**** Job/operation management. ****/
1818
1819 int cryptocop_job_queue_insert_csum(struct cryptocop_operation *operation)
1820 {
1821         return cryptocop_job_queue_insert(cryptocop_prio_kernel_csum, operation);
1822 }
1823
1824 int cryptocop_job_queue_insert_crypto(struct cryptocop_operation *operation)
1825 {
1826         return cryptocop_job_queue_insert(cryptocop_prio_kernel, operation);
1827 }
1828
1829 int cryptocop_job_queue_insert_user_job(struct cryptocop_operation *operation)
1830 {
1831         return cryptocop_job_queue_insert(cryptocop_prio_user, operation);
1832 }
1833
1834 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation)
1835 {
1836         int                           ret;
1837         struct cryptocop_prio_job     *pj = NULL;
1838         unsigned long int             flags;
1839
1840         DEBUG(printk("cryptocop_job_queue_insert(%d, 0x%p)\n", prio, operation));
1841
1842         if (!operation || !operation->cb){
1843                 DEBUG_API(printk("cryptocop_job_queue_insert oper=0x%p, NULL operation or callback\n", operation));
1844                 return -EINVAL;
1845         }
1846
1847         if ((ret = cryptocop_job_setup(&pj, operation)) != 0){
1848                 DEBUG_API(printk("cryptocop_job_queue_insert: job setup failed\n"));
1849                 return ret;
1850         }
1851         assert(pj != NULL);
1852
1853         spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1854         list_add_tail(&pj->node, &cryptocop_job_queues[prio].jobs);
1855         spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1856
1857         /* Make sure a job is running */
1858         cryptocop_start_job();
1859         return 0;
1860 }
1861
1862 static void cryptocop_do_tasklet(unsigned long unused);
1863 DECLARE_TASKLET (cryptocop_tasklet, cryptocop_do_tasklet, 0);
1864
1865 static void cryptocop_do_tasklet(unsigned long unused)
1866 {
1867         struct list_head             *node;
1868         struct cryptocop_prio_job    *pj = NULL;
1869         unsigned long                flags;
1870
1871         DEBUG(printk("cryptocop_do_tasklet: entering\n"));
1872
1873         do {
1874                 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
1875                 if (!list_empty(&cryptocop_completed_jobs)){
1876                         node = cryptocop_completed_jobs.next;
1877                         list_del(node);
1878                         pj = list_entry(node, struct cryptocop_prio_job, node);
1879                 } else {
1880                         pj = NULL;
1881                 }
1882                 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
1883                 if (pj) {
1884                         assert(pj->oper != NULL);
1885
1886                         /* Notify consumer of operation completeness. */
1887                         DEBUG(printk("cryptocop_do_tasklet: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
1888
1889                         pj->oper->operation_status = 0; /* Job is completed. */
1890                         pj->oper->cb(pj->oper, pj->oper->cb_data);
1891                         delete_internal_operation(pj->iop);
1892                         kfree(pj);
1893                 }
1894         } while (pj != NULL);
1895
1896         DEBUG(printk("cryptocop_do_tasklet: exiting\n"));
1897 }
1898
1899 static irqreturn_t
1900 dma_done_interrupt(int irq, void *dev_id)
1901 {
1902         struct cryptocop_prio_job *done_job;
1903         reg_dma_rw_ack_intr ack_intr = {
1904                 .data = 1,
1905         };
1906
1907         REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1908
1909         DEBUG(printk("cryptocop DMA done\n"));
1910
1911         spin_lock(&running_job_lock);
1912         if (cryptocop_running_job == NULL){
1913                 printk("stream co-processor got interrupt when not busy\n");
1914                 spin_unlock(&running_job_lock);
1915                 return IRQ_HANDLED;
1916         }
1917         done_job = cryptocop_running_job;
1918         cryptocop_running_job = NULL;
1919         spin_unlock(&running_job_lock);
1920
1921         /* Start processing a job. */
1922         if (!spin_trylock(&cryptocop_process_lock)){
1923                 DEBUG(printk("cryptocop irq handler, not starting a job\n"));
1924         } else {
1925                 cryptocop_start_job();
1926                 spin_unlock(&cryptocop_process_lock);
1927         }
1928
1929         done_job->oper->operation_status = 0; /* Job is completed. */
1930         if (done_job->oper->fast_callback){
1931                 /* This operation wants callback from interrupt. */
1932                 done_job->oper->cb(done_job->oper, done_job->oper->cb_data);
1933                 delete_internal_operation(done_job->iop);
1934                 kfree(done_job);
1935         } else {
1936                 spin_lock(&cryptocop_completed_jobs_lock);
1937                 list_add_tail(&(done_job->node), &cryptocop_completed_jobs);
1938                 spin_unlock(&cryptocop_completed_jobs_lock);
1939                 tasklet_schedule(&cryptocop_tasklet);
1940         }
1941
1942         DEBUG(printk("cryptocop leave irq handler\n"));
1943         return IRQ_HANDLED;
1944 }
1945
1946
1947 /* Setup interrupts and DMA channels. */
1948 static int init_cryptocop(void)
1949 {
1950         unsigned long          flags;
1951         reg_dma_rw_cfg         dma_cfg = {.en = 1};
1952         reg_dma_rw_intr_mask   intr_mask_in = {.data = regk_dma_yes}; /* Only want descriptor interrupts from the DMA in channel. */
1953         reg_dma_rw_ack_intr    ack_intr = {.data = 1,.in_eop = 1 };
1954         reg_strcop_rw_cfg      strcop_cfg = {
1955                 .ipend = regk_strcop_little,
1956                 .td1 = regk_strcop_e,
1957                 .td2 = regk_strcop_d,
1958                 .td3 = regk_strcop_e,
1959                 .ignore_sync = 0,
1960                 .en = 1
1961         };
1962
1963         if (request_irq(DMA_IRQ, dma_done_interrupt, 0,
1964                         "stream co-processor DMA", NULL))
1965                 panic("request_irq stream co-processor irq dma9");
1966
1967         (void)crisv32_request_dma(OUT_DMA, "strcop", DMA_PANIC_ON_ERROR,
1968                 0, dma_strp);
1969         (void)crisv32_request_dma(IN_DMA, "strcop", DMA_PANIC_ON_ERROR,
1970                 0, dma_strp);
1971
1972         local_irq_save(flags);
1973
1974         /* Reset and enable the cryptocop. */
1975         strcop_cfg.en = 0;
1976         REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1977         strcop_cfg.en = 1;
1978         REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1979
1980         /* Enable DMAs. */
1981         REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
1982         REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
1983
1984         /* Set up wordsize = 4 for DMAs. */
1985         DMA_WR_CMD(OUT_DMA_INST, regk_dma_set_w_size4);
1986         DMA_WR_CMD(IN_DMA_INST, regk_dma_set_w_size4);
1987
1988         /* Enable interrupts. */
1989         REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
1990
1991         /* Clear intr ack. */
1992         REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1993
1994         local_irq_restore(flags);
1995
1996         return 0;
1997 }
1998
1999 /* Free used cryptocop hw resources (interrupt and DMA channels). */
2000 static void release_cryptocop(void)
2001 {
2002         unsigned long          flags;
2003         reg_dma_rw_cfg         dma_cfg = {.en = 0};
2004         reg_dma_rw_intr_mask   intr_mask_in = {0};
2005         reg_dma_rw_ack_intr    ack_intr = {.data = 1,.in_eop = 1 };
2006
2007         local_irq_save(flags);
2008
2009         /* Clear intr ack. */
2010         REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
2011
2012         /* Disable DMAs. */
2013         REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
2014         REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
2015
2016         /* Disable interrupts. */
2017         REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
2018
2019         local_irq_restore(flags);
2020
2021         free_irq(DMA_IRQ, NULL);
2022
2023         (void)crisv32_free_dma(OUT_DMA);
2024         (void)crisv32_free_dma(IN_DMA);
2025 }
2026
2027
2028 /* Init job queue. */
2029 static int cryptocop_job_queue_init(void)
2030 {
2031         int i;
2032
2033         INIT_LIST_HEAD(&cryptocop_completed_jobs);
2034
2035         for (i = 0; i < cryptocop_prio_no_prios; i++){
2036                 cryptocop_job_queues[i].prio = (cryptocop_queue_priority)i;
2037                 INIT_LIST_HEAD(&cryptocop_job_queues[i].jobs);
2038         }
2039         return 0;
2040 }
2041
2042
2043 static void cryptocop_job_queue_close(void)
2044 {
2045         struct list_head               *node, *tmp;
2046         struct cryptocop_prio_job      *pj = NULL;
2047         unsigned long int              process_flags, flags;
2048         int                            i;
2049
2050         /* FIXME: This is as yet untested code. */
2051
2052         /* Stop strcop from getting an operation to process while we are closing the
2053            module. */
2054         spin_lock_irqsave(&cryptocop_process_lock, process_flags);
2055
2056         /* Empty the job queue. */
2057         for (i = 0; i < cryptocop_prio_no_prios; i++){
2058                 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
2059                         list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
2060                                 pj = list_entry(node, struct cryptocop_prio_job, node);
2061                                 list_del(node);
2062
2063                                 /* Call callback to notify consumer of job removal. */
2064                                 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2065                                 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2066                                 pj->oper->cb(pj->oper, pj->oper->cb_data);
2067
2068                                 delete_internal_operation(pj->iop);
2069                                 kfree(pj);
2070                         }
2071                 }
2072         }
2073         spin_unlock_irqrestore(&cryptocop_process_lock, process_flags);
2074
2075         /* Remove the running job, if any. */
2076         spin_lock_irqsave(&running_job_lock, flags);
2077         if (cryptocop_running_job){
2078                 reg_strcop_rw_cfg rw_cfg;
2079                 reg_dma_rw_cfg    dma_out_cfg, dma_in_cfg;
2080
2081                 /* Stop DMA. */
2082                 dma_out_cfg = REG_RD(dma, OUT_DMA_INST, rw_cfg);
2083                 dma_out_cfg.en = regk_dma_no;
2084                 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_out_cfg);
2085
2086                 dma_in_cfg = REG_RD(dma, IN_DMA_INST, rw_cfg);
2087                 dma_in_cfg.en = regk_dma_no;
2088                 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_in_cfg);
2089
2090                 /* Disble the cryptocop. */
2091                 rw_cfg = REG_RD(strcop, regi_strcop, rw_cfg);
2092                 rw_cfg.en = 0;
2093                 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2094
2095                 pj = cryptocop_running_job;
2096                 cryptocop_running_job = NULL;
2097
2098                 /* Call callback to notify consumer of job removal. */
2099                 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2100                 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2101                 pj->oper->cb(pj->oper, pj->oper->cb_data);
2102
2103                 delete_internal_operation(pj->iop);
2104                 kfree(pj);
2105         }
2106         spin_unlock_irqrestore(&running_job_lock, flags);
2107
2108         /* Remove completed jobs, if any. */
2109         spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
2110
2111         list_for_each_safe(node, tmp, &cryptocop_completed_jobs) {
2112                 pj = list_entry(node, struct cryptocop_prio_job, node);
2113                 list_del(node);
2114                 /* Call callback to notify consumer of job removal. */
2115                 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2116                 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2117                 pj->oper->cb(pj->oper, pj->oper->cb_data);
2118
2119                 delete_internal_operation(pj->iop);
2120                 kfree(pj);
2121         }
2122         spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
2123 }
2124
2125
2126 static void cryptocop_start_job(void)
2127 {
2128         int                          i;
2129         struct cryptocop_prio_job    *pj;
2130         unsigned long int            flags;
2131         unsigned long int            running_job_flags;
2132         reg_strcop_rw_cfg            rw_cfg = {.en = 1, .ignore_sync = 0};
2133
2134         DEBUG(printk("cryptocop_start_job: entering\n"));
2135
2136         spin_lock_irqsave(&running_job_lock, running_job_flags);
2137         if (cryptocop_running_job != NULL){
2138                 /* Already running. */
2139                 DEBUG(printk("cryptocop_start_job: already running, exit\n"));
2140                 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2141                 return;
2142         }
2143         spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
2144
2145         /* Check the queues in priority order. */
2146         for (i = cryptocop_prio_kernel_csum; (i < cryptocop_prio_no_prios) && list_empty(&cryptocop_job_queues[i].jobs); i++);
2147         if (i == cryptocop_prio_no_prios) {
2148                 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2149                 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2150                 DEBUG(printk("cryptocop_start_job: no jobs to run\n"));
2151                 return; /* No jobs to run */
2152         }
2153         DEBUG(printk("starting job for prio %d\n", i));
2154
2155         /* TODO: Do not starve lower priority jobs.  Let in a lower
2156          * prio job for every N-th processed higher prio job or some
2157          * other scheduling policy.  This could reasonably be
2158          * tweakable since the optimal balance would depend on the
2159          * type of load on the system. */
2160
2161         /* Pull the DMA lists from the job and start the DMA client. */
2162         pj = list_entry(cryptocop_job_queues[i].jobs.next, struct cryptocop_prio_job, node);
2163         list_del(&pj->node);
2164         spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2165         cryptocop_running_job = pj;
2166
2167         /* Set config register (3DES and CSUM modes). */
2168         switch (pj->iop->tdes_mode){
2169         case cryptocop_3des_eee:
2170                 rw_cfg.td1 = regk_strcop_e;
2171                 rw_cfg.td2 = regk_strcop_e;
2172                 rw_cfg.td3 = regk_strcop_e;
2173                 break;
2174         case cryptocop_3des_eed:
2175                 rw_cfg.td1 = regk_strcop_e;
2176                 rw_cfg.td2 = regk_strcop_e;
2177                 rw_cfg.td3 = regk_strcop_d;
2178                 break;
2179         case cryptocop_3des_ede:
2180                 rw_cfg.td1 = regk_strcop_e;
2181                 rw_cfg.td2 = regk_strcop_d;
2182                 rw_cfg.td3 = regk_strcop_e;
2183                 break;
2184         case cryptocop_3des_edd:
2185                 rw_cfg.td1 = regk_strcop_e;
2186                 rw_cfg.td2 = regk_strcop_d;
2187                 rw_cfg.td3 = regk_strcop_d;
2188                 break;
2189         case cryptocop_3des_dee:
2190                 rw_cfg.td1 = regk_strcop_d;
2191                 rw_cfg.td2 = regk_strcop_e;
2192                 rw_cfg.td3 = regk_strcop_e;
2193                 break;
2194         case cryptocop_3des_ded:
2195                 rw_cfg.td1 = regk_strcop_d;
2196                 rw_cfg.td2 = regk_strcop_e;
2197                 rw_cfg.td3 = regk_strcop_d;
2198                 break;
2199         case cryptocop_3des_dde:
2200                 rw_cfg.td1 = regk_strcop_d;
2201                 rw_cfg.td2 = regk_strcop_d;
2202                 rw_cfg.td3 = regk_strcop_e;
2203                 break;
2204         case cryptocop_3des_ddd:
2205                 rw_cfg.td1 = regk_strcop_d;
2206                 rw_cfg.td2 = regk_strcop_d;
2207                 rw_cfg.td3 = regk_strcop_d;
2208                 break;
2209         default:
2210                 DEBUG(printk("cryptocop_setup_dma_list: bad 3DES mode\n"));
2211         }
2212         switch (pj->iop->csum_mode){
2213         case cryptocop_csum_le:
2214                 rw_cfg.ipend = regk_strcop_little;
2215                 break;
2216         case cryptocop_csum_be:
2217                 rw_cfg.ipend = regk_strcop_big;
2218                 break;
2219         default:
2220                 DEBUG(printk("cryptocop_setup_dma_list: bad checksum mode\n"));
2221         }
2222         REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2223
2224         DEBUG(printk("cryptocop_start_job: starting DMA, new cryptocop_running_job=0x%p\n"
2225                      "ctx_in: 0x%p, phys: 0x%p\n"
2226                      "ctx_out: 0x%p, phys: 0x%p\n",
2227                      pj,
2228                      &pj->iop->ctx_in, (char*)virt_to_phys(&pj->iop->ctx_in),
2229                      &pj->iop->ctx_out, (char*)virt_to_phys(&pj->iop->ctx_out)));
2230
2231         /* Start input DMA. */
2232         flush_dma_context(&pj->iop->ctx_in);
2233         DMA_START_CONTEXT(IN_DMA_INST, virt_to_phys(&pj->iop->ctx_in));
2234
2235         /* Start output DMA. */
2236         DMA_START_CONTEXT(OUT_DMA_INST, virt_to_phys(&pj->iop->ctx_out));
2237
2238         spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2239         DEBUG(printk("cryptocop_start_job: exiting\n"));
2240 }
2241
2242
2243 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation)
2244 {
2245         int  err;
2246         int  alloc_flag = operation->in_interrupt ? GFP_ATOMIC : GFP_KERNEL;
2247         void *iop_alloc_ptr = NULL;
2248
2249         *pj = kmalloc(sizeof (struct cryptocop_prio_job), alloc_flag);
2250         if (!*pj) return -ENOMEM;
2251
2252         DEBUG(printk("cryptocop_job_setup: operation=0x%p\n", operation));
2253
2254         (*pj)->oper = operation;
2255         DEBUG(printk("cryptocop_job_setup, cb=0x%p cb_data=0x%p\n",  (*pj)->oper->cb, (*pj)->oper->cb_data));
2256
2257         if (operation->use_dmalists) {
2258                 DEBUG(print_user_dma_lists(&operation->list_op));
2259                 if (!operation->list_op.inlist || !operation->list_op.outlist || !operation->list_op.out_data_buf || !operation->list_op.in_data_buf){
2260                         DEBUG_API(printk("cryptocop_job_setup: bad indata (use_dmalists)\n"));
2261                         kfree(*pj);
2262                         return -EINVAL;
2263                 }
2264                 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
2265                 if (!iop_alloc_ptr) {
2266                         DEBUG_API(printk("cryptocop_job_setup: kmalloc cryptocop_int_operation\n"));
2267                         kfree(*pj);
2268                         return -ENOMEM;
2269                 }
2270                 (*pj)->iop = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
2271                 DEBUG(memset((*pj)->iop, 0xff, sizeof(struct cryptocop_int_operation)));
2272                 (*pj)->iop->alloc_ptr = iop_alloc_ptr;
2273                 (*pj)->iop->sid = operation->sid;
2274                 (*pj)->iop->cdesc_out = NULL;
2275                 (*pj)->iop->cdesc_in = NULL;
2276                 (*pj)->iop->tdes_mode = operation->list_op.tdes_mode;
2277                 (*pj)->iop->csum_mode = operation->list_op.csum_mode;
2278                 (*pj)->iop->ddesc_out = operation->list_op.outlist;
2279                 (*pj)->iop->ddesc_in = operation->list_op.inlist;
2280
2281                 /* Setup DMA contexts. */
2282                 (*pj)->iop->ctx_out.next = NULL;
2283                 (*pj)->iop->ctx_out.eol = 1;
2284                 (*pj)->iop->ctx_out.saved_data = operation->list_op.outlist;
2285                 (*pj)->iop->ctx_out.saved_data_buf = operation->list_op.out_data_buf;
2286
2287                 (*pj)->iop->ctx_in.next = NULL;
2288                 (*pj)->iop->ctx_in.eol = 1;
2289                 (*pj)->iop->ctx_in.saved_data = operation->list_op.inlist;
2290                 (*pj)->iop->ctx_in.saved_data_buf = operation->list_op.in_data_buf;
2291         } else {
2292                 if ((err = cryptocop_setup_dma_list(operation, &(*pj)->iop, alloc_flag))) {
2293                         DEBUG_API(printk("cryptocop_job_setup: cryptocop_setup_dma_list failed %d\n", err));
2294                         kfree(*pj);
2295                         return err;
2296                 }
2297         }
2298         DEBUG(print_dma_descriptors((*pj)->iop));
2299
2300         DEBUG(printk("cryptocop_job_setup, DMA list setup successful\n"));
2301
2302         return 0;
2303 }
2304
2305
2306 static int cryptocop_open(struct inode *inode, struct file *filp)
2307 {
2308         int p = iminor(inode);
2309
2310         if (p != CRYPTOCOP_MINOR) return -EINVAL;
2311
2312         filp->private_data = NULL;
2313         return 0;
2314 }
2315
2316
2317 static int cryptocop_release(struct inode *inode, struct file *filp)
2318 {
2319         struct cryptocop_private *dev = filp->private_data;
2320         struct cryptocop_private *dev_next;
2321
2322         while (dev){
2323                 dev_next = dev->next;
2324                 if (dev->sid != CRYPTOCOP_SESSION_ID_NONE) {
2325                         (void)cryptocop_free_session(dev->sid);
2326                 }
2327                 kfree(dev);
2328                 dev = dev_next;
2329         }
2330
2331         return 0;
2332 }
2333
2334
2335 static int cryptocop_ioctl_close_session(struct inode *inode, struct file *filp,
2336                                          unsigned int cmd, unsigned long arg)
2337 {
2338         struct cryptocop_private  *dev = filp->private_data;
2339         struct cryptocop_private  *prev_dev = NULL;
2340         struct strcop_session_op  *sess_op = (struct strcop_session_op *)arg;
2341         struct strcop_session_op  sop;
2342         int                       err;
2343
2344         DEBUG(printk("cryptocop_ioctl_close_session\n"));
2345
2346         if (!access_ok(VERIFY_READ, sess_op, sizeof(struct strcop_session_op)))
2347                 return -EFAULT;
2348         err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2349         if (err) return -EFAULT;
2350
2351         while (dev && (dev->sid != sop.ses_id)) {
2352                 prev_dev = dev;
2353                 dev = dev->next;
2354         }
2355         if (dev){
2356                 if (prev_dev){
2357                         prev_dev->next = dev->next;
2358                 } else {
2359                         filp->private_data = dev->next;
2360                 }
2361                 err = cryptocop_free_session(dev->sid);
2362                 if (err) return -EFAULT;
2363         } else {
2364                 DEBUG_API(printk("cryptocop_ioctl_close_session: session %lld not found\n", sop.ses_id));
2365                 return -EINVAL;
2366         }
2367         return 0;
2368 }
2369
2370
2371 static void ioctl_process_job_callback(struct cryptocop_operation *op, void*cb_data)
2372 {
2373         struct ioctl_job_cb_ctx *jc = (struct ioctl_job_cb_ctx *)cb_data;
2374
2375         DEBUG(printk("ioctl_process_job_callback: op=0x%p, cb_data=0x%p\n", op, cb_data));
2376
2377         jc->processed = 1;
2378         wake_up(&cryptocop_ioc_process_wq);
2379 }
2380
2381
2382 #define CRYPTOCOP_IOCTL_CIPHER_TID  (1)
2383 #define CRYPTOCOP_IOCTL_DIGEST_TID  (2)
2384 #define CRYPTOCOP_IOCTL_CSUM_TID    (3)
2385
2386 static size_t first_cfg_change_ix(struct strcop_crypto_op *crp_op)
2387 {
2388         size_t ch_ix = 0;
2389
2390         if (crp_op->do_cipher) ch_ix = crp_op->cipher_start;
2391         if (crp_op->do_digest && (crp_op->digest_start < ch_ix)) ch_ix = crp_op->digest_start;
2392         if (crp_op->do_csum && (crp_op->csum_start < ch_ix)) ch_ix = crp_op->csum_start;
2393
2394         DEBUG(printk("first_cfg_change_ix: ix=%d\n", ch_ix));
2395         return ch_ix;
2396 }
2397
2398
2399 static size_t next_cfg_change_ix(struct strcop_crypto_op *crp_op, size_t ix)
2400 {
2401         size_t ch_ix = INT_MAX;
2402         size_t tmp_ix = 0;
2403
2404         if (crp_op->do_cipher && ((crp_op->cipher_start + crp_op->cipher_len) > ix)){
2405                 if (crp_op->cipher_start > ix) {
2406                         ch_ix = crp_op->cipher_start;
2407                 } else {
2408                         ch_ix = crp_op->cipher_start + crp_op->cipher_len;
2409                 }
2410         }
2411         if (crp_op->do_digest && ((crp_op->digest_start + crp_op->digest_len) > ix)){
2412                 if (crp_op->digest_start > ix) {
2413                         tmp_ix = crp_op->digest_start;
2414                 } else {
2415                         tmp_ix = crp_op->digest_start + crp_op->digest_len;
2416                 }
2417                 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2418         }
2419         if (crp_op->do_csum && ((crp_op->csum_start + crp_op->csum_len) > ix)){
2420                 if (crp_op->csum_start > ix) {
2421                         tmp_ix = crp_op->csum_start;
2422                 } else {
2423                         tmp_ix = crp_op->csum_start + crp_op->csum_len;
2424                 }
2425                 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2426         }
2427         if (ch_ix == INT_MAX) ch_ix = ix;
2428         DEBUG(printk("next_cfg_change_ix prev ix=%d, next ix=%d\n", ix, ch_ix));
2429         return ch_ix;
2430 }
2431
2432
2433 /* Map map_length bytes from the pages starting on *pageix and *pageoffset to iovecs starting on *iovix.
2434  * Return -1 for ok, 0 for fail. */
2435 static int map_pages_to_iovec(struct iovec *iov, int iovlen, int *iovix, struct page **pages, int nopages, int *pageix, int *pageoffset, int map_length )
2436 {
2437         int tmplen;
2438
2439         assert(iov != NULL);
2440         assert(iovix != NULL);
2441         assert(pages != NULL);
2442         assert(pageix != NULL);
2443         assert(pageoffset != NULL);
2444
2445         DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2446
2447         while (map_length > 0){
2448                 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2449                 if (*iovix >= iovlen){
2450                         DEBUG_API(printk("map_page_to_iovec: *iovix=%d >= iovlen=%d\n", *iovix, iovlen));
2451                         return 0;
2452                 }
2453                 if (*pageix >= nopages){
2454                         DEBUG_API(printk("map_page_to_iovec: *pageix=%d >= nopages=%d\n", *pageix, nopages));
2455                         return 0;
2456                 }
2457                 iov[*iovix].iov_base = (unsigned char*)page_address(pages[*pageix]) + *pageoffset;
2458                 tmplen = PAGE_SIZE - *pageoffset;
2459                 if (tmplen < map_length){
2460                         (*pageoffset) = 0;
2461                         (*pageix)++;
2462                 } else {
2463                         tmplen = map_length;
2464                         (*pageoffset) += map_length;
2465                 }
2466                 DEBUG(printk("mapping %d bytes from page %d (or %d) to iovec %d\n", tmplen, *pageix, *pageix-1, *iovix));
2467                 iov[*iovix].iov_len = tmplen;
2468                 map_length -= tmplen;
2469                 (*iovix)++;
2470         }
2471         DEBUG(printk("map_page_to_iovec, exit, *iovix=%d\n", *iovix));
2472         return -1;
2473 }
2474
2475
2476
2477 static int cryptocop_ioctl_process(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2478 {
2479         int                             i;
2480         struct cryptocop_private        *dev = filp->private_data;
2481         struct strcop_crypto_op         *crp_oper = (struct strcop_crypto_op *)arg;
2482         struct strcop_crypto_op         oper = {0};
2483         int                             err = 0;
2484         struct cryptocop_operation      *cop = NULL;
2485
2486         struct ioctl_job_cb_ctx         *jc = NULL;
2487
2488         struct page                     **inpages = NULL;
2489         struct page                     **outpages = NULL;
2490         int                             noinpages = 0;
2491         int                             nooutpages = 0;
2492
2493         struct cryptocop_desc           descs[5]; /* Max 5 descriptors are needed, there are three transforms that
2494                                                    * can get connected/disconnected on different places in the indata. */
2495         struct cryptocop_desc_cfg       dcfgs[5*3];
2496         int                             desc_ix = 0;
2497         int                             dcfg_ix = 0;
2498         struct cryptocop_tfrm_cfg       ciph_tcfg = {0};
2499         struct cryptocop_tfrm_cfg       digest_tcfg = {0};
2500         struct cryptocop_tfrm_cfg       csum_tcfg = {0};
2501
2502         unsigned char                   *digest_result = NULL;
2503         int                             digest_length = 0;
2504         int                             cblocklen = 0;
2505         unsigned char                   csum_result[CSUM_BLOCK_LENGTH];
2506         struct cryptocop_session        *sess;
2507
2508         int    iovlen = 0;
2509         int    iovix = 0;
2510         int    pageix = 0;
2511         int    pageoffset = 0;
2512
2513         size_t prev_ix = 0;
2514         size_t next_ix;
2515
2516         int    cipher_active, digest_active, csum_active;
2517         int    end_digest, end_csum;
2518         int    digest_done = 0;
2519         int    cipher_done = 0;
2520         int    csum_done = 0;
2521
2522         DEBUG(printk("cryptocop_ioctl_process\n"));
2523
2524         if (!access_ok(VERIFY_WRITE, crp_oper, sizeof(struct strcop_crypto_op))){
2525                 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok crp_oper!\n"));
2526                 return -EFAULT;
2527         }
2528         if (copy_from_user(&oper, crp_oper, sizeof(struct strcop_crypto_op))) {
2529                 DEBUG_API(printk("cryptocop_ioctl_process: copy_from_user\n"));
2530                 return -EFAULT;
2531         }
2532         DEBUG(print_strcop_crypto_op(&oper));
2533
2534         while (dev && dev->sid != oper.ses_id) dev = dev->next;
2535         if (!dev){
2536                 DEBUG_API(printk("cryptocop_ioctl_process: session %lld not found\n", oper.ses_id));
2537                 return -EINVAL;
2538         }
2539
2540         /* Check buffers. */
2541         if (((oper.indata + oper.inlen) < oper.indata) || ((oper.cipher_outdata + oper.cipher_outlen) < oper.cipher_outdata)){
2542                 DEBUG_API(printk("cryptocop_ioctl_process: user buffers wrapped around, bad user!\n"));
2543                 return -EINVAL;
2544         }
2545
2546         if (!access_ok(VERIFY_WRITE, oper.cipher_outdata, oper.cipher_outlen)){
2547                 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok out data!\n"));
2548                 return -EFAULT;
2549         }
2550         if (!access_ok(VERIFY_READ, oper.indata, oper.inlen)){
2551                 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok in data!\n"));
2552                 return -EFAULT;
2553         }
2554
2555         cop = kmalloc(sizeof(struct cryptocop_operation), GFP_KERNEL);
2556         if (!cop) {
2557                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2558                 return -ENOMEM;
2559         }
2560         jc = kmalloc(sizeof(struct ioctl_job_cb_ctx), GFP_KERNEL);
2561         if (!jc) {
2562                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2563                 err = -ENOMEM;
2564                 goto error_cleanup;
2565         }
2566         jc->processed = 0;
2567
2568         cop->cb_data = jc;
2569         cop->cb = ioctl_process_job_callback;
2570         cop->operation_status = 0;
2571         cop->use_dmalists = 0;
2572         cop->in_interrupt = 0;
2573         cop->fast_callback = 0;
2574         cop->tfrm_op.tfrm_cfg = NULL;
2575         cop->tfrm_op.desc = NULL;
2576         cop->tfrm_op.indata = NULL;
2577         cop->tfrm_op.incount = 0;
2578         cop->tfrm_op.inlen = 0;
2579         cop->tfrm_op.outdata = NULL;
2580         cop->tfrm_op.outcount = 0;
2581         cop->tfrm_op.outlen = 0;
2582
2583         sess = get_session(oper.ses_id);
2584         if (!sess){
2585                 DEBUG_API(printk("cryptocop_ioctl_process: bad session id.\n"));
2586                 kfree(cop);
2587                 kfree(jc);
2588                 return -EINVAL;
2589         }
2590
2591         if (oper.do_cipher) {
2592                 unsigned int                    cipher_outlen = 0;
2593                 struct cryptocop_transform_ctx  *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_CIPHER_TID);
2594                 if (!tc) {
2595                         DEBUG_API(printk("cryptocop_ioctl_process: no cipher transform in session.\n"));
2596                         err = -EINVAL;
2597                         goto error_cleanup;
2598                 }
2599                 ciph_tcfg.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2600                 ciph_tcfg.inject_ix = 0;
2601                 ciph_tcfg.flags = 0;
2602                 if ((oper.cipher_start < 0) || (oper.cipher_len <= 0) || (oper.cipher_start > oper.inlen) || ((oper.cipher_start + oper.cipher_len) > oper.inlen)){
2603                         DEBUG_API(printk("cryptocop_ioctl_process: bad cipher length\n"));
2604                         kfree(cop);
2605                         kfree(jc);
2606                         return -EINVAL;
2607                 }
2608                 cblocklen = tc->init.alg == cryptocop_alg_aes ? AES_BLOCK_LENGTH : DES_BLOCK_LENGTH;
2609                 if (oper.cipher_len % cblocklen) {
2610                         kfree(cop);
2611                         kfree(jc);
2612                         DEBUG_API(printk("cryptocop_ioctl_process: cipher inlength not multiple of block length.\n"));
2613                         return -EINVAL;
2614                 }
2615                 cipher_outlen = oper.cipher_len;
2616                 if (tc->init.cipher_mode == cryptocop_cipher_mode_cbc){
2617                         if (oper.cipher_explicit) {
2618                                 ciph_tcfg.flags |= CRYPTOCOP_EXPLICIT_IV;
2619                                 memcpy(ciph_tcfg.iv, oper.cipher_iv, cblocklen);
2620                         } else {
2621                                 cipher_outlen = oper.cipher_len - cblocklen;
2622                         }
2623                 } else {
2624                         if (oper.cipher_explicit){
2625                                 kfree(cop);
2626                                 kfree(jc);
2627                                 DEBUG_API(printk("cryptocop_ioctl_process: explicit_iv when not CBC mode\n"));
2628                                 return -EINVAL;
2629                         }
2630                 }
2631                 if (oper.cipher_outlen != cipher_outlen) {
2632                         kfree(cop);
2633                         kfree(jc);
2634                         DEBUG_API(printk("cryptocop_ioctl_process: cipher_outlen incorrect, should be %d not %d.\n", cipher_outlen, oper.cipher_outlen));
2635                         return -EINVAL;
2636                 }
2637
2638                 if (oper.decrypt){
2639                         ciph_tcfg.flags |= CRYPTOCOP_DECRYPT;
2640                 } else {
2641                         ciph_tcfg.flags |= CRYPTOCOP_ENCRYPT;
2642                 }
2643                 ciph_tcfg.next = cop->tfrm_op.tfrm_cfg;
2644                 cop->tfrm_op.tfrm_cfg = &ciph_tcfg;
2645         }
2646         if (oper.do_digest){
2647                 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_DIGEST_TID);
2648                 if (!tc) {
2649                         DEBUG_API(printk("cryptocop_ioctl_process: no digest transform in session.\n"));
2650                         err = -EINVAL;
2651                         goto error_cleanup;
2652                 }
2653                 digest_length = tc->init.alg == cryptocop_alg_md5 ? 16 : 20;
2654                 digest_result = kmalloc(digest_length, GFP_KERNEL);
2655                 if (!digest_result) {
2656                         DEBUG_API(printk("cryptocop_ioctl_process: kmalloc digest_result\n"));
2657                         err = -EINVAL;
2658                         goto error_cleanup;
2659                 }
2660                 DEBUG(memset(digest_result, 0xff, digest_length));
2661
2662                 digest_tcfg.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2663                 digest_tcfg.inject_ix = 0;
2664                 ciph_tcfg.inject_ix += digest_length;
2665                 if ((oper.digest_start < 0) || (oper.digest_len <= 0) || (oper.digest_start > oper.inlen) || ((oper.digest_start + oper.digest_len) > oper.inlen)){
2666                         DEBUG_API(printk("cryptocop_ioctl_process: bad digest length\n"));
2667                         err = -EINVAL;
2668                         goto error_cleanup;
2669                 }
2670
2671                 digest_tcfg.next = cop->tfrm_op.tfrm_cfg;
2672                 cop->tfrm_op.tfrm_cfg = &digest_tcfg;
2673         }
2674         if (oper.do_csum){
2675                 csum_tcfg.tid = CRYPTOCOP_IOCTL_CSUM_TID;
2676                 csum_tcfg.inject_ix = digest_length;
2677                 ciph_tcfg.inject_ix += 2;
2678
2679                 if ((oper.csum_start < 0) || (oper.csum_len <= 0) || (oper.csum_start > oper.inlen) || ((oper.csum_start + oper.csum_len) > oper.inlen)){
2680                         DEBUG_API(printk("cryptocop_ioctl_process: bad csum length\n"));
2681                         kfree(cop);
2682                         kfree(jc);
2683                         return -EINVAL;
2684                 }
2685
2686                 csum_tcfg.next = cop->tfrm_op.tfrm_cfg;
2687                 cop->tfrm_op.tfrm_cfg = &csum_tcfg;
2688         }
2689
2690         prev_ix = first_cfg_change_ix(&oper);
2691         if (prev_ix > oper.inlen) {
2692                 DEBUG_API(printk("cryptocop_ioctl_process: length mismatch\n"));
2693                 nooutpages = noinpages = 0;
2694                 err = -EINVAL;
2695                 goto error_cleanup;
2696         }
2697         DEBUG(printk("cryptocop_ioctl_process: inlen=%d, cipher_outlen=%d\n", oper.inlen, oper.cipher_outlen));
2698
2699         /* Map user pages for in and out data of the operation. */
2700         noinpages = (((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK) + oper.inlen - 1 - prev_ix + ~PAGE_MASK) >> PAGE_SHIFT;
2701         DEBUG(printk("cryptocop_ioctl_process: noinpages=%d\n", noinpages));
2702         inpages = kmalloc(noinpages * sizeof(struct page*), GFP_KERNEL);
2703         if (!inpages){
2704                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc inpages\n"));
2705                 nooutpages = noinpages = 0;
2706                 err = -ENOMEM;
2707                 goto error_cleanup;
2708         }
2709         if (oper.do_cipher){
2710                 nooutpages = (((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) + oper.cipher_outlen - 1 + ~PAGE_MASK) >> PAGE_SHIFT;
2711                 DEBUG(printk("cryptocop_ioctl_process: nooutpages=%d\n", nooutpages));
2712                 outpages = kmalloc(nooutpages * sizeof(struct page*), GFP_KERNEL);
2713                 if (!outpages){
2714                         DEBUG_API(printk("cryptocop_ioctl_process: kmalloc outpages\n"));
2715                         nooutpages = noinpages = 0;
2716                         err = -ENOMEM;
2717                         goto error_cleanup;
2718                 }
2719         }
2720
2721         /* Acquire the mm page semaphore. */
2722         down_read(&current->mm->mmap_sem);
2723
2724         err = get_user_pages(current,
2725                              current->mm,
2726                              (unsigned long int)(oper.indata + prev_ix),
2727                              noinpages,
2728                              0,  /* read access only for in data */
2729                              0, /* no force */
2730                              inpages,
2731                              NULL);
2732
2733         if (err < 0) {
2734                 up_read(&current->mm->mmap_sem);
2735                 nooutpages = noinpages = 0;
2736                 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages indata\n"));
2737                 goto error_cleanup;
2738         }
2739         noinpages = err;
2740         if (oper.do_cipher){
2741                 err = get_user_pages(current,
2742                                      current->mm,
2743                                      (unsigned long int)oper.cipher_outdata,
2744                                      nooutpages,
2745                                      1, /* write access for out data */
2746                                      0, /* no force */
2747                                      outpages,
2748                                      NULL);
2749                 up_read(&current->mm->mmap_sem);
2750                 if (err < 0) {
2751                         nooutpages = 0;
2752                         DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages outdata\n"));
2753                         goto error_cleanup;
2754                 }
2755                 nooutpages = err;
2756         } else {
2757                 up_read(&current->mm->mmap_sem);
2758         }
2759
2760         /* Add 6 to nooutpages to make room for possibly inserted buffers for storing digest and
2761          * csum output and splits when units are (dis-)connected. */
2762         cop->tfrm_op.indata = kmalloc((noinpages) * sizeof(struct iovec), GFP_KERNEL);
2763         cop->tfrm_op.outdata = kmalloc((6 + nooutpages) * sizeof(struct iovec), GFP_KERNEL);
2764         if (!cop->tfrm_op.indata || !cop->tfrm_op.outdata) {
2765                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc iovecs\n"));
2766                 err = -ENOMEM;
2767                 goto error_cleanup;
2768         }
2769
2770         cop->tfrm_op.inlen = oper.inlen - prev_ix;
2771         cop->tfrm_op.outlen = 0;
2772         if (oper.do_cipher) cop->tfrm_op.outlen += oper.cipher_outlen;
2773         if (oper.do_digest) cop->tfrm_op.outlen += digest_length;
2774         if (oper.do_csum) cop->tfrm_op.outlen += 2;
2775
2776         /* Setup the in iovecs. */
2777         cop->tfrm_op.incount = noinpages;
2778         if (noinpages > 1){
2779                 size_t tmplen = cop->tfrm_op.inlen;
2780
2781                 cop->tfrm_op.indata[0].iov_len = PAGE_SIZE - ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2782                 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2783                 tmplen -= cop->tfrm_op.indata[0].iov_len;
2784                 for (i = 1; i<noinpages; i++){
2785                         cop->tfrm_op.indata[i].iov_len = tmplen < PAGE_SIZE ? tmplen : PAGE_SIZE;
2786                         cop->tfrm_op.indata[i].iov_base = (unsigned char*)page_address(inpages[i]);
2787                         tmplen -= PAGE_SIZE;
2788                 }
2789         } else {
2790                 cop->tfrm_op.indata[0].iov_len = oper.inlen - prev_ix;
2791                 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2792         }
2793
2794         iovlen = nooutpages + 6;
2795         pageoffset = oper.do_cipher ? ((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) : 0;
2796
2797         next_ix = next_cfg_change_ix(&oper, prev_ix);
2798         if (prev_ix == next_ix){
2799                 DEBUG_API(printk("cryptocop_ioctl_process: length configuration broken.\n"));
2800                 err = -EINVAL;  /* This should be impossible barring bugs. */
2801                 goto error_cleanup;
2802         }
2803         while (prev_ix != next_ix){
2804                 end_digest = end_csum = cipher_active = digest_active = csum_active = 0;
2805                 descs[desc_ix].cfg = NULL;
2806                 descs[desc_ix].length = next_ix - prev_ix;
2807
2808                 if (oper.do_cipher && (oper.cipher_start < next_ix) && (prev_ix < (oper.cipher_start + oper.cipher_len))) {
2809                         dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2810                         dcfgs[dcfg_ix].src = cryptocop_source_dma;
2811                         cipher_active = 1;
2812
2813                         if (next_ix == (oper.cipher_start + oper.cipher_len)){
2814                                 cipher_done = 1;
2815                                 dcfgs[dcfg_ix].last = 1;
2816                         } else {
2817                                 dcfgs[dcfg_ix].last = 0;
2818                         }
2819                         dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2820                         descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2821                         ++dcfg_ix;
2822                 }
2823                 if (oper.do_digest && (oper.digest_start < next_ix) && (prev_ix < (oper.digest_start + oper.digest_len))) {
2824                         digest_active = 1;
2825                         dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2826                         dcfgs[dcfg_ix].src = cryptocop_source_dma;
2827                         if (next_ix == (oper.digest_start + oper.digest_len)){
2828                                 assert(!digest_done);
2829                                 digest_done = 1;
2830                                 dcfgs[dcfg_ix].last = 1;
2831                         } else {
2832                                 dcfgs[dcfg_ix].last = 0;
2833                         }
2834                         dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2835                         descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2836                         ++dcfg_ix;
2837                 }
2838                 if (oper.do_csum && (oper.csum_start < next_ix) && (prev_ix < (oper.csum_start + oper.csum_len))){
2839                         csum_active = 1;
2840                         dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CSUM_TID;
2841                         dcfgs[dcfg_ix].src = cryptocop_source_dma;
2842                         if (next_ix == (oper.csum_start + oper.csum_len)){
2843                                 csum_done = 1;
2844                                 dcfgs[dcfg_ix].last = 1;
2845                         } else {
2846                                 dcfgs[dcfg_ix].last = 0;
2847                         }
2848                         dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2849                         descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2850                         ++dcfg_ix;
2851                 }
2852                 if (!descs[desc_ix].cfg){
2853                         DEBUG_API(printk("cryptocop_ioctl_process: data segment %d (%d to %d) had no active transforms\n", desc_ix, prev_ix, next_ix));
2854                         err = -EINVAL;
2855                         goto error_cleanup;
2856                 }
2857                 descs[desc_ix].next = &(descs[desc_ix]) + 1;
2858                 ++desc_ix;
2859                 prev_ix = next_ix;
2860                 next_ix = next_cfg_change_ix(&oper, prev_ix);
2861         }
2862         if (desc_ix > 0){
2863                 descs[desc_ix-1].next = NULL;
2864         } else {
2865                 descs[0].next = NULL;
2866         }
2867         if (oper.do_digest) {
2868                 DEBUG(printk("cryptocop_ioctl_process: mapping %d byte digest output to iovec %d\n", digest_length, iovix));
2869                 /* Add outdata iovec, length == <length of type of digest> */
2870                 cop->tfrm_op.outdata[iovix].iov_base = digest_result;
2871                 cop->tfrm_op.outdata[iovix].iov_len = digest_length;
2872                 ++iovix;
2873         }
2874         if (oper.do_csum) {
2875                 /* Add outdata iovec, length == 2, the length of csum. */
2876                 DEBUG(printk("cryptocop_ioctl_process: mapping 2 byte csum output to iovec %d\n", iovix));
2877                 /* Add outdata iovec, length == <length of type of digest> */
2878                 cop->tfrm_op.outdata[iovix].iov_base = csum_result;
2879                 cop->tfrm_op.outdata[iovix].iov_len = 2;
2880                 ++iovix;
2881         }
2882         if (oper.do_cipher) {
2883                 if (!map_pages_to_iovec(cop->tfrm_op.outdata, iovlen, &iovix, outpages, nooutpages, &pageix, &pageoffset, oper.cipher_outlen)){
2884                         DEBUG_API(printk("cryptocop_ioctl_process: failed to map pages to iovec.\n"));
2885                         err = -ENOSYS; /* This should be impossible barring bugs. */
2886                         goto error_cleanup;
2887                 }
2888         }
2889         DEBUG(printk("cryptocop_ioctl_process: setting cop->tfrm_op.outcount %d\n", iovix));
2890         cop->tfrm_op.outcount = iovix;
2891         assert(iovix <= (nooutpages + 6));
2892
2893         cop->sid = oper.ses_id;
2894         cop->tfrm_op.desc = &descs[0];
2895
2896         DEBUG(printk("cryptocop_ioctl_process: inserting job, cb_data=0x%p\n", cop->cb_data));
2897
2898         if ((err = cryptocop_job_queue_insert_user_job(cop)) != 0) {
2899                 DEBUG_API(printk("cryptocop_ioctl_process: insert job %d\n", err));
2900                 err = -EINVAL;
2901                 goto error_cleanup;
2902         }
2903
2904         DEBUG(printk("cryptocop_ioctl_process: begin wait for result\n"));
2905
2906         wait_event(cryptocop_ioc_process_wq, (jc->processed != 0));
2907         DEBUG(printk("cryptocop_ioctl_process: end wait for result\n"));
2908         if (!jc->processed){
2909                 printk(KERN_WARNING "cryptocop_ioctl_process: job not processed at completion\n");
2910                 err = -EIO;
2911                 goto error_cleanup;
2912         }
2913
2914         /* Job process done.  Cipher output should already be correct in job so no post processing of outdata. */
2915         DEBUG(printk("cryptocop_ioctl_process: operation_status = %d\n", cop->operation_status));
2916         if (cop->operation_status == 0){
2917                 if (oper.do_digest){
2918                         DEBUG(printk("cryptocop_ioctl_process: copy %d bytes digest to user\n", digest_length));
2919                         err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, digest), digest_result, digest_length);
2920                         if (0 != err){
2921                                 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, digest length %d, err %d\n", digest_length, err));
2922                                 err = -EFAULT;
2923                                 goto error_cleanup;
2924                         }
2925                 }
2926                 if (oper.do_csum){
2927                         DEBUG(printk("cryptocop_ioctl_process: copy 2 bytes checksum to user\n"));
2928                         err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, csum), csum_result, 2);
2929                         if (0 != err){
2930                                 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, csum, err %d\n", err));
2931                                 err = -EFAULT;
2932                                 goto error_cleanup;
2933                         }
2934                 }
2935                 err = 0;
2936         } else {
2937                 DEBUG(printk("cryptocop_ioctl_process: returning err = operation_status = %d\n", cop->operation_status));
2938                 err = cop->operation_status;
2939         }
2940
2941  error_cleanup:
2942         /* Release page caches. */
2943         for (i = 0; i < noinpages; i++){
2944                 put_page(inpages[i]);
2945         }
2946         for (i = 0; i < nooutpages; i++){
2947                 int spdl_err;
2948                 /* Mark output pages dirty. */
2949                 spdl_err = set_page_dirty_lock(outpages[i]);
2950                 DEBUG(if (spdl_err < 0)printk("cryptocop_ioctl_process: set_page_dirty_lock returned %d\n", spdl_err));
2951         }
2952         for (i = 0; i < nooutpages; i++){
2953                 put_page(outpages[i]);
2954         }
2955
2956         kfree(digest_result);
2957         kfree(inpages);
2958         kfree(outpages);
2959         if (cop){
2960                 kfree(cop->tfrm_op.indata);
2961                 kfree(cop->tfrm_op.outdata);
2962                 kfree(cop);
2963         }
2964         kfree(jc);
2965
2966         DEBUG(print_lock_status());
2967
2968         return err;
2969 }
2970
2971
2972 static int cryptocop_ioctl_create_session(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2973 {
2974         cryptocop_session_id             sid;
2975         int                              err;
2976         struct cryptocop_private         *dev;
2977         struct strcop_session_op         *sess_op = (struct strcop_session_op *)arg;
2978         struct strcop_session_op         sop;
2979         struct cryptocop_transform_init  *tis = NULL;
2980         struct cryptocop_transform_init  ti_cipher = {0};
2981         struct cryptocop_transform_init  ti_digest = {0};
2982         struct cryptocop_transform_init  ti_csum = {0};
2983
2984         if (!access_ok(VERIFY_WRITE, sess_op, sizeof(struct strcop_session_op)))
2985                 return -EFAULT;
2986         err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2987         if (err) return -EFAULT;
2988         if (sop.cipher != cryptocop_cipher_none) {
2989                 if (!access_ok(VERIFY_READ, sop.key, sop.keylen)) return -EFAULT;
2990         }
2991         DEBUG(printk("cryptocop_ioctl_create_session, sess_op:\n"));
2992
2993         DEBUG(printk("\tcipher:%d\n"
2994                      "\tcipher_mode:%d\n"
2995                      "\tdigest:%d\n"
2996                      "\tcsum:%d\n",
2997                      (int)sop.cipher,
2998                      (int)sop.cmode,
2999                      (int)sop.digest,
3000                      (int)sop.csum));
3001
3002         if (sop.cipher != cryptocop_cipher_none){
3003                 /* Init the cipher. */
3004                 switch (sop.cipher){
3005                 case cryptocop_cipher_des:
3006                         ti_cipher.alg = cryptocop_alg_des;
3007                         break;
3008                 case cryptocop_cipher_3des:
3009                         ti_cipher.alg = cryptocop_alg_3des;
3010                         break;
3011                 case cryptocop_cipher_aes:
3012                         ti_cipher.alg = cryptocop_alg_aes;
3013                         break;
3014                 default:
3015                         DEBUG_API(printk("create session, bad cipher algorithm %d\n", sop.cipher));
3016                         return -EINVAL;
3017                 };
3018                 DEBUG(printk("setting cipher transform %d\n", ti_cipher.alg));
3019                 copy_from_user(ti_cipher.key, sop.key, sop.keylen/8);
3020                 ti_cipher.keylen = sop.keylen;
3021                 switch (sop.cmode){
3022                 case cryptocop_cipher_mode_cbc:
3023                 case cryptocop_cipher_mode_ecb:
3024                         ti_cipher.cipher_mode = sop.cmode;
3025                         break;
3026                 default:
3027                         DEBUG_API(printk("create session, bad cipher mode %d\n", sop.cmode));
3028                         return -EINVAL;
3029                 }
3030                 DEBUG(printk("cryptocop_ioctl_create_session: setting CBC mode %d\n", ti_cipher.cipher_mode));
3031                 switch (sop.des3_mode){
3032                 case cryptocop_3des_eee:
3033                 case cryptocop_3des_eed:
3034                 case cryptocop_3des_ede:
3035                 case cryptocop_3des_edd:
3036                 case cryptocop_3des_dee:
3037                 case cryptocop_3des_ded:
3038                 case cryptocop_3des_dde:
3039                 case cryptocop_3des_ddd:
3040                         ti_cipher.tdes_mode = sop.des3_mode;
3041                         break;
3042                 default:
3043                         DEBUG_API(printk("create session, bad 3DES mode %d\n", sop.des3_mode));
3044                         return -EINVAL;
3045                 }
3046                 ti_cipher.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
3047                 ti_cipher.next = tis;
3048                 tis = &ti_cipher;
3049         } /* if (sop.cipher != cryptocop_cipher_none) */
3050         if (sop.digest != cryptocop_digest_none){
3051                 DEBUG(printk("setting digest transform\n"));
3052                 switch (sop.digest){
3053                 case cryptocop_digest_md5:
3054                         ti_digest.alg = cryptocop_alg_md5;
3055                         break;
3056                 case cryptocop_digest_sha1:
3057                         ti_digest.alg = cryptocop_alg_sha1;
3058                         break;
3059                 default:
3060                         DEBUG_API(printk("create session, bad digest algorithm %d\n", sop.digest));
3061                         return -EINVAL;
3062                 }
3063                 ti_digest.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
3064                 ti_digest.next = tis;
3065                 tis = &ti_digest;
3066         } /* if (sop.digest != cryptocop_digest_none) */
3067         if (sop.csum != cryptocop_csum_none){
3068                 DEBUG(printk("setting csum transform\n"));
3069                 switch (sop.csum){
3070                 case cryptocop_csum_le:
3071                 case cryptocop_csum_be:
3072                         ti_csum.csum_mode = sop.csum;
3073                         break;
3074                 default:
3075                         DEBUG_API(printk("create session, bad checksum algorithm %d\n", sop.csum));
3076                         return -EINVAL;
3077                 }
3078                 ti_csum.alg = cryptocop_alg_csum;
3079                 ti_csum.tid = CRYPTOCOP_IOCTL_CSUM_TID;
3080                 ti_csum.next = tis;
3081                 tis = &ti_csum;
3082         } /* (sop.csum != cryptocop_csum_none) */
3083         dev = kmalloc(sizeof(struct cryptocop_private), GFP_KERNEL);
3084         if (!dev){
3085                 DEBUG_API(printk("create session, alloc dev\n"));
3086                 return -ENOMEM;
3087         }
3088
3089         err = cryptocop_new_session(&sid, tis, GFP_KERNEL);
3090         DEBUG({ if (err) printk("create session, cryptocop_new_session %d\n", err);});
3091
3092         if (err) {
3093                 kfree(dev);
3094                 return err;
3095         }
3096         sess_op->ses_id = sid;
3097         dev->sid = sid;
3098         dev->next = filp->private_data;
3099         filp->private_data = dev;
3100
3101         return 0;
3102 }
3103
3104 static int cryptocop_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
3105 {
3106         int err = 0;
3107         if (_IOC_TYPE(cmd) != ETRAXCRYPTOCOP_IOCTYPE) {
3108                 DEBUG_API(printk("cryptocop_ioctl: wrong type\n"));
3109                 return -ENOTTY;
3110         }
3111         if (_IOC_NR(cmd) > CRYPTOCOP_IO_MAXNR){
3112                 return -ENOTTY;
3113         }
3114         /* Access check of the argument.  Some commands, e.g. create session and process op,
3115            needs additional checks.  Those are handled in the command handling functions. */
3116         if (_IOC_DIR(cmd) & _IOC_READ)
3117                 err = !access_ok(VERIFY_WRITE, (void *)arg, _IOC_SIZE(cmd));
3118         else if (_IOC_DIR(cmd) & _IOC_WRITE)
3119                 err = !access_ok(VERIFY_READ, (void *)arg, _IOC_SIZE(cmd));
3120         if (err) return -EFAULT;
3121
3122         switch (cmd) {
3123         case CRYPTOCOP_IO_CREATE_SESSION:
3124                 return cryptocop_ioctl_create_session(inode, filp, cmd, arg);
3125         case CRYPTOCOP_IO_CLOSE_SESSION:
3126                 return cryptocop_ioctl_close_session(inode, filp, cmd, arg);
3127         case CRYPTOCOP_IO_PROCESS_OP:
3128                 return cryptocop_ioctl_process(inode, filp, cmd, arg);
3129         default:
3130                 DEBUG_API(printk("cryptocop_ioctl: unknown command\n"));
3131                 return -ENOTTY;
3132         }
3133         return 0;
3134 }
3135
3136
3137 #ifdef LDEBUG
3138 static void print_dma_descriptors(struct cryptocop_int_operation *iop)
3139 {
3140         struct cryptocop_dma_desc *cdesc_out = iop->cdesc_out;
3141         struct cryptocop_dma_desc *cdesc_in = iop->cdesc_in;
3142         int                       i;
3143
3144         printk("print_dma_descriptors start\n");
3145
3146         printk("iop:\n");
3147         printk("\tsid: 0x%lld\n", iop->sid);
3148
3149         printk("\tcdesc_out: 0x%p\n", iop->cdesc_out);
3150         printk("\tcdesc_in: 0x%p\n", iop->cdesc_in);
3151         printk("\tddesc_out: 0x%p\n", iop->ddesc_out);
3152         printk("\tddesc_in: 0x%p\n", iop->ddesc_in);
3153
3154         printk("\niop->ctx_out: 0x%p phys: 0x%p\n", &iop->ctx_out, (char*)virt_to_phys(&iop->ctx_out));
3155         printk("\tnext: 0x%p\n"
3156                "\tsaved_data: 0x%p\n"
3157                "\tsaved_data_buf: 0x%p\n",
3158                iop->ctx_out.next,
3159                iop->ctx_out.saved_data,
3160                iop->ctx_out.saved_data_buf);
3161
3162         printk("\niop->ctx_in: 0x%p phys: 0x%p\n", &iop->ctx_in, (char*)virt_to_phys(&iop->ctx_in));
3163         printk("\tnext: 0x%p\n"
3164                "\tsaved_data: 0x%p\n"
3165                "\tsaved_data_buf: 0x%p\n",
3166                iop->ctx_in.next,
3167                iop->ctx_in.saved_data,
3168                iop->ctx_in.saved_data_buf);
3169
3170         i = 0;
3171         while (cdesc_out) {
3172                 dma_descr_data *td;
3173                 printk("cdesc_out %d, desc=0x%p\n", i, cdesc_out->dma_descr);
3174                 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_out->dma_descr));
3175                 td = cdesc_out->dma_descr;
3176                 printk("\n\tbuf: 0x%p\n"
3177                        "\tafter: 0x%p\n"
3178                        "\tmd: 0x%04x\n"
3179                        "\tnext: 0x%p\n",
3180                        td->buf,
3181                        td->after,
3182                        td->md,
3183                        td->next);
3184                 printk("flags:\n"
3185                        "\twait:\t%d\n"
3186                        "\teol:\t%d\n"
3187                        "\touteop:\t%d\n"
3188                        "\tineop:\t%d\n"
3189                        "\tintr:\t%d\n",
3190                        td->wait,
3191                        td->eol,
3192                        td->out_eop,
3193                        td->in_eop,
3194                        td->intr);
3195                 cdesc_out = cdesc_out->next;
3196                 i++;
3197         }
3198         i = 0;
3199         while (cdesc_in) {
3200                 dma_descr_data *td;
3201                 printk("cdesc_in %d, desc=0x%p\n", i, cdesc_in->dma_descr);
3202                 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_in->dma_descr));
3203                 td = cdesc_in->dma_descr;
3204                 printk("\n\tbuf: 0x%p\n"
3205                        "\tafter: 0x%p\n"
3206                        "\tmd: 0x%04x\n"
3207                        "\tnext: 0x%p\n",
3208                        td->buf,
3209                        td->after,
3210                        td->md,
3211                        td->next);
3212                 printk("flags:\n"
3213                        "\twait:\t%d\n"
3214                        "\teol:\t%d\n"
3215                        "\touteop:\t%d\n"
3216                        "\tineop:\t%d\n"
3217                        "\tintr:\t%d\n",
3218                        td->wait,
3219                        td->eol,
3220                        td->out_eop,
3221                        td->in_eop,
3222                        td->intr);
3223                 cdesc_in = cdesc_in->next;
3224                 i++;
3225         }
3226
3227         printk("print_dma_descriptors end\n");
3228 }
3229
3230
3231 static void print_strcop_crypto_op(struct strcop_crypto_op *cop)
3232 {
3233         printk("print_strcop_crypto_op, 0x%p\n", cop);
3234
3235         /* Indata. */
3236         printk("indata=0x%p\n"
3237                "inlen=%d\n"
3238                "do_cipher=%d\n"
3239                "decrypt=%d\n"
3240                "cipher_explicit=%d\n"
3241                "cipher_start=%d\n"
3242                "cipher_len=%d\n"
3243                "outdata=0x%p\n"
3244                "outlen=%d\n",
3245                cop->indata,
3246                cop->inlen,
3247                cop->do_cipher,
3248                cop->decrypt,
3249                cop->cipher_explicit,
3250                cop->cipher_start,
3251                cop->cipher_len,
3252                cop->cipher_outdata,
3253                cop->cipher_outlen);
3254
3255         printk("do_digest=%d\n"
3256                "digest_start=%d\n"
3257                "digest_len=%d\n",
3258                cop->do_digest,
3259                cop->digest_start,
3260                cop->digest_len);
3261
3262         printk("do_csum=%d\n"
3263                "csum_start=%d\n"
3264                "csum_len=%d\n",
3265                cop->do_csum,
3266                cop->csum_start,
3267                cop->csum_len);
3268 }
3269
3270 static void print_cryptocop_operation(struct cryptocop_operation *cop)
3271 {
3272         struct cryptocop_desc      *d;
3273         struct cryptocop_tfrm_cfg  *tc;
3274         struct cryptocop_desc_cfg  *dc;
3275         int                        i;
3276
3277         printk("print_cryptocop_operation, cop=0x%p\n\n", cop);
3278         printk("sid: %lld\n", cop->sid);
3279         printk("operation_status=%d\n"
3280                "use_dmalists=%d\n"
3281                "in_interrupt=%d\n"
3282                "fast_callback=%d\n",
3283                cop->operation_status,
3284                cop->use_dmalists,
3285                cop->in_interrupt,
3286                cop->fast_callback);
3287
3288         if (cop->use_dmalists){
3289                 print_user_dma_lists(&cop->list_op);
3290         } else {
3291                 printk("cop->tfrm_op\n"
3292                        "tfrm_cfg=0x%p\n"
3293                        "desc=0x%p\n"
3294                        "indata=0x%p\n"
3295                        "incount=%d\n"
3296                        "inlen=%d\n"
3297                        "outdata=0x%p\n"
3298                        "outcount=%d\n"
3299                        "outlen=%d\n\n",
3300                        cop->tfrm_op.tfrm_cfg,
3301                        cop->tfrm_op.desc,
3302                        cop->tfrm_op.indata,
3303                        cop->tfrm_op.incount,
3304                        cop->tfrm_op.inlen,
3305                        cop->tfrm_op.outdata,
3306                        cop->tfrm_op.outcount,
3307                        cop->tfrm_op.outlen);
3308
3309                 tc = cop->tfrm_op.tfrm_cfg;
3310                 while (tc){
3311                         printk("tfrm_cfg, 0x%p\n"
3312                                "tid=%d\n"
3313                                "flags=%d\n"
3314                                "inject_ix=%d\n"
3315                                "next=0x%p\n",
3316                                tc,
3317                                tc->tid,
3318                                tc->flags,
3319                                tc->inject_ix,
3320                                tc->next);
3321                         tc = tc->next;
3322                 }
3323                 d = cop->tfrm_op.desc;
3324                 while (d){
3325                         printk("\n======================desc, 0x%p\n"
3326                                "length=%d\n"
3327                                "cfg=0x%p\n"
3328                                "next=0x%p\n",
3329                                d,
3330                                d->length,
3331                                d->cfg,
3332                                d->next);
3333                         dc = d->cfg;
3334                         while (dc){
3335                                 printk("=========desc_cfg, 0x%p\n"
3336                                        "tid=%d\n"
3337                                        "src=%d\n"
3338                                        "last=%d\n"
3339                                        "next=0x%p\n",
3340                                        dc,
3341                                        dc->tid,
3342                                        dc->src,
3343                                        dc->last,
3344                                        dc->next);
3345                                 dc = dc->next;
3346                         }
3347                         d = d->next;
3348                 }
3349                 printk("\n====iniov\n");
3350                 for (i = 0; i < cop->tfrm_op.incount; i++){
3351                         printk("indata[%d]\n"
3352                                "base=0x%p\n"
3353                                "len=%d\n",
3354                                i,
3355                                cop->tfrm_op.indata[i].iov_base,
3356                                cop->tfrm_op.indata[i].iov_len);
3357                 }
3358                 printk("\n====outiov\n");
3359                 for (i = 0; i < cop->tfrm_op.outcount; i++){
3360                         printk("outdata[%d]\n"
3361                                "base=0x%p\n"
3362                                "len=%d\n",
3363                                i,
3364                                cop->tfrm_op.outdata[i].iov_base,
3365                                cop->tfrm_op.outdata[i].iov_len);
3366                 }
3367         }
3368         printk("------------end print_cryptocop_operation\n");
3369 }
3370
3371
3372 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op)
3373 {
3374         dma_descr_data *dd;
3375         int i;
3376
3377         printk("print_user_dma_lists, dma_op=0x%p\n", dma_op);
3378
3379         printk("out_data_buf = 0x%p, phys_to_virt(out_data_buf) = 0x%p\n", dma_op->out_data_buf, phys_to_virt((unsigned long int)dma_op->out_data_buf));
3380         printk("in_data_buf = 0x%p, phys_to_virt(in_data_buf) = 0x%p\n", dma_op->in_data_buf, phys_to_virt((unsigned long int)dma_op->in_data_buf));
3381
3382         printk("##############outlist\n");
3383         dd = phys_to_virt((unsigned long int)dma_op->outlist);
3384         i = 0;
3385         while (dd != NULL) {
3386                 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3387                 printk("\n\tbuf: 0x%p\n"
3388                        "\tafter: 0x%p\n"
3389                        "\tmd: 0x%04x\n"
3390                        "\tnext: 0x%p\n",
3391                        dd->buf,
3392                        dd->after,
3393                        dd->md,
3394                        dd->next);
3395                 printk("flags:\n"
3396                        "\twait:\t%d\n"
3397                        "\teol:\t%d\n"
3398                        "\touteop:\t%d\n"
3399                        "\tineop:\t%d\n"
3400                        "\tintr:\t%d\n",
3401                        dd->wait,
3402                        dd->eol,
3403                        dd->out_eop,
3404                        dd->in_eop,
3405                        dd->intr);
3406                 if (dd->eol)
3407                         dd = NULL;
3408                 else
3409                         dd = phys_to_virt((unsigned long int)dd->next);
3410                 ++i;
3411         }
3412
3413         printk("##############inlist\n");
3414         dd = phys_to_virt((unsigned long int)dma_op->inlist);
3415         i = 0;
3416         while (dd != NULL) {
3417                 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3418                 printk("\n\tbuf: 0x%p\n"
3419                        "\tafter: 0x%p\n"
3420                        "\tmd: 0x%04x\n"
3421                        "\tnext: 0x%p\n",
3422                        dd->buf,
3423                        dd->after,
3424                        dd->md,
3425                        dd->next);
3426                 printk("flags:\n"
3427                        "\twait:\t%d\n"
3428                        "\teol:\t%d\n"
3429                        "\touteop:\t%d\n"
3430                        "\tineop:\t%d\n"
3431                        "\tintr:\t%d\n",
3432                        dd->wait,
3433                        dd->eol,
3434                        dd->out_eop,
3435                        dd->in_eop,
3436                        dd->intr);
3437                 if (dd->eol)
3438                         dd = NULL;
3439                 else
3440                         dd = phys_to_virt((unsigned long int)dd->next);
3441                 ++i;
3442         }
3443 }
3444
3445
3446 static void print_lock_status(void)
3447 {
3448         printk("**********************print_lock_status\n");
3449         printk("cryptocop_completed_jobs_lock %d\n", spin_is_locked(&cryptocop_completed_jobs_lock));
3450         printk("cryptocop_job_queue_lock %d\n", spin_is_locked(&cryptocop_job_queue_lock));
3451         printk("descr_pool_lock %d\n", spin_is_locked(&descr_pool_lock));
3452         printk("cryptocop_sessions_lock %d\n", spin_is_locked(cryptocop_sessions_lock));
3453         printk("running_job_lock %d\n", spin_is_locked(running_job_lock));
3454         printk("cryptocop_process_lock %d\n", spin_is_locked(cryptocop_process_lock));
3455 }
3456 #endif /* LDEBUG */
3457
3458
3459 static const char cryptocop_name[] = "ETRAX FS stream co-processor";
3460
3461 static int init_stream_coprocessor(void)
3462 {
3463         int err;
3464         int i;
3465         static int initialized = 0;
3466
3467         if (initialized)
3468                 return 0;
3469
3470         initialized = 1;
3471
3472         printk("ETRAX FS stream co-processor driver v0.01, (c) 2003 Axis Communications AB\n");
3473
3474         err = register_chrdev(CRYPTOCOP_MAJOR, cryptocop_name, &cryptocop_fops);
3475         if (err < 0) {
3476                 printk(KERN_ERR "stream co-processor: could not get major number.\n");
3477                 return err;
3478         }
3479
3480         err = init_cryptocop();
3481         if (err) {
3482                 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3483                 return err;
3484         }
3485         err = cryptocop_job_queue_init();
3486         if (err) {
3487                 release_cryptocop();
3488                 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3489                 return err;
3490         }
3491         /* Init the descriptor pool. */
3492         for (i = 0; i < CRYPTOCOP_DESCRIPTOR_POOL_SIZE - 1; i++) {
3493                 descr_pool[i].from_pool = 1;
3494                 descr_pool[i].next = &descr_pool[i + 1];
3495         }
3496         descr_pool[i].from_pool = 1;
3497         descr_pool[i].next = NULL;
3498         descr_pool_free_list = &descr_pool[0];
3499         descr_pool_no_free = CRYPTOCOP_DESCRIPTOR_POOL_SIZE;
3500
3501         spin_lock_init(&cryptocop_completed_jobs_lock);
3502         spin_lock_init(&cryptocop_job_queue_lock);
3503         spin_lock_init(&descr_pool_lock);
3504         spin_lock_init(&cryptocop_sessions_lock);
3505         spin_lock_init(&running_job_lock);
3506         spin_lock_init(&cryptocop_process_lock);
3507
3508         cryptocop_sessions = NULL;
3509         next_sid = 1;
3510
3511         cryptocop_running_job = NULL;
3512
3513         printk("stream co-processor: init done.\n");
3514         return 0;
3515 }
3516
3517 static void __exit exit_stream_coprocessor(void)
3518 {
3519         release_cryptocop();
3520         cryptocop_job_queue_close();
3521 }
3522
3523 module_init(init_stream_coprocessor);
3524 module_exit(exit_stream_coprocessor);
3525