arch/cris/arch-v32/drivers/cryptocop.c

   1 /*
   2  * Stream co-processor driver for the ETRAX FS
   3  *
   4  *    Copyright (C) 2003-2007  Axis Communications AB
   5  */
   6
   7 #include <linux/init.h>
   8 #include <linux/sched.h>
   9 #include <linux/module.h>
  10 #include <linux/slab.h>
  11 #include <linux/string.h>
  12 #include <linux/fs.h>
  13 #include <linux/mm.h>
  14 #include <linux/spinlock.h>
  15 #include <linux/stddef.h>
  16
  17 #include <asm/uaccess.h>
  18 #include <asm/io.h>
  19 #include <asm/atomic.h>
  20
  21 #include <linux/list.h>
  22 #include <linux/interrupt.h>
  23
  24 #include <asm/signal.h>
  25 #include <asm/irq.h>
  26
  27 #include <dma.h>
  28 #include <hwregs/dma.h>
  29 #include <hwregs/reg_map.h>
  30 #include <hwregs/reg_rdwr.h>
  31 #include <hwregs/intr_vect_defs.h>
  32
  33 #include <hwregs/strcop.h>
  34 #include <hwregs/strcop_defs.h>
  35 #include <cryptocop.h>
  36
  37 #ifdef CONFIG_ETRAXFS
  38 #define IN_DMA 9
  39 #define OUT_DMA 8
  40 #define IN_DMA_INST regi_dma9
  41 #define OUT_DMA_INST regi_dma8
  42 #define DMA_IRQ DMA9_INTR_VECT
  43 #else
  44 #define IN_DMA 3
  45 #define OUT_DMA 2
  46 #define IN_DMA_INST regi_dma3
  47 #define OUT_DMA_INST regi_dma2
  48 #define DMA_IRQ DMA3_INTR_VECT
  49 #endif
  50
  51 #define DESCR_ALLOC_PAD  (31)
  52
  53 struct cryptocop_dma_desc {
  54         char *free_buf; /* If non-null will be kfreed in free_cdesc() */
  55         dma_descr_data *dma_descr;
  56
  57         unsigned char dma_descr_buf[sizeof(dma_descr_data) + DESCR_ALLOC_PAD];
  58
  59         unsigned int from_pool:1; /* If 1 'allocated' from the descriptor pool. */
  60         struct cryptocop_dma_desc *next;
  61 };
  62
  63
  64 struct cryptocop_int_operation{
  65         void                        *alloc_ptr;
  66         cryptocop_session_id        sid;
  67
  68         dma_descr_context           ctx_out;
  69         dma_descr_context           ctx_in;
  70
  71         /* DMA descriptors allocated by driver. */
  72         struct cryptocop_dma_desc   *cdesc_out;
  73         struct cryptocop_dma_desc   *cdesc_in;
  74
  75         /* Strcop config to use. */
  76         cryptocop_3des_mode         tdes_mode;
  77         cryptocop_csum_type         csum_mode;
  78
  79         /* DMA descrs provided by consumer. */
  80         dma_descr_data              *ddesc_out;
  81         dma_descr_data              *ddesc_in;
  82 };
  83
  84
  85 struct cryptocop_tfrm_ctx {
  86         cryptocop_tfrm_id tid;
  87         unsigned int blocklength;
  88
  89         unsigned int start_ix;
  90
  91         struct cryptocop_tfrm_cfg *tcfg;
  92         struct cryptocop_transform_ctx *tctx;
  93
  94         unsigned char previous_src;
  95         unsigned char current_src;
  96
  97         /* Values to use in metadata out. */
  98         unsigned char hash_conf;
  99         unsigned char hash_mode;
 100         unsigned char ciph_conf;
 101         unsigned char cbcmode;
 102         unsigned char decrypt;
 103
 104         unsigned int requires_padding:1;
 105         unsigned int strict_block_length:1;
 106         unsigned int active:1;
 107         unsigned int done:1;
 108         size_t consumed;
 109         size_t produced;
 110
 111         /* Pad (input) descriptors to put in the DMA out list when the transform
 112          * output is put on the DMA in list. */
 113         struct cryptocop_dma_desc *pad_descs;
 114
 115         struct cryptocop_tfrm_ctx *prev_src;
 116         struct cryptocop_tfrm_ctx *curr_src;
 117
 118         /* Mapping to HW. */
 119         unsigned char unit_no;
 120 };
 121
 122
 123 struct cryptocop_private{
 124         cryptocop_session_id sid;
 125         struct cryptocop_private *next;
 126 };
 127
 128 /* Session list. */
 129
 130 struct cryptocop_transform_ctx{
 131         struct cryptocop_transform_init init;
 132         unsigned char dec_key[CRYPTOCOP_MAX_KEY_LENGTH];
 133         unsigned int dec_key_set:1;
 134
 135         struct cryptocop_transform_ctx *next;
 136 };
 137
 138
 139 struct cryptocop_session{
 140         cryptocop_session_id sid;
 141
 142         struct cryptocop_transform_ctx *tfrm_ctx;
 143
 144         struct cryptocop_session *next;
 145 };
 146
 147 /* Priority levels for jobs sent to the cryptocop.  Checksum operations from
 148    kernel have highest priority since TCPIP stack processing must not
 149    be a bottleneck. */
 150 typedef enum {
 151         cryptocop_prio_kernel_csum = 0,
 152         cryptocop_prio_kernel = 1,
 153         cryptocop_prio_user = 2,
 154         cryptocop_prio_no_prios = 3
 155 } cryptocop_queue_priority;
 156
 157 struct cryptocop_prio_queue{
 158         struct list_head jobs;
 159         cryptocop_queue_priority prio;
 160 };
 161
 162 struct cryptocop_prio_job{
 163         struct list_head node;
 164         cryptocop_queue_priority prio;
 165
 166         struct cryptocop_operation *oper;
 167         struct cryptocop_int_operation *iop;
 168 };
 169
 170 struct ioctl_job_cb_ctx {
 171         unsigned int processed:1;
 172 };
 173
 174
 175 static struct cryptocop_session *cryptocop_sessions = NULL;
 176 spinlock_t cryptocop_sessions_lock;
 177
 178 /* Next Session ID to assign. */
 179 static cryptocop_session_id next_sid = 1;
 180
 181 /* Pad for checksum. */
 182 static const char csum_zero_pad[1] = {0x00};
 183
 184 /* Trash buffer for mem2mem operations. */
 185 #define MEM2MEM_DISCARD_BUF_LENGTH  (512)
 186 static unsigned char mem2mem_discard_buf[MEM2MEM_DISCARD_BUF_LENGTH];
 187
 188 /* Descriptor pool. */
 189 /* FIXME Tweak this value. */
 190 #define CRYPTOCOP_DESCRIPTOR_POOL_SIZE   (100)
 191 static struct cryptocop_dma_desc descr_pool[CRYPTOCOP_DESCRIPTOR_POOL_SIZE];
 192 static struct cryptocop_dma_desc *descr_pool_free_list;
 193 static int descr_pool_no_free;
 194 static spinlock_t descr_pool_lock;
 195
 196 /* Lock to stop cryptocop to start processing of a new operation. The holder
 197    of this lock MUST call cryptocop_start_job() after it is unlocked. */
 198 spinlock_t cryptocop_process_lock;
 199
 200 static struct cryptocop_prio_queue cryptocop_job_queues[cryptocop_prio_no_prios];
 201 static spinlock_t cryptocop_job_queue_lock;
 202 static struct cryptocop_prio_job *cryptocop_running_job = NULL;
 203 static spinlock_t running_job_lock;
 204
 205 /* The interrupt handler appends completed jobs to this list. The scehduled
 206  * tasklet removes them upon sending the response to the crypto consumer. */
 207 static struct list_head cryptocop_completed_jobs;
 208 static spinlock_t cryptocop_completed_jobs_lock;
 209
 210 DECLARE_WAIT_QUEUE_HEAD(cryptocop_ioc_process_wq);
 211
 212
 213 /** Local functions. **/
 214
 215 static int cryptocop_open(struct inode *, struct file *);
 216
 217 static int cryptocop_release(struct inode *, struct file *);
 218
 219 static long cryptocop_ioctl(struct file *file,
 220                            unsigned int cmd, unsigned long arg);
 221
 222 static void cryptocop_start_job(void);
 223
 224 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation);
 225 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation);
 226
 227 static int cryptocop_job_queue_init(void);
 228 static void cryptocop_job_queue_close(void);
 229
 230 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
 231
 232 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
 233
 234 static int transform_ok(struct cryptocop_transform_init *tinit);
 235
 236 static struct cryptocop_session *get_session(cryptocop_session_id sid);
 237
 238 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid);
 239
 240 static void delete_internal_operation(struct cryptocop_int_operation *iop);
 241
 242 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned  char *key, unsigned int keylength);
 243
 244 static int init_stream_coprocessor(void);
 245
 246 static void __exit exit_stream_coprocessor(void);
 247
 248 /*#define LDEBUG*/
 249 #ifdef LDEBUG
 250 #define DEBUG(s) s
 251 #define DEBUG_API(s) s
 252 static void print_cryptocop_operation(struct cryptocop_operation *cop);
 253 static void print_dma_descriptors(struct cryptocop_int_operation *iop);
 254 static void print_strcop_crypto_op(struct strcop_crypto_op *cop);
 255 static void print_lock_status(void);
 256 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op);
 257 #define assert(s) do{if (!(s)) panic(#s);} while(0);
 258 #else
 259 #define DEBUG(s)
 260 #define DEBUG_API(s)
 261 #define assert(s)
 262 #endif
 263
 264
 265 /* Transform constants. */
 266 #define DES_BLOCK_LENGTH   (8)
 267 #define AES_BLOCK_LENGTH   (16)
 268 #define MD5_BLOCK_LENGTH   (64)
 269 #define SHA1_BLOCK_LENGTH  (64)
 270 #define CSUM_BLOCK_LENGTH  (2)
 271 #define MD5_STATE_LENGTH   (16)
 272 #define SHA1_STATE_LENGTH  (20)
 273
 274 /* The device number. */
 275 #define CRYPTOCOP_MAJOR    (254)
 276 #define CRYPTOCOP_MINOR    (0)
 277
 278
 279
 280 const struct file_operations cryptocop_fops = {
 281         .owner          = THIS_MODULE,
 282         .open           = cryptocop_open,
 283         .release        = cryptocop_release,
 284         .unlocked_ioctl = cryptocop_ioctl
 285 };
 286
 287
 288 static void free_cdesc(struct cryptocop_dma_desc *cdesc)
 289 {
 290         DEBUG(printk("free_cdesc: cdesc 0x%p, from_pool=%d\n", cdesc, cdesc->from_pool));
 291         kfree(cdesc->free_buf);
 292
 293         if (cdesc->from_pool) {
 294                 unsigned long int flags;
 295                 spin_lock_irqsave(&descr_pool_lock, flags);
 296                 cdesc->next = descr_pool_free_list;
 297                 descr_pool_free_list = cdesc;
 298                 ++descr_pool_no_free;
 299                 spin_unlock_irqrestore(&descr_pool_lock, flags);
 300         } else {
 301                 kfree(cdesc);
 302         }
 303 }
 304
 305
 306 static struct cryptocop_dma_desc *alloc_cdesc(int alloc_flag)
 307 {
 308         int use_pool = (alloc_flag & GFP_ATOMIC) ? 1 : 0;
 309         struct cryptocop_dma_desc *cdesc;
 310
 311         if (use_pool) {
 312                 unsigned long int flags;
 313                 spin_lock_irqsave(&descr_pool_lock, flags);
 314                 if (!descr_pool_free_list) {
 315                         spin_unlock_irqrestore(&descr_pool_lock, flags);
 316                         DEBUG_API(printk("alloc_cdesc: pool is empty\n"));
 317                         return NULL;
 318                 }
 319                 cdesc = descr_pool_free_list;
 320                 descr_pool_free_list = descr_pool_free_list->next;
 321                 --descr_pool_no_free;
 322                 spin_unlock_irqrestore(&descr_pool_lock, flags);
 323                 cdesc->from_pool = 1;
 324         } else {
 325                 cdesc = kmalloc(sizeof(struct cryptocop_dma_desc), alloc_flag);
 326                 if (!cdesc) {
 327                         DEBUG_API(printk("alloc_cdesc: kmalloc\n"));
 328                         return NULL;
 329                 }
 330                 cdesc->from_pool = 0;
 331         }
 332         cdesc->dma_descr = (dma_descr_data*)(((unsigned long int)cdesc + offsetof(struct cryptocop_dma_desc, dma_descr_buf) + DESCR_ALLOC_PAD) & ~0x0000001F);
 333
 334         cdesc->next = NULL;
 335
 336         cdesc->free_buf = NULL;
 337         cdesc->dma_descr->out_eop = 0;
 338         cdesc->dma_descr->in_eop = 0;
 339         cdesc->dma_descr->intr = 0;
 340         cdesc->dma_descr->eol = 0;
 341         cdesc->dma_descr->wait = 0;
 342         cdesc->dma_descr->buf = NULL;
 343         cdesc->dma_descr->after = NULL;
 344
 345         DEBUG_API(printk("alloc_cdesc: return 0x%p, cdesc->dma_descr=0x%p, from_pool=%d\n", cdesc, cdesc->dma_descr, cdesc->from_pool));
 346         return cdesc;
 347 }
 348
 349
 350 static void setup_descr_chain(struct cryptocop_dma_desc *cd)
 351 {
 352         DEBUG(printk("setup_descr_chain: entering\n"));
 353         while (cd) {
 354                 if (cd->next) {
 355                         cd->dma_descr->next = (dma_descr_data*)virt_to_phys(cd->next->dma_descr);
 356                 } else {
 357                         cd->dma_descr->next = NULL;
 358                 }
 359                 cd = cd->next;
 360         }
 361         DEBUG(printk("setup_descr_chain: exit\n"));
 362 }
 363
 364
 365 /* Create a pad descriptor for the transform.
 366  * Return -1 for error, 0 if pad created. */
 367 static int create_pad_descriptor(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **pad_desc, int alloc_flag)
 368 {
 369         struct cryptocop_dma_desc        *cdesc = NULL;
 370         int                              error = 0;
 371         struct strcop_meta_out           mo = {
 372                 .ciphsel = src_none,
 373                 .hashsel = src_none,
 374                 .csumsel = src_none
 375         };
 376         char                             *pad;
 377         size_t                           plen;
 378
 379         DEBUG(printk("create_pad_descriptor: start.\n"));
 380         /* Setup pad descriptor. */
 381
 382         DEBUG(printk("create_pad_descriptor: setting up padding.\n"));
 383         cdesc = alloc_cdesc(alloc_flag);
 384         if (!cdesc){
 385                 DEBUG_API(printk("create_pad_descriptor: alloc pad desc\n"));
 386                 goto error_cleanup;
 387         }
 388         switch (tc->unit_no) {
 389         case src_md5:
 390                 error = create_md5_pad(alloc_flag, tc->consumed, &pad, &plen);
 391                 if (error){
 392                         DEBUG_API(printk("create_pad_descriptor: create_md5_pad_failed\n"));
 393                         goto error_cleanup;
 394                 }
 395                 cdesc->free_buf = pad;
 396                 mo.hashsel = src_dma;
 397                 mo.hashconf = tc->hash_conf;
 398                 mo.hashmode = tc->hash_mode;
 399                 break;
 400         case src_sha1:
 401                 error = create_sha1_pad(alloc_flag, tc->consumed, &pad, &plen);
 402                 if (error){
 403                         DEBUG_API(printk("create_pad_descriptor: create_sha1_pad_failed\n"));
 404                         goto error_cleanup;
 405                 }
 406                 cdesc->free_buf = pad;
 407                 mo.hashsel = src_dma;
 408                 mo.hashconf = tc->hash_conf;
 409                 mo.hashmode = tc->hash_mode;
 410                 break;
 411         case src_csum:
 412                 if (tc->consumed % tc->blocklength){
 413                         pad = (char*)csum_zero_pad;
 414                         plen = 1;
 415                 } else {
 416                         pad = (char*)cdesc; /* Use any pointer. */
 417                         plen = 0;
 418                 }
 419                 mo.csumsel = src_dma;
 420                 break;
 421         }
 422         cdesc->dma_descr->wait = 1;
 423         cdesc->dma_descr->out_eop = 1; /* Since this is a pad output is pushed.  EOP is ok here since the padded unit is the only one active. */
 424         cdesc->dma_descr->buf = (char*)virt_to_phys((char*)pad);
 425         cdesc->dma_descr->after = cdesc->dma_descr->buf + plen;
 426
 427         cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
 428         *pad_desc = cdesc;
 429
 430         return 0;
 431
 432  error_cleanup:
 433         if (cdesc) free_cdesc(cdesc);
 434         return -1;
 435 }
 436
 437
 438 static int setup_key_dl_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **kd, int alloc_flag)
 439 {
 440         struct cryptocop_dma_desc  *key_desc = alloc_cdesc(alloc_flag);
 441         struct strcop_meta_out     mo = {0};
 442
 443         DEBUG(printk("setup_key_dl_desc\n"));
 444
 445         if (!key_desc) {
 446                 DEBUG_API(printk("setup_key_dl_desc: failed descriptor allocation.\n"));
 447                 return -ENOMEM;
 448         }
 449
 450         /* Download key. */
 451         if ((tc->tctx->init.alg == cryptocop_alg_aes) && (tc->tcfg->flags & CRYPTOCOP_DECRYPT)) {
 452                 /* Precook the AES decrypt key. */
 453                 if (!tc->tctx->dec_key_set){
 454                         get_aes_decrypt_key(tc->tctx->dec_key, tc->tctx->init.key, tc->tctx->init.keylen);
 455                         tc->tctx->dec_key_set = 1;
 456                 }
 457                 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->dec_key);
 458                 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
 459         } else {
 460                 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->init.key);
 461                 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
 462         }
 463         /* Setup metadata. */
 464         mo.dlkey = 1;
 465         switch (tc->tctx->init.keylen) {
 466         case 64:
 467                 mo.decrypt = 0;
 468                 mo.hashmode = 0;
 469                 break;
 470         case 128:
 471                 mo.decrypt = 0;
 472                 mo.hashmode = 1;
 473                 break;
 474         case 192:
 475                 mo.decrypt = 1;
 476                 mo.hashmode = 0;
 477                 break;
 478         case 256:
 479                 mo.decrypt = 1;
 480                 mo.hashmode = 1;
 481                 break;
 482         default:
 483                 break;
 484         }
 485         mo.ciphsel = mo.hashsel = mo.csumsel = src_none;
 486         key_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
 487
 488         key_desc->dma_descr->out_eop = 1;
 489         key_desc->dma_descr->wait = 1;
 490         key_desc->dma_descr->intr = 0;
 491
 492         *kd = key_desc;
 493         return 0;
 494 }
 495
 496 static int setup_cipher_iv_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
 497 {
 498         struct cryptocop_dma_desc  *iv_desc = alloc_cdesc(alloc_flag);
 499         struct strcop_meta_out     mo = {0};
 500
 501         DEBUG(printk("setup_cipher_iv_desc\n"));
 502
 503         if (!iv_desc) {
 504                 DEBUG_API(printk("setup_cipher_iv_desc: failed CBC IV descriptor allocation.\n"));
 505                 return -ENOMEM;
 506         }
 507         /* Download IV. */
 508         iv_desc->dma_descr->buf = (char*)virt_to_phys(tc->tcfg->iv);
 509         iv_desc->dma_descr->after = iv_desc->dma_descr->buf + tc->blocklength;
 510
 511         /* Setup metadata. */
 512         mo.hashsel = mo.csumsel = src_none;
 513         mo.ciphsel = src_dma;
 514         mo.ciphconf = tc->ciph_conf;
 515         mo.cbcmode = tc->cbcmode;
 516
 517         iv_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
 518
 519         iv_desc->dma_descr->out_eop = 0;
 520         iv_desc->dma_descr->wait = 1;
 521         iv_desc->dma_descr->intr = 0;
 522
 523         *id = iv_desc;
 524         return 0;
 525 }
 526
 527 /* Map the ouput length of the transform to operation output starting on the inject index. */
 528 static int create_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
 529 {
 530         int                        err = 0;
 531         struct cryptocop_dma_desc  head = {0};
 532         struct cryptocop_dma_desc  *outdesc = &head;
 533         size_t                     iov_offset = 0;
 534         size_t                     out_ix = 0;
 535         int                        outiov_ix = 0;
 536         struct strcop_meta_in      mi = {0};
 537
 538         size_t                     out_length = tc->produced;
 539         int                        rem_length;
 540         int                        dlength;
 541
 542         assert(out_length != 0);
 543         if (((tc->produced + tc->tcfg->inject_ix) > operation->tfrm_op.outlen) || (tc->produced && (operation->tfrm_op.outlen == 0))) {
 544                 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
 545                 return -EINVAL;
 546         }
 547         /* Traverse the out iovec until the result inject index is reached. */
 548         while ((outiov_ix < operation->tfrm_op.outcount) && ((out_ix + operation->tfrm_op.outdata[outiov_ix].iov_len) <= tc->tcfg->inject_ix)){
 549                 out_ix += operation->tfrm_op.outdata[outiov_ix].iov_len;
 550                 outiov_ix++;
 551         }
 552         if (outiov_ix >= operation->tfrm_op.outcount){
 553                 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
 554                 return -EINVAL;
 555         }
 556         iov_offset = tc->tcfg->inject_ix - out_ix;
 557         mi.dmasel = tc->unit_no;
 558
 559         /* Setup the output descriptors. */
 560         while ((out_length > 0) && (outiov_ix < operation->tfrm_op.outcount)) {
 561                 outdesc->next = alloc_cdesc(alloc_flag);
 562                 if (!outdesc->next) {
 563                         DEBUG_API(printk("create_input_descriptors: alloc_cdesc\n"));
 564                         err = -ENOMEM;
 565                         goto error_cleanup;
 566                 }
 567                 outdesc = outdesc->next;
 568                 rem_length = operation->tfrm_op.outdata[outiov_ix].iov_len - iov_offset;
 569                 dlength = (out_length < rem_length) ? out_length : rem_length;
 570
 571                 DEBUG(printk("create_input_descriptors:\n"
 572                              "outiov_ix=%d, rem_length=%d, dlength=%d\n"
 573                              "iov_offset=%d, outdata[outiov_ix].iov_len=%d\n"
 574                              "outcount=%d, outiov_ix=%d\n",
 575                              outiov_ix, rem_length, dlength, iov_offset, operation->tfrm_op.outdata[outiov_ix].iov_len, operation->tfrm_op.outcount, outiov_ix));
 576
 577                 outdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.outdata[outiov_ix].iov_base + iov_offset);
 578                 outdesc->dma_descr->after = outdesc->dma_descr->buf + dlength;
 579                 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 580
 581                 out_length -= dlength;
 582                 iov_offset += dlength;
 583                 if (iov_offset >= operation->tfrm_op.outdata[outiov_ix].iov_len) {
 584                         iov_offset = 0;
 585                         ++outiov_ix;
 586                 }
 587         }
 588         if (out_length > 0){
 589                 DEBUG_API(printk("create_input_descriptors: not enough room for output, %d remained\n", out_length));
 590                 err = -EINVAL;
 591                 goto error_cleanup;
 592         }
 593         /* Set sync in last descriptor. */
 594         mi.sync = 1;
 595         outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 596
 597         *id = head.next;
 598         return 0;
 599
 600  error_cleanup:
 601         while (head.next) {
 602                 outdesc = head.next->next;
 603                 free_cdesc(head.next);
 604                 head.next = outdesc;
 605         }
 606         return err;
 607 }
 608
 609
 610 static int create_output_descriptors(struct cryptocop_operation *operation, int *iniov_ix, int *iniov_offset, size_t desc_len, struct cryptocop_dma_desc **current_out_cdesc, struct strcop_meta_out *meta_out, int alloc_flag)
 611 {
 612         while (desc_len != 0) {
 613                 struct cryptocop_dma_desc  *cdesc;
 614                 int                        rem_length = operation->tfrm_op.indata[*iniov_ix].iov_len - *iniov_offset;
 615                 int                        dlength = (desc_len < rem_length) ? desc_len : rem_length;
 616
 617                 cdesc = alloc_cdesc(alloc_flag);
 618                 if (!cdesc) {
 619                         DEBUG_API(printk("create_output_descriptors: alloc_cdesc\n"));
 620                         return -ENOMEM;
 621                 }
 622                 (*current_out_cdesc)->next = cdesc;
 623                 (*current_out_cdesc) = cdesc;
 624
 625                 cdesc->free_buf = NULL;
 626
 627                 cdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.indata[*iniov_ix].iov_base + *iniov_offset);
 628                 cdesc->dma_descr->after = cdesc->dma_descr->buf + dlength;
 629
 630                 assert(desc_len >= dlength);
 631                 desc_len -= dlength;
 632                 *iniov_offset += dlength;
 633                 if (*iniov_offset >= operation->tfrm_op.indata[*iniov_ix].iov_len) {
 634                         *iniov_offset = 0;
 635                         ++(*iniov_ix);
 636                         if (*iniov_ix > operation->tfrm_op.incount) {
 637                                 DEBUG_API(printk("create_output_descriptors: not enough indata in operation."));
 638                                 return  -EINVAL;
 639                         }
 640                 }
 641                 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, (*meta_out));
 642         } /* while (desc_len != 0) */
 643         /* Last DMA descriptor gets a 'wait' bit to signal expected change in metadata. */
 644         (*current_out_cdesc)->dma_descr->wait = 1; /* This will set extraneous WAIT in some situations, e.g. when padding hashes and checksums. */
 645
 646         return 0;
 647 }
 648
 649
 650 static int append_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_dma_desc **current_in_cdesc, struct cryptocop_dma_desc **current_out_cdesc, struct cryptocop_tfrm_ctx *tc, int alloc_flag)
 651 {
 652         DEBUG(printk("append_input_descriptors, tc=0x%p, unit_no=%d\n", tc, tc->unit_no));
 653         if (tc->tcfg) {
 654                 int                        failed = 0;
 655                 struct cryptocop_dma_desc  *idescs = NULL;
 656                 DEBUG(printk("append_input_descriptors: pushing output, consumed %d produced %d bytes.\n", tc->consumed, tc->produced));
 657                 if (tc->pad_descs) {
 658                         DEBUG(printk("append_input_descriptors: append pad descriptors to DMA out list.\n"));
 659                         while (tc->pad_descs) {
 660                                 DEBUG(printk("append descriptor 0x%p\n", tc->pad_descs));
 661                                 (*current_out_cdesc)->next = tc->pad_descs;
 662                                 tc->pad_descs = tc->pad_descs->next;
 663                                 (*current_out_cdesc) = (*current_out_cdesc)->next;
 664                         }
 665                 }
 666
 667                 /* Setup and append output descriptors to DMA in list. */
 668                 if (tc->unit_no == src_dma){
 669                         /* mem2mem.  Setup DMA in descriptors to discard all input prior to the requested mem2mem data. */
 670                         struct strcop_meta_in mi = {.sync = 0, .dmasel = src_dma};
 671                         unsigned int start_ix = tc->start_ix;
 672                         while (start_ix){
 673                                 unsigned int desclen = start_ix < MEM2MEM_DISCARD_BUF_LENGTH ? start_ix : MEM2MEM_DISCARD_BUF_LENGTH;
 674                                 (*current_in_cdesc)->next = alloc_cdesc(alloc_flag);
 675                                 if (!(*current_in_cdesc)->next){
 676                                         DEBUG_API(printk("append_input_descriptors: alloc_cdesc mem2mem discard failed\n"));
 677                                         return -ENOMEM;
 678                                 }
 679                                 (*current_in_cdesc) = (*current_in_cdesc)->next;
 680                                 (*current_in_cdesc)->dma_descr->buf = (char*)virt_to_phys(mem2mem_discard_buf);
 681                                 (*current_in_cdesc)->dma_descr->after = (*current_in_cdesc)->dma_descr->buf + desclen;
 682                                 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 683                                 start_ix -= desclen;
 684                         }
 685                         mi.sync = 1;
 686                         (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 687                 }
 688
 689                 failed = create_input_descriptors(operation, tc, &idescs, alloc_flag);
 690                 if (failed){
 691                         DEBUG_API(printk("append_input_descriptors: output descriptor setup failed\n"));
 692                         return failed;
 693                 }
 694                 DEBUG(printk("append_input_descriptors: append output descriptors to DMA in list.\n"));
 695                 while (idescs) {
 696                         DEBUG(printk("append descriptor 0x%p\n", idescs));
 697                         (*current_in_cdesc)->next = idescs;
 698                         idescs = idescs->next;
 699                         (*current_in_cdesc) = (*current_in_cdesc)->next;
 700                 }
 701         }
 702         return 0;
 703 }
 704
 705
 706
 707 static int cryptocop_setup_dma_list(struct cryptocop_operation *operation, struct cryptocop_int_operation **int_op, int alloc_flag)
 708 {
 709         struct cryptocop_session *sess;
 710         struct cryptocop_transform_ctx *tctx;
 711
 712         struct cryptocop_tfrm_ctx digest_ctx = {
 713                 .previous_src = src_none,
 714                 .current_src = src_none,
 715                 .start_ix = 0,
 716                 .requires_padding = 1,
 717                 .strict_block_length = 0,
 718                 .hash_conf = 0,
 719                 .hash_mode = 0,
 720                 .ciph_conf = 0,
 721                 .cbcmode = 0,
 722                 .decrypt = 0,
 723                 .consumed = 0,
 724                 .produced = 0,
 725                 .pad_descs = NULL,
 726                 .active = 0,
 727                 .done = 0,
 728                 .prev_src = NULL,
 729                 .curr_src = NULL,
 730                 .tcfg = NULL};
 731         struct cryptocop_tfrm_ctx cipher_ctx = {
 732                 .previous_src = src_none,
 733                 .current_src = src_none,
 734                 .start_ix = 0,
 735                 .requires_padding = 0,
 736                 .strict_block_length = 1,
 737                 .hash_conf = 0,
 738                 .hash_mode = 0,
 739                 .ciph_conf = 0,
 740                 .cbcmode = 0,
 741                 .decrypt = 0,
 742                 .consumed = 0,
 743                 .produced = 0,
 744                 .pad_descs = NULL,
 745                 .active = 0,
 746                 .done = 0,
 747                 .prev_src = NULL,
 748                 .curr_src = NULL,
 749                 .tcfg = NULL};
 750         struct cryptocop_tfrm_ctx csum_ctx = {
 751                 .previous_src = src_none,
 752                 .current_src = src_none,
 753                 .start_ix = 0,
 754                 .blocklength = 2,
 755                 .requires_padding = 1,
 756                 .strict_block_length = 0,
 757                 .hash_conf = 0,
 758                 .hash_mode = 0,
 759                 .ciph_conf = 0,
 760                 .cbcmode = 0,
 761                 .decrypt = 0,
 762                 .consumed = 0,
 763                 .produced = 0,
 764                 .pad_descs = NULL,
 765                 .active = 0,
 766                 .done = 0,
 767                 .tcfg = NULL,
 768                 .prev_src = NULL,
 769                 .curr_src = NULL,
 770                 .unit_no = src_csum};
 771         struct cryptocop_tfrm_cfg *tcfg = operation->tfrm_op.tfrm_cfg;
 772
 773         unsigned int indata_ix = 0;
 774
 775         /* iovec accounting. */
 776         int iniov_ix = 0;
 777         int iniov_offset = 0;
 778
 779         /* Operation descriptor cfg traversal pointer. */
 780         struct cryptocop_desc *odsc;
 781
 782         int failed = 0;
 783         /* List heads for allocated descriptors. */
 784         struct cryptocop_dma_desc out_cdesc_head = {0};
 785         struct cryptocop_dma_desc in_cdesc_head = {0};
 786
 787         struct cryptocop_dma_desc *current_out_cdesc = &out_cdesc_head;
 788         struct cryptocop_dma_desc *current_in_cdesc = &in_cdesc_head;
 789
 790         struct cryptocop_tfrm_ctx *output_tc = NULL;
 791         void                      *iop_alloc_ptr;
 792
 793         assert(operation != NULL);
 794         assert(int_op != NULL);
 795
 796         DEBUG(printk("cryptocop_setup_dma_list: start\n"));
 797         DEBUG(print_cryptocop_operation(operation));
 798
 799         sess = get_session(operation->sid);
 800         if (!sess) {
 801                 DEBUG_API(printk("cryptocop_setup_dma_list: no session found for operation.\n"));
 802                 failed = -EINVAL;
 803                 goto error_cleanup;
 804         }
 805         iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
 806         if (!iop_alloc_ptr) {
 807                 DEBUG_API(printk("cryptocop_setup_dma_list:  kmalloc cryptocop_int_operation\n"));
 808                 failed = -ENOMEM;
 809                 goto error_cleanup;
 810         }
 811         (*int_op) = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
 812         DEBUG(memset((*int_op), 0xff, sizeof(struct cryptocop_int_operation)));
 813         (*int_op)->alloc_ptr = iop_alloc_ptr;
 814         DEBUG(printk("cryptocop_setup_dma_list: *int_op=0x%p, alloc_ptr=0x%p\n", *int_op, (*int_op)->alloc_ptr));
 815
 816         (*int_op)->sid = operation->sid;
 817         (*int_op)->cdesc_out = NULL;
 818         (*int_op)->cdesc_in = NULL;
 819         (*int_op)->tdes_mode = cryptocop_3des_ede;
 820         (*int_op)->csum_mode = cryptocop_csum_le;
 821         (*int_op)->ddesc_out = NULL;
 822         (*int_op)->ddesc_in = NULL;
 823
 824         /* Scan operation->tfrm_op.tfrm_cfg for bad configuration and set up the local contexts. */
 825         if (!tcfg) {
 826                 DEBUG_API(printk("cryptocop_setup_dma_list: no configured transforms in operation.\n"));
 827                 failed = -EINVAL;
 828                 goto error_cleanup;
 829         }
 830         while (tcfg) {
 831                 tctx = get_transform_ctx(sess, tcfg->tid);
 832                 if (!tctx) {
 833                         DEBUG_API(printk("cryptocop_setup_dma_list: no transform id %d in session.\n", tcfg->tid));
 834                         failed = -EINVAL;
 835                         goto error_cleanup;
 836                 }
 837                 if (tcfg->inject_ix > operation->tfrm_op.outlen){
 838                         DEBUG_API(printk("cryptocop_setup_dma_list: transform id %d inject_ix (%d) > operation->tfrm_op.outlen(%d)", tcfg->tid, tcfg->inject_ix, operation->tfrm_op.outlen));
 839                         failed = -EINVAL;
 840                         goto error_cleanup;
 841                 }
 842                 switch (tctx->init.alg){
 843                 case cryptocop_alg_mem2mem:
 844                         if (cipher_ctx.tcfg != NULL){
 845                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
 846                                 failed = -EINVAL;
 847                                 goto error_cleanup;
 848                         }
 849                         /* mem2mem is handled as a NULL cipher. */
 850                         cipher_ctx.cbcmode = 0;
 851                         cipher_ctx.decrypt = 0;
 852                         cipher_ctx.blocklength = 1;
 853                         cipher_ctx.ciph_conf = 0;
 854                         cipher_ctx.unit_no = src_dma;
 855                         cipher_ctx.tcfg = tcfg;
 856                         cipher_ctx.tctx = tctx;
 857                         break;
 858                 case cryptocop_alg_des:
 859                 case cryptocop_alg_3des:
 860                 case cryptocop_alg_aes:
 861                         /* cipher */
 862                         if (cipher_ctx.tcfg != NULL){
 863                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
 864                                 failed = -EINVAL;
 865                                 goto error_cleanup;
 866                         }
 867                         cipher_ctx.tcfg = tcfg;
 868                         cipher_ctx.tctx = tctx;
 869                         if (cipher_ctx.tcfg->flags & CRYPTOCOP_DECRYPT){
 870                                 cipher_ctx.decrypt = 1;
 871                         }
 872                         switch (tctx->init.cipher_mode) {
 873                         case cryptocop_cipher_mode_ecb:
 874                                 cipher_ctx.cbcmode = 0;
 875                                 break;
 876                         case cryptocop_cipher_mode_cbc:
 877                                 cipher_ctx.cbcmode = 1;
 878                                 break;
 879                         default:
 880                                 DEBUG_API(printk("cryptocop_setup_dma_list: cipher_ctx, bad cipher mode==%d\n", tctx->init.cipher_mode));
 881                                 failed = -EINVAL;
 882                                 goto error_cleanup;
 883                         }
 884                         DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx, set CBC mode==%d\n", cipher_ctx.cbcmode));
 885                         switch (tctx->init.alg){
 886                         case cryptocop_alg_des:
 887                                 cipher_ctx.ciph_conf = 0;
 888                                 cipher_ctx.unit_no = src_des;
 889                                 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
 890                                 break;
 891                         case cryptocop_alg_3des:
 892                                 cipher_ctx.ciph_conf = 1;
 893                                 cipher_ctx.unit_no = src_des;
 894                                 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
 895                                 break;
 896                         case cryptocop_alg_aes:
 897                                 cipher_ctx.ciph_conf = 2;
 898                                 cipher_ctx.unit_no = src_aes;
 899                                 cipher_ctx.blocklength = AES_BLOCK_LENGTH;
 900                                 break;
 901                         default:
 902                                 panic("cryptocop_setup_dma_list: impossible algorithm %d\n", tctx->init.alg);
 903                         }
 904                         (*int_op)->tdes_mode = tctx->init.tdes_mode;
 905                         break;
 906                 case cryptocop_alg_md5:
 907                 case cryptocop_alg_sha1:
 908                         /* digest */
 909                         if (digest_ctx.tcfg != NULL){
 910                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple digests in operation.\n"));
 911                                 failed = -EINVAL;
 912                                 goto error_cleanup;
 913                         }
 914                         digest_ctx.tcfg = tcfg;
 915                         digest_ctx.tctx = tctx;
 916                         digest_ctx.hash_mode = 0; /* Don't use explicit IV in this API. */
 917                         switch (tctx->init.alg){
 918                         case cryptocop_alg_md5:
 919                                 digest_ctx.blocklength = MD5_BLOCK_LENGTH;
 920                                 digest_ctx.unit_no = src_md5;
 921                                 digest_ctx.hash_conf = 1; /* 1 => MD-5 */
 922                                 break;
 923                         case cryptocop_alg_sha1:
 924                                 digest_ctx.blocklength = SHA1_BLOCK_LENGTH;
 925                                 digest_ctx.unit_no = src_sha1;
 926                                 digest_ctx.hash_conf = 0; /* 0 => SHA-1 */
 927                                 break;
 928                         default:
 929                                 panic("cryptocop_setup_dma_list: impossible digest algorithm\n");
 930                         }
 931                         break;
 932                 case cryptocop_alg_csum:
 933                         /* digest */
 934                         if (csum_ctx.tcfg != NULL){
 935                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple checksums in operation.\n"));
 936                                 failed = -EINVAL;
 937                                 goto error_cleanup;
 938                         }
 939                         (*int_op)->csum_mode = tctx->init.csum_mode;
 940                         csum_ctx.tcfg = tcfg;
 941                         csum_ctx.tctx = tctx;
 942                         break;
 943                 default:
 944                         /* no algorithm. */
 945                         DEBUG_API(printk("cryptocop_setup_dma_list: invalid algorithm %d specified in tfrm %d.\n", tctx->init.alg, tcfg->tid));
 946                         failed = -EINVAL;
 947                         goto error_cleanup;
 948                 }
 949                 tcfg = tcfg->next;
 950         }
 951         /* Download key if a cipher is used. */
 952         if (cipher_ctx.tcfg && (cipher_ctx.tctx->init.alg != cryptocop_alg_mem2mem)){
 953                 struct cryptocop_dma_desc  *key_desc = NULL;
 954
 955                 failed = setup_key_dl_desc(&cipher_ctx, &key_desc, alloc_flag);
 956                 if (failed) {
 957                         DEBUG_API(printk("cryptocop_setup_dma_list: setup key dl\n"));
 958                         goto error_cleanup;
 959                 }
 960                 current_out_cdesc->next = key_desc;
 961                 current_out_cdesc = key_desc;
 962                 indata_ix += (unsigned int)(key_desc->dma_descr->after - key_desc->dma_descr->buf);
 963
 964                 /* Download explicit IV if a cipher is used and CBC mode and explicit IV selected. */
 965                 if ((cipher_ctx.tctx->init.cipher_mode == cryptocop_cipher_mode_cbc) && (cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV)) {
 966                         struct cryptocop_dma_desc  *iv_desc = NULL;
 967
 968                         DEBUG(printk("cryptocop_setup_dma_list: setup cipher CBC IV descriptor.\n"));
 969
 970                         failed = setup_cipher_iv_desc(&cipher_ctx, &iv_desc, alloc_flag);
 971                         if (failed) {
 972                                 DEBUG_API(printk("cryptocop_setup_dma_list: CBC IV descriptor.\n"));
 973                                 goto error_cleanup;
 974                         }
 975                         current_out_cdesc->next = iv_desc;
 976                         current_out_cdesc = iv_desc;
 977                         indata_ix += (unsigned int)(iv_desc->dma_descr->after - iv_desc->dma_descr->buf);
 978                 }
 979         }
 980
 981         /* Process descriptors. */
 982         odsc = operation->tfrm_op.desc;
 983         while (odsc) {
 984                 struct cryptocop_desc_cfg   *dcfg = odsc->cfg;
 985                 struct strcop_meta_out      meta_out = {0};
 986                 size_t                      desc_len = odsc->length;
 987                 int                         active_count, eop_needed_count;
 988
 989                 output_tc = NULL;
 990
 991                 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor\n"));
 992
 993                 while (dcfg) {
 994                         struct cryptocop_tfrm_ctx  *tc = NULL;
 995
 996                         DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor configuration.\n"));
 997                         /* Get the local context for the transform and mark it as the output unit if it produces output. */
 998                         if (digest_ctx.tcfg && (digest_ctx.tcfg->tid == dcfg->tid)){
 999                                 tc = &digest_ctx;
1000                         } else if (cipher_ctx.tcfg && (cipher_ctx.tcfg->tid == dcfg->tid)){
1001                                 tc = &cipher_ctx;
1002                         } else if (csum_ctx.tcfg && (csum_ctx.tcfg->tid == dcfg->tid)){
1003                                 tc = &csum_ctx;
1004                         }
1005                         if (!tc) {
1006                                 DEBUG_API(printk("cryptocop_setup_dma_list: invalid transform %d specified in descriptor.\n", dcfg->tid));
1007                                 failed = -EINVAL;
1008                                 goto error_cleanup;
1009                         }
1010                         if (tc->done) {
1011                                 DEBUG_API(printk("cryptocop_setup_dma_list: completed transform %d reused.\n", dcfg->tid));
1012                                 failed = -EINVAL;
1013                                 goto error_cleanup;
1014                         }
1015                         if (!tc->active) {
1016                                 tc->start_ix = indata_ix;
1017                                 tc->active = 1;
1018                         }
1019
1020                         tc->previous_src = tc->current_src;
1021                         tc->prev_src = tc->curr_src;
1022                         /* Map source unit id to DMA source config. */
1023                         switch (dcfg->src){
1024                         case cryptocop_source_dma:
1025                                 tc->current_src = src_dma;
1026                                 break;
1027                         case cryptocop_source_des:
1028                                 tc->current_src = src_des;
1029                                 break;
1030                         case cryptocop_source_3des:
1031                                 tc->current_src = src_des;
1032                                 break;
1033                         case cryptocop_source_aes:
1034                                 tc->current_src = src_aes;
1035                                 break;
1036                         case cryptocop_source_md5:
1037                         case cryptocop_source_sha1:
1038                         case cryptocop_source_csum:
1039                         case cryptocop_source_none:
1040                         default:
1041                                 /* We do not allow using accumulating style units (SHA-1, MD5, checksum) as sources to other units.
1042                                  */
1043                                 DEBUG_API(printk("cryptocop_setup_dma_list: bad unit source configured %d.\n", dcfg->src));
1044                                 failed = -EINVAL;
1045                                 goto error_cleanup;
1046                         }
1047                         if (tc->current_src != src_dma) {
1048                                 /* Find the unit we are sourcing from. */
1049                                 if (digest_ctx.unit_no == tc->current_src){
1050                                         tc->curr_src = &digest_ctx;
1051                                 } else if (cipher_ctx.unit_no == tc->current_src){
1052                                         tc->curr_src = &cipher_ctx;
1053                                 } else if (csum_ctx.unit_no == tc->current_src){
1054                                         tc->curr_src = &csum_ctx;
1055                                 }
1056                                 if ((tc->curr_src == tc) && (tc->unit_no != src_dma)){
1057                                         DEBUG_API(printk("cryptocop_setup_dma_list: unit %d configured to source from itself.\n", tc->unit_no));
1058                                         failed = -EINVAL;
1059                                         goto error_cleanup;
1060                                 }
1061                         } else {
1062                                 tc->curr_src = NULL;
1063                         }
1064
1065                         /* Detect source switch. */
1066                         DEBUG(printk("cryptocop_setup_dma_list: tc->active=%d tc->unit_no=%d tc->current_src=%d tc->previous_src=%d, tc->curr_src=0x%p, tc->prev_srv=0x%p\n", tc->active, tc->unit_no, tc->current_src, tc->previous_src, tc->curr_src, tc->prev_src));
1067                         if (tc->active && (tc->current_src != tc->previous_src)) {
1068                                 /* Only allow source switch when both the old source unit and the new one have
1069                                  * no pending data to process (i.e. the consumed length must be a multiple of the
1070                                  * transform blocklength). */
1071                                 /* Note: if the src == NULL we are actually sourcing from DMA out. */
1072                                 if (((tc->prev_src != NULL) && (tc->prev_src->consumed % tc->prev_src->blocklength)) ||
1073                                     ((tc->curr_src != NULL) && (tc->curr_src->consumed % tc->curr_src->blocklength)))
1074                                 {
1075                                         DEBUG_API(printk("cryptocop_setup_dma_list: can only disconnect from or connect to a unit on a multiple of the blocklength, old: cons=%d, prod=%d, block=%d, new: cons=%d prod=%d, block=%d.\n", tc->prev_src ? tc->prev_src->consumed : INT_MIN, tc->prev_src ? tc->prev_src->produced : INT_MIN, tc->prev_src ? tc->prev_src->blocklength : INT_MIN, tc->curr_src ? tc->curr_src->consumed : INT_MIN, tc->curr_src ? tc->curr_src->produced : INT_MIN, tc->curr_src ? tc->curr_src->blocklength : INT_MIN));
1076                                         failed = -EINVAL;
1077                                         goto error_cleanup;
1078                                 }
1079                         }
1080                         /* Detect unit deactivation. */
1081                         if (dcfg->last) {
1082                                 /* Length check of this is handled below. */
1083                                 tc->done = 1;
1084                         }
1085                         dcfg = dcfg->next;
1086                 } /* while (dcfg) */
1087                 DEBUG(printk("cryptocop_setup_dma_list: parsing operation descriptor configuration complete.\n"));
1088
1089                 if (cipher_ctx.active && (cipher_ctx.curr_src != NULL) && !cipher_ctx.curr_src->active){
1090                         DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", cipher_ctx.curr_src->unit_no));
1091                         failed = -EINVAL;
1092                         goto error_cleanup;
1093                 }
1094                 if (digest_ctx.active && (digest_ctx.curr_src != NULL) && !digest_ctx.curr_src->active){
1095                         DEBUG_API(printk("cryptocop_setup_dma_list: digest source from inactive unit %d\n", digest_ctx.curr_src->unit_no));
1096                         failed = -EINVAL;
1097                         goto error_cleanup;
1098                 }
1099                 if (csum_ctx.active && (csum_ctx.curr_src != NULL) && !csum_ctx.curr_src->active){
1100                         DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", csum_ctx.curr_src->unit_no));
1101                         failed = -EINVAL;
1102                         goto error_cleanup;
1103                 }
1104
1105                 /* Update consumed and produced lengths.
1106
1107                    The consumed length accounting here is actually cheating.  If a unit source from DMA (or any
1108                    other unit that process data in blocks of one octet) it is correct, but if it source from a
1109                    block processing unit, i.e. a cipher, it will be temporarily incorrect at some times.  However
1110                    since it is only allowed--by the HW--to change source to or from a block processing unit at times where that
1111                    unit has processed an exact multiple of its block length the end result will be correct.
1112                    Beware that if the source change restriction change this code will need to be (much) reworked.
1113                 */
1114                 DEBUG(printk("cryptocop_setup_dma_list: desc->length=%d, desc_len=%d.\n", odsc->length, desc_len));
1115
1116                 if (csum_ctx.active) {
1117                         csum_ctx.consumed += desc_len;
1118                         if (csum_ctx.done) {
1119                                 csum_ctx.produced = 2;
1120                         }
1121                         DEBUG(printk("cryptocop_setup_dma_list: csum_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", csum_ctx.consumed, csum_ctx.produced, csum_ctx.blocklength));
1122                 }
1123                 if (digest_ctx.active) {
1124                         digest_ctx.consumed += desc_len;
1125                         if (digest_ctx.done) {
1126                                 if (digest_ctx.unit_no == src_md5) {
1127                                         digest_ctx.produced = MD5_STATE_LENGTH;
1128                                 } else {
1129                                         digest_ctx.produced = SHA1_STATE_LENGTH;
1130                                 }
1131                         }
1132                         DEBUG(printk("cryptocop_setup_dma_list: digest_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", digest_ctx.consumed, digest_ctx.produced, digest_ctx.blocklength));
1133                 }
1134                 if (cipher_ctx.active) {
1135                         /* Ciphers are allowed only to source from DMA out.  That is filtered above. */
1136                         assert(cipher_ctx.current_src == src_dma);
1137                         cipher_ctx.consumed += desc_len;
1138                         cipher_ctx.produced = cipher_ctx.blocklength * (cipher_ctx.consumed / cipher_ctx.blocklength);
1139                         if (cipher_ctx.cbcmode && !(cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV) && cipher_ctx.produced){
1140                                 cipher_ctx.produced -= cipher_ctx.blocklength; /* Compensate for CBC iv. */
1141                         }
1142                         DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", cipher_ctx.consumed, cipher_ctx.produced, cipher_ctx.blocklength));
1143                 }
1144
1145                 /* Setup the DMA out descriptors. */
1146                 /* Configure the metadata. */
1147                 active_count = 0;
1148                 eop_needed_count = 0;
1149                 if (cipher_ctx.active) {
1150                         ++active_count;
1151                         if (cipher_ctx.unit_no == src_dma){
1152                                 /* mem2mem */
1153                                 meta_out.ciphsel = src_none;
1154                         } else {
1155                                 meta_out.ciphsel = cipher_ctx.current_src;
1156                         }
1157                         meta_out.ciphconf = cipher_ctx.ciph_conf;
1158                         meta_out.cbcmode = cipher_ctx.cbcmode;
1159                         meta_out.decrypt = cipher_ctx.decrypt;
1160                         DEBUG(printk("set ciphsel=%d ciphconf=%d cbcmode=%d decrypt=%d\n", meta_out.ciphsel, meta_out.ciphconf, meta_out.cbcmode, meta_out.decrypt));
1161                         if (cipher_ctx.done) ++eop_needed_count;
1162                 } else {
1163                         meta_out.ciphsel = src_none;
1164                 }
1165
1166                 if (digest_ctx.active) {
1167                         ++active_count;
1168                         meta_out.hashsel = digest_ctx.current_src;
1169                         meta_out.hashconf = digest_ctx.hash_conf;
1170                         meta_out.hashmode = 0; /* Explicit mode is not used here. */
1171                         DEBUG(printk("set hashsel=%d hashconf=%d hashmode=%d\n", meta_out.hashsel, meta_out.hashconf, meta_out.hashmode));
1172                         if (digest_ctx.done) {
1173                                 assert(digest_ctx.pad_descs == NULL);
1174                                 failed = create_pad_descriptor(&digest_ctx, &digest_ctx.pad_descs, alloc_flag);
1175                                 if (failed) {
1176                                         DEBUG_API(printk("cryptocop_setup_dma_list: failed digest pad creation.\n"));
1177                                         goto error_cleanup;
1178                                 }
1179                         }
1180                 } else {
1181                         meta_out.hashsel = src_none;
1182                 }
1183
1184                 if (csum_ctx.active) {
1185                         ++active_count;
1186                         meta_out.csumsel = csum_ctx.current_src;
1187                         if (csum_ctx.done) {
1188                                 assert(csum_ctx.pad_descs == NULL);
1189                                 failed = create_pad_descriptor(&csum_ctx, &csum_ctx.pad_descs, alloc_flag);
1190                                 if (failed) {
1191                                         DEBUG_API(printk("cryptocop_setup_dma_list: failed csum pad creation.\n"));
1192                                         goto error_cleanup;
1193                                 }
1194                         }
1195                 } else {
1196                         meta_out.csumsel = src_none;
1197                 }
1198                 DEBUG(printk("cryptocop_setup_dma_list: %d eop needed, %d active units\n", eop_needed_count, active_count));
1199                 /* Setup DMA out descriptors for the indata. */
1200                 failed = create_output_descriptors(operation, &iniov_ix, &iniov_offset, desc_len, &current_out_cdesc, &meta_out, alloc_flag);
1201                 if (failed) {
1202                         DEBUG_API(printk("cryptocop_setup_dma_list: create_output_descriptors %d\n", failed));
1203                         goto error_cleanup;
1204                 }
1205                 /* Setup out EOP.  If there are active units that are not done here they cannot get an EOP
1206                  * so we ust setup a zero length descriptor to DMA to signal EOP only to done units.
1207                  * If there is a pad descriptor EOP for the padded unit will be EOPed by it.
1208                  */
1209                 assert(active_count >= eop_needed_count);
1210                 assert((eop_needed_count == 0) || (eop_needed_count == 1));
1211                 if (eop_needed_count) {
1212                         /* This means that the bulk operation (cipeher/m2m) is terminated. */
1213                         if (active_count > 1) {
1214                                 /* Use zero length EOP descriptor. */
1215                                 struct cryptocop_dma_desc *ed = alloc_cdesc(alloc_flag);
1216                                 struct strcop_meta_out    ed_mo = {0};
1217                                 if (!ed) {
1218                                         DEBUG_API(printk("cryptocop_setup_dma_list: alloc EOP descriptor for cipher\n"));
1219                                         failed = -ENOMEM;
1220                                         goto error_cleanup;
1221                                 }
1222
1223                                 assert(cipher_ctx.active && cipher_ctx.done);
1224
1225                                 if (cipher_ctx.unit_no == src_dma){
1226                                         /* mem2mem */
1227                                         ed_mo.ciphsel = src_none;
1228                                 } else {
1229                                         ed_mo.ciphsel = cipher_ctx.current_src;
1230                                 }
1231                                 ed_mo.ciphconf = cipher_ctx.ciph_conf;
1232                                 ed_mo.cbcmode = cipher_ctx.cbcmode;
1233                                 ed_mo.decrypt = cipher_ctx.decrypt;
1234
1235                                 ed->free_buf = NULL;
1236                                 ed->dma_descr->wait = 1;
1237                                 ed->dma_descr->out_eop = 1;
1238
1239                                 ed->dma_descr->buf = (char*)virt_to_phys(&ed); /* Use any valid physical address for zero length descriptor. */
1240                                 ed->dma_descr->after = ed->dma_descr->buf;
1241                                 ed->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, ed_mo);
1242                                 current_out_cdesc->next = ed;
1243                                 current_out_cdesc = ed;
1244                         } else {
1245                                 /* Set EOP in the current out descriptor since the only active module is
1246                                  * the one needing the EOP. */
1247
1248                                 current_out_cdesc->dma_descr->out_eop = 1;
1249                         }
1250                 }
1251
1252                 if (cipher_ctx.done && cipher_ctx.active) cipher_ctx.active = 0;
1253                 if (digest_ctx.done && digest_ctx.active) digest_ctx.active = 0;
1254                 if (csum_ctx.done && csum_ctx.active) csum_ctx.active = 0;
1255                 indata_ix += odsc->length;
1256                 odsc = odsc->next;
1257         } /* while (odsc) */ /* Process descriptors. */
1258         DEBUG(printk("cryptocop_setup_dma_list: done parsing operation descriptors\n"));
1259         if (cipher_ctx.tcfg && (cipher_ctx.active || !cipher_ctx.done)){
1260                 DEBUG_API(printk("cryptocop_setup_dma_list: cipher operation not terminated.\n"));
1261                 failed = -EINVAL;
1262                 goto error_cleanup;
1263         }
1264         if (digest_ctx.tcfg && (digest_ctx.active || !digest_ctx.done)){
1265                 DEBUG_API(printk("cryptocop_setup_dma_list: digest operation not terminated.\n"));
1266                 failed = -EINVAL;
1267                 goto error_cleanup;
1268         }
1269         if (csum_ctx.tcfg && (csum_ctx.active || !csum_ctx.done)){
1270                 DEBUG_API(printk("cryptocop_setup_dma_list: csum operation not terminated.\n"));
1271                 failed = -EINVAL;
1272                 goto error_cleanup;
1273         }
1274
1275         failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &cipher_ctx, alloc_flag);
1276         if (failed){
1277                 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1278                 goto error_cleanup;
1279         }
1280         failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &digest_ctx, alloc_flag);
1281         if (failed){
1282                 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1283                 goto error_cleanup;
1284         }
1285         failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &csum_ctx, alloc_flag);
1286         if (failed){
1287                 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1288                 goto error_cleanup;
1289         }
1290
1291         DEBUG(printk("cryptocop_setup_dma_list: int_op=0x%p, *int_op=0x%p\n", int_op, *int_op));
1292         (*int_op)->cdesc_out = out_cdesc_head.next;
1293         (*int_op)->cdesc_in = in_cdesc_head.next;
1294         DEBUG(printk("cryptocop_setup_dma_list: out_cdesc_head=0x%p in_cdesc_head=0x%p\n", (*int_op)->cdesc_out, (*int_op)->cdesc_in));
1295
1296         setup_descr_chain(out_cdesc_head.next);
1297         setup_descr_chain(in_cdesc_head.next);
1298
1299         /* Last but not least: mark the last DMA in descriptor for a INTR and EOL and the the
1300          * last DMA out descriptor for EOL.
1301          */
1302         current_in_cdesc->dma_descr->intr = 1;
1303         current_in_cdesc->dma_descr->eol = 1;
1304         current_out_cdesc->dma_descr->eol = 1;
1305
1306         /* Setup DMA contexts. */
1307         (*int_op)->ctx_out.next = NULL;
1308         (*int_op)->ctx_out.eol = 1;
1309         (*int_op)->ctx_out.intr = 0;
1310         (*int_op)->ctx_out.store_mode = 0;
1311         (*int_op)->ctx_out.en = 0;
1312         (*int_op)->ctx_out.dis = 0;
1313         (*int_op)->ctx_out.md0 = 0;
1314         (*int_op)->ctx_out.md1 = 0;
1315         (*int_op)->ctx_out.md2 = 0;
1316         (*int_op)->ctx_out.md3 = 0;
1317         (*int_op)->ctx_out.md4 = 0;
1318         (*int_op)->ctx_out.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_out->dma_descr);
1319         (*int_op)->ctx_out.saved_data_buf = (*int_op)->cdesc_out->dma_descr->buf; /* Already physical address. */
1320
1321         (*int_op)->ctx_in.next = NULL;
1322         (*int_op)->ctx_in.eol = 1;
1323         (*int_op)->ctx_in.intr = 0;
1324         (*int_op)->ctx_in.store_mode = 0;
1325         (*int_op)->ctx_in.en = 0;
1326         (*int_op)->ctx_in.dis = 0;
1327         (*int_op)->ctx_in.md0 = 0;
1328         (*int_op)->ctx_in.md1 = 0;
1329         (*int_op)->ctx_in.md2 = 0;
1330         (*int_op)->ctx_in.md3 = 0;
1331         (*int_op)->ctx_in.md4 = 0;
1332
1333         (*int_op)->ctx_in.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_in->dma_descr);
1334         (*int_op)->ctx_in.saved_data_buf = (*int_op)->cdesc_in->dma_descr->buf; /* Already physical address. */
1335
1336         DEBUG(printk("cryptocop_setup_dma_list: done\n"));
1337         return 0;
1338
1339 error_cleanup:
1340         {
1341                 /* Free all allocated resources. */
1342                 struct cryptocop_dma_desc *tmp_cdesc;
1343                 while (digest_ctx.pad_descs){
1344                         tmp_cdesc = digest_ctx.pad_descs->next;
1345                         free_cdesc(digest_ctx.pad_descs);
1346                         digest_ctx.pad_descs = tmp_cdesc;
1347                 }
1348                 while (csum_ctx.pad_descs){
1349                         tmp_cdesc = csum_ctx.pad_descs->next;
1350                         free_cdesc(csum_ctx.pad_descs);
1351                         csum_ctx.pad_descs = tmp_cdesc;
1352                 }
1353                 assert(cipher_ctx.pad_descs == NULL); /* The ciphers are never padded. */
1354
1355                 if (*int_op != NULL) delete_internal_operation(*int_op);
1356         }
1357         DEBUG_API(printk("cryptocop_setup_dma_list: done with error %d\n", failed));
1358         return failed;
1359 }
1360
1361
1362 static void delete_internal_operation(struct cryptocop_int_operation *iop)
1363 {
1364         void                      *ptr = iop->alloc_ptr;
1365         struct cryptocop_dma_desc *cd = iop->cdesc_out;
1366         struct cryptocop_dma_desc *next;
1367
1368         DEBUG(printk("delete_internal_operation: iop=0x%p, alloc_ptr=0x%p\n", iop, ptr));
1369
1370         while (cd) {
1371                 next = cd->next;
1372                 free_cdesc(cd);
1373                 cd = next;
1374         }
1375         cd = iop->cdesc_in;
1376         while (cd) {
1377                 next = cd->next;
1378                 free_cdesc(cd);
1379                 cd = next;
1380         }
1381         kfree(ptr);
1382 }
1383
1384 #define MD5_MIN_PAD_LENGTH (9)
1385 #define MD5_PAD_LENGTH_FIELD_LENGTH (8)
1386
1387 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1388 {
1389         size_t                  padlen = MD5_BLOCK_LENGTH - (hashed_length % MD5_BLOCK_LENGTH);
1390         unsigned char           *p;
1391         int                     i;
1392         unsigned long long int  bit_length = hashed_length << 3;
1393
1394         if (padlen < MD5_MIN_PAD_LENGTH) padlen += MD5_BLOCK_LENGTH;
1395
1396         p = kmalloc(padlen, alloc_flag);
1397         if (!p) return -ENOMEM;
1398
1399         *p = 0x80;
1400         memset(p+1, 0, padlen - 1);
1401
1402         DEBUG(printk("create_md5_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1403
1404         i = padlen - MD5_PAD_LENGTH_FIELD_LENGTH;
1405         while (bit_length != 0){
1406                 p[i++] = bit_length % 0x100;
1407                 bit_length >>= 8;
1408         }
1409
1410         *pad = (char*)p;
1411         *pad_length = padlen;
1412
1413         return 0;
1414 }
1415
1416 #define SHA1_MIN_PAD_LENGTH (9)
1417 #define SHA1_PAD_LENGTH_FIELD_LENGTH (8)
1418
1419 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1420 {
1421         size_t                  padlen = SHA1_BLOCK_LENGTH - (hashed_length % SHA1_BLOCK_LENGTH);
1422         unsigned char           *p;
1423         int                     i;
1424         unsigned long long int  bit_length = hashed_length << 3;
1425
1426         if (padlen < SHA1_MIN_PAD_LENGTH) padlen += SHA1_BLOCK_LENGTH;
1427
1428         p = kmalloc(padlen, alloc_flag);
1429         if (!p) return -ENOMEM;
1430
1431         *p = 0x80;
1432         memset(p+1, 0, padlen - 1);
1433
1434         DEBUG(printk("create_sha1_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1435
1436         i = padlen - 1;
1437         while (bit_length != 0){
1438                 p[i--] = bit_length % 0x100;
1439                 bit_length >>= 8;
1440         }
1441
1442         *pad = (char*)p;
1443         *pad_length = padlen;
1444
1445         return 0;
1446 }
1447
1448
1449 static int transform_ok(struct cryptocop_transform_init *tinit)
1450 {
1451         switch (tinit->alg){
1452         case cryptocop_alg_csum:
1453                 switch (tinit->csum_mode){
1454                 case cryptocop_csum_le:
1455                 case cryptocop_csum_be:
1456                         break;
1457                 default:
1458                         DEBUG_API(printk("transform_ok: Bad mode set for csum transform\n"));
1459                         return -EINVAL;
1460                 }
1461         case cryptocop_alg_mem2mem:
1462         case cryptocop_alg_md5:
1463         case cryptocop_alg_sha1:
1464                 if (tinit->keylen != 0) {
1465                         DEBUG_API(printk("transform_ok: non-zero keylength, %d, for a digest/csum algorithm\n", tinit->keylen));
1466                         return -EINVAL; /* This check is a bit strict. */
1467                 }
1468                 break;
1469         case cryptocop_alg_des:
1470                 if (tinit->keylen != 64) {
1471                         DEBUG_API(printk("transform_ok: keylen %d invalid for DES\n", tinit->keylen));
1472                         return -EINVAL;
1473                 }
1474                 break;
1475         case cryptocop_alg_3des:
1476                 if (tinit->keylen != 192) {
1477                         DEBUG_API(printk("transform_ok: keylen %d invalid for 3DES\n", tinit->keylen));
1478                         return -EINVAL;
1479                 }
1480                 break;
1481         case cryptocop_alg_aes:
1482                 if (tinit->keylen != 128 && tinit->keylen != 192 && tinit->keylen != 256) {
1483                         DEBUG_API(printk("transform_ok: keylen %d invalid for AES\n", tinit->keylen));
1484                         return -EINVAL;
1485                 }
1486                 break;
1487         case cryptocop_no_alg:
1488         default:
1489                 DEBUG_API(printk("transform_ok: no such algorithm %d\n", tinit->alg));
1490                 return -EINVAL;
1491         }
1492
1493         switch (tinit->alg){
1494         case cryptocop_alg_des:
1495         case cryptocop_alg_3des:
1496         case cryptocop_alg_aes:
1497                 if (tinit->cipher_mode != cryptocop_cipher_mode_ecb && tinit->cipher_mode != cryptocop_cipher_mode_cbc) return -EINVAL;
1498         default:
1499                  break;
1500         }
1501         return 0;
1502 }
1503
1504
1505 int cryptocop_new_session(cryptocop_session_id *sid, struct cryptocop_transform_init *tinit, int alloc_flag)
1506 {
1507         struct cryptocop_session         *sess;
1508         struct cryptocop_transform_init  *tfrm_in = tinit;
1509         struct cryptocop_transform_init  *tmp_in;
1510         int                              no_tfrms = 0;
1511         int                              i;
1512         unsigned long int                flags;
1513
1514         init_stream_coprocessor(); /* For safety if we are called early */
1515
1516         while (tfrm_in){
1517                 int err;
1518                 ++no_tfrms;
1519                 if ((err = transform_ok(tfrm_in))) {
1520                         DEBUG_API(printk("cryptocop_new_session, bad transform\n"));
1521                         return err;
1522                 }
1523                 tfrm_in = tfrm_in->next;
1524         }
1525         if (0 == no_tfrms) {
1526                 DEBUG_API(printk("cryptocop_new_session, no transforms specified\n"));
1527                 return -EINVAL;
1528         }
1529
1530         sess = kmalloc(sizeof(struct cryptocop_session), alloc_flag);
1531         if (!sess){
1532                 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_session\n"));
1533                 return -ENOMEM;
1534         }
1535
1536         sess->tfrm_ctx = kmalloc(no_tfrms * sizeof(struct cryptocop_transform_ctx), alloc_flag);
1537         if (!sess->tfrm_ctx) {
1538                 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_transform_ctx\n"));
1539                 kfree(sess);
1540                 return -ENOMEM;
1541         }
1542
1543         tfrm_in = tinit;
1544         for (i = 0; i < no_tfrms; i++){
1545                 tmp_in = tfrm_in->next;
1546                 while (tmp_in){
1547                         if (tmp_in->tid == tfrm_in->tid) {
1548                                 DEBUG_API(printk("cryptocop_new_session, duplicate transform ids\n"));
1549                                 kfree(sess->tfrm_ctx);
1550                                 kfree(sess);
1551                                 return -EINVAL;
1552                         }
1553                         tmp_in = tmp_in->next;
1554                 }
1555                 memcpy(&sess->tfrm_ctx[i].init, tfrm_in, sizeof(struct cryptocop_transform_init));
1556                 sess->tfrm_ctx[i].dec_key_set = 0;
1557                 sess->tfrm_ctx[i].next = &sess->tfrm_ctx[i] + 1;
1558
1559                 tfrm_in = tfrm_in->next;
1560         }
1561         sess->tfrm_ctx[i-1].next = NULL;
1562
1563         spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1564         sess->sid = next_sid;
1565         next_sid++;
1566         /* TODO If we are really paranoid we should do duplicate check to handle sid wraparound.
1567          *      OTOH 2^64 is a really large number of session. */
1568         if (next_sid == 0) next_sid = 1;
1569
1570         /* Prepend to session list. */
1571         sess->next = cryptocop_sessions;
1572         cryptocop_sessions = sess;
1573         spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1574         *sid = sess->sid;
1575         return 0;
1576 }
1577
1578
1579 int cryptocop_free_session(cryptocop_session_id sid)
1580 {
1581         struct cryptocop_transform_ctx    *tc;
1582         struct cryptocop_session          *sess = NULL;
1583         struct cryptocop_session          *psess = NULL;
1584         unsigned long int                 flags;
1585         int                               i;
1586         LIST_HEAD(remove_list);
1587         struct list_head                  *node, *tmp;
1588         struct cryptocop_prio_job         *pj;
1589
1590         DEBUG(printk("cryptocop_free_session: sid=%lld\n", sid));
1591
1592         spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1593         sess = cryptocop_sessions;
1594         while (sess && sess->sid != sid){
1595                 psess = sess;
1596                 sess = sess->next;
1597         }
1598         if (sess){
1599                 if (psess){
1600                         psess->next = sess->next;
1601                 } else {
1602                         cryptocop_sessions = sess->next;
1603                 }
1604         }
1605         spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1606
1607         if (!sess) return -EINVAL;
1608
1609         /* Remove queued jobs. */
1610         spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1611
1612         for (i = 0; i < cryptocop_prio_no_prios; i++){
1613                 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
1614                         list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
1615                                 pj = list_entry(node, struct cryptocop_prio_job, node);
1616                                 if (pj->oper->sid == sid) {
1617                                         list_move_tail(node, &remove_list);
1618                                 }
1619                         }
1620                 }
1621         }
1622         spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1623
1624         list_for_each_safe(node, tmp, &remove_list) {
1625                 list_del(node);
1626                 pj = list_entry(node, struct cryptocop_prio_job, node);
1627                 pj->oper->operation_status = -EAGAIN;  /* EAGAIN is not ideal for job/session terminated but it's the best choice I know of. */
1628                 DEBUG(printk("cryptocop_free_session: pj=0x%p, pj->oper=0x%p, pj->iop=0x%p\n", pj, pj->oper, pj->iop));
1629                 pj->oper->cb(pj->oper, pj->oper->cb_data);
1630                 delete_internal_operation(pj->iop);
1631                 kfree(pj);
1632         }
1633
1634         tc = sess->tfrm_ctx;
1635         /* Erase keying data. */
1636         while (tc){
1637                 DEBUG(printk("cryptocop_free_session: memset keys, tfrm id=%d\n", tc->init.tid));
1638                 memset(tc->init.key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1639                 memset(tc->dec_key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1640                 tc = tc->next;
1641         }
1642         kfree(sess->tfrm_ctx);
1643         kfree(sess);
1644
1645         return 0;
1646 }
1647
1648 static struct cryptocop_session *get_session(cryptocop_session_id sid)
1649 {
1650         struct cryptocop_session    *sess;
1651         unsigned long int           flags;
1652
1653         spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1654         sess = cryptocop_sessions;
1655         while (sess && (sess->sid != sid)){
1656                 sess = sess->next;
1657         }
1658         spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1659
1660         return sess;
1661 }
1662
1663 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid)
1664 {
1665         struct cryptocop_transform_ctx *tc = sess->tfrm_ctx;
1666
1667         DEBUG(printk("get_transform_ctx, sess=0x%p, tid=%d\n", sess, tid));
1668         assert(sess != NULL);
1669         while (tc && tc->init.tid != tid){
1670                 DEBUG(printk("tc=0x%p, tc->next=0x%p\n", tc, tc->next));
1671                 tc = tc->next;
1672         }
1673         DEBUG(printk("get_transform_ctx, returning tc=0x%p\n", tc));
1674         return tc;
1675 }
1676
1677
1678
1679 /* The AES s-transform matrix (s-box). */
1680 static const u8 aes_sbox[256] = {
1681         99,  124, 119, 123, 242, 107, 111, 197, 48,  1,   103, 43,  254, 215, 171, 118,
1682         202, 130, 201, 125, 250, 89,  71,  240, 173, 212, 162, 175, 156, 164, 114, 192,
1683         183, 253, 147, 38,  54,  63,  247, 204, 52,  165, 229, 241, 113, 216, 49,  21,
1684         4,   199, 35,  195, 24,  150, 5,   154, 7,   18,  128, 226, 235, 39,  178, 117,
1685         9,   131, 44,  26,  27,  110, 90,  160, 82,  59,  214, 179, 41,  227, 47,  132,
1686         83,  209, 0,   237, 32,  252, 177, 91,  106, 203, 190, 57,  74,  76,  88,  207,
1687         208, 239, 170, 251, 67,  77,  51,  133, 69,  249, 2,   127, 80,  60,  159, 168,
1688         81,  163, 64,  143, 146, 157, 56,  245, 188, 182, 218, 33,  16,  255, 243, 210,
1689         205, 12,  19,  236, 95,  151, 68,  23,  196, 167, 126, 61,  100, 93,  25,  115,
1690         96,  129, 79,  220, 34,  42,  144, 136, 70,  238, 184, 20,  222, 94,  11,  219,
1691         224, 50,  58,  10,  73,  6,   36,  92,  194, 211, 172, 98,  145, 149, 228, 121,
1692         231, 200, 55,  109, 141, 213, 78,  169, 108, 86,  244, 234, 101, 122, 174, 8,
1693         186, 120, 37,  46,  28,  166, 180, 198, 232, 221, 116, 31,  75,  189, 139, 138,
1694         112, 62,  181, 102, 72,  3,   246, 14,  97,  53,  87,  185, 134, 193, 29,  158,
1695         225, 248, 152, 17,  105, 217, 142, 148, 155, 30,  135, 233, 206, 85,  40,  223,
1696         140, 161, 137, 13,  191, 230, 66,  104, 65,  153, 45,  15,  176, 84,  187, 22
1697 };
1698
1699 /* AES has a 32 bit word round constants for each round in the
1700  * key schedule.  round_constant[i] is really Rcon[i+1] in FIPS187.
1701  */
1702 static u32 round_constant[11] = {
1703         0x01000000, 0x02000000, 0x04000000, 0x08000000,
1704         0x10000000, 0x20000000, 0x40000000, 0x80000000,
1705         0x1B000000, 0x36000000, 0x6C000000
1706 };
1707
1708 /* Apply the s-box to each of the four occtets in w. */
1709 static u32 aes_ks_subword(const u32 w)
1710 {
1711         u8 bytes[4];
1712
1713         *(u32*)(&bytes[0]) = w;
1714         bytes[0] = aes_sbox[bytes[0]];
1715         bytes[1] = aes_sbox[bytes[1]];
1716         bytes[2] = aes_sbox[bytes[2]];
1717         bytes[3] = aes_sbox[bytes[3]];
1718         return *(u32*)(&bytes[0]);
1719 }
1720
1721 /* The encrypt (forward) Rijndael key schedule algorithm pseudo code:
1722  * (Note that AES words are 32 bit long)
1723  *
1724  * KeyExpansion(byte key[4*Nk], word w[Nb*(Nr+1)], Nk){
1725  * word temp
1726  * i = 0
1727  * while (i < Nk) {
1728  *   w[i] = word(key[4*i, 4*i + 1, 4*i + 2, 4*i + 3])
1729  *   i = i + 1
1730  * }
1731  * i = Nk
1732  *
1733  * while (i < (Nb * (Nr + 1))) {
1734  *   temp = w[i - 1]
1735  *   if ((i mod Nk) == 0) {
1736  *     temp = SubWord(RotWord(temp)) xor Rcon[i/Nk]
1737  *   }
1738  *   else if ((Nk > 6) && ((i mod Nk) == 4)) {
1739  *     temp = SubWord(temp)
1740  *   }
1741  *   w[i] = w[i - Nk] xor temp
1742  * }
1743  * RotWord(t) does a 8 bit cyclic shift left on a 32 bit word.
1744  * SubWord(t) applies the AES s-box individually to each octet
1745  * in a 32 bit word.
1746  *
1747  * For AES Nk can have the values 4, 6, and 8 (corresponding to
1748  * values for Nr of 10, 12, and 14).  Nb is always 4.
1749  *
1750  * To construct w[i], w[i - 1] and w[i - Nk] must be
1751  * available.  Consequently we must keep a state of the last Nk words
1752  * to be able to create the last round keys.
1753  */
1754 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned  char *key, unsigned int keylength)
1755 {
1756         u32 temp;
1757         u32 w_ring[8]; /* nk is max 8, use elements 0..(nk - 1) as a ringbuffer */
1758         u8  w_last_ix;
1759         int i;
1760         u8  nr, nk;
1761
1762         switch (keylength){
1763         case 128:
1764                 nk = 4;
1765                 nr = 10;
1766                 break;
1767         case 192:
1768                 nk = 6;
1769                 nr = 12;
1770                 break;
1771         case 256:
1772                 nk = 8;
1773                 nr = 14;
1774                 break;
1775         default:
1776                 panic("stream co-processor: bad aes key length in get_aes_decrypt_key\n");
1777         };
1778
1779         /* Need to do host byte order correction here since key is byte oriented and the
1780          * kx algorithm is word (u32) oriented. */
1781         for (i = 0; i < nk; i+=1) {
1782                 w_ring[i] = be32_to_cpu(*(u32*)&key[4*i]);
1783         }
1784
1785         i = (int)nk;
1786         w_last_ix = i - 1;
1787         while (i < (4 * (nr + 2))) {
1788                 temp = w_ring[w_last_ix];
1789                 if (!(i % nk)) {
1790                         /* RotWord(temp) */
1791                         temp = (temp << 8) | (temp >> 24);
1792                         temp = aes_ks_subword(temp);
1793                         temp ^= round_constant[i/nk - 1];
1794                 } else if ((nk > 6) && ((i % nk) == 4)) {
1795                         temp = aes_ks_subword(temp);
1796                 }
1797                 w_last_ix = (w_last_ix + 1) % nk; /* This is the same as (i-Nk) mod Nk */
1798                 temp ^= w_ring[w_last_ix];
1799                 w_ring[w_last_ix] = temp;
1800
1801                 /* We need the round keys for round Nr+1 and Nr+2 (round key
1802                  * Nr+2 is the round key beyond the last one used when
1803                  * encrypting).  Rounds are numbered starting from 0, Nr=10
1804                  * implies 11 rounds are used in encryption/decryption.
1805                  */
1806                 if (i >= (4 * nr)) {
1807                         /* Need to do host byte order correction here, the key
1808                          * is byte oriented. */
1809                         *(u32*)dec_key = cpu_to_be32(temp);
1810                         dec_key += 4;
1811                 }
1812                 ++i;
1813         }
1814 }
1815
1816
1817 /**** Job/operation management. ****/
1818
1819 int cryptocop_job_queue_insert_csum(struct cryptocop_operation *operation)
1820 {
1821         return cryptocop_job_queue_insert(cryptocop_prio_kernel_csum, operation);
1822 }
1823
1824 int cryptocop_job_queue_insert_crypto(struct cryptocop_operation *operation)
1825 {
1826         return cryptocop_job_queue_insert(cryptocop_prio_kernel, operation);
1827 }
1828
1829 int cryptocop_job_queue_insert_user_job(struct cryptocop_operation *operation)
1830 {
1831         return cryptocop_job_queue_insert(cryptocop_prio_user, operation);
1832 }
1833
1834 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation)
1835 {
1836         int                           ret;
1837         struct cryptocop_prio_job     *pj = NULL;
1838         unsigned long int             flags;
1839
1840         DEBUG(printk("cryptocop_job_queue_insert(%d, 0x%p)\n", prio, operation));
1841
1842         if (!operation || !operation->cb){
1843                 DEBUG_API(printk("cryptocop_job_queue_insert oper=0x%p, NULL operation or callback\n", operation));
1844                 return -EINVAL;
1845         }
1846
1847         if ((ret = cryptocop_job_setup(&pj, operation)) != 0){
1848                 DEBUG_API(printk("cryptocop_job_queue_insert: job setup failed\n"));
1849                 return ret;
1850         }
1851         assert(pj != NULL);
1852
1853         spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1854         list_add_tail(&pj->node, &cryptocop_job_queues[prio].jobs);
1855         spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1856
1857         /* Make sure a job is running */
1858         cryptocop_start_job();
1859         return 0;
1860 }
1861
1862 static void cryptocop_do_tasklet(unsigned long unused);
1863 DECLARE_TASKLET (cryptocop_tasklet, cryptocop_do_tasklet, 0);
1864
1865 static void cryptocop_do_tasklet(unsigned long unused)
1866 {
1867         struct list_head             *node;
1868         struct cryptocop_prio_job    *pj = NULL;
1869         unsigned long                flags;
1870
1871         DEBUG(printk("cryptocop_do_tasklet: entering\n"));
1872
1873         do {
1874                 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
1875                 if (!list_empty(&cryptocop_completed_jobs)){
1876                         node = cryptocop_completed_jobs.next;
1877                         list_del(node);
1878                         pj = list_entry(node, struct cryptocop_prio_job, node);
1879                 } else {
1880                         pj = NULL;
1881                 }
1882                 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
1883                 if (pj) {
1884                         assert(pj->oper != NULL);
1885
1886                         /* Notify consumer of operation completeness. */
1887                         DEBUG(printk("cryptocop_do_tasklet: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
1888
1889                         pj->oper->operation_status = 0; /* Job is completed. */
1890                         pj->oper->cb(pj->oper, pj->oper->cb_data);
1891                         delete_internal_operation(pj->iop);
1892                         kfree(pj);
1893                 }
1894         } while (pj != NULL);
1895
1896         DEBUG(printk("cryptocop_do_tasklet: exiting\n"));
1897 }
1898
1899 static irqreturn_t
1900 dma_done_interrupt(int irq, void *dev_id)
1901 {
1902         struct cryptocop_prio_job *done_job;
1903         reg_dma_rw_ack_intr ack_intr = {
1904                 .data = 1,
1905         };
1906
1907         REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1908
1909         DEBUG(printk("cryptocop DMA done\n"));
1910
1911         spin_lock(&running_job_lock);
1912         if (cryptocop_running_job == NULL){
1913                 printk("stream co-processor got interrupt when not busy\n");
1914                 spin_unlock(&running_job_lock);
1915                 return IRQ_HANDLED;
1916         }
1917         done_job = cryptocop_running_job;
1918         cryptocop_running_job = NULL;
1919         spin_unlock(&running_job_lock);
1920
1921         /* Start processing a job. */
1922         if (!spin_trylock(&cryptocop_process_lock)){
1923                 DEBUG(printk("cryptocop irq handler, not starting a job\n"));
1924         } else {
1925                 cryptocop_start_job();
1926                 spin_unlock(&cryptocop_process_lock);
1927         }
1928
1929         done_job->oper->operation_status = 0; /* Job is completed. */
1930         if (done_job->oper->fast_callback){
1931                 /* This operation wants callback from interrupt. */
1932                 done_job->oper->cb(done_job->oper, done_job->oper->cb_data);
1933                 delete_internal_operation(done_job->iop);
1934                 kfree(done_job);
1935         } else {
1936                 spin_lock(&cryptocop_completed_jobs_lock);
1937                 list_add_tail(&(done_job->node), &cryptocop_completed_jobs);
1938                 spin_unlock(&cryptocop_completed_jobs_lock);
1939                 tasklet_schedule(&cryptocop_tasklet);
1940         }
1941
1942         DEBUG(printk("cryptocop leave irq handler\n"));
1943         return IRQ_HANDLED;
1944 }
1945
1946
1947 /* Setup interrupts and DMA channels. */
1948 static int init_cryptocop(void)
1949 {
1950         unsigned long          flags;
1951         reg_dma_rw_cfg         dma_cfg = {.en = 1};
1952         reg_dma_rw_intr_mask   intr_mask_in = {.data = regk_dma_yes}; /* Only want descriptor interrupts from the DMA in channel. */
1953         reg_dma_rw_ack_intr    ack_intr = {.data = 1,.in_eop = 1 };
1954         reg_strcop_rw_cfg      strcop_cfg = {
1955                 .ipend = regk_strcop_little,
1956                 .td1 = regk_strcop_e,
1957                 .td2 = regk_strcop_d,
1958                 .td3 = regk_strcop_e,
1959                 .ignore_sync = 0,
1960                 .en = 1
1961         };
1962
1963         if (request_irq(DMA_IRQ, dma_done_interrupt, 0,
1964                         "stream co-processor DMA", NULL))
1965                 panic("request_irq stream co-processor irq dma9");
1966
1967         (void)crisv32_request_dma(OUT_DMA, "strcop", DMA_PANIC_ON_ERROR,
1968                 0, dma_strp);
1969         (void)crisv32_request_dma(IN_DMA, "strcop", DMA_PANIC_ON_ERROR,
1970                 0, dma_strp);
1971
1972         local_irq_save(flags);
1973
1974         /* Reset and enable the cryptocop. */
1975         strcop_cfg.en = 0;
1976         REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1977         strcop_cfg.en = 1;
1978         REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1979
1980         /* Enable DMAs. */
1981         REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
1982         REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
1983
1984         /* Set up wordsize = 4 for DMAs. */
1985         DMA_WR_CMD(OUT_DMA_INST, regk_dma_set_w_size4);
1986         DMA_WR_CMD(IN_DMA_INST, regk_dma_set_w_size4);
1987
1988         /* Enable interrupts. */
1989         REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
1990
1991         /* Clear intr ack. */
1992         REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1993
1994         local_irq_restore(flags);
1995
1996         return 0;
1997 }
1998
1999 /* Free used cryptocop hw resources (interrupt and DMA channels). */
2000 static void release_cryptocop(void)
2001 {
2002         unsigned long          flags;
2003         reg_dma_rw_cfg         dma_cfg = {.en = 0};
2004         reg_dma_rw_intr_mask   intr_mask_in = {0};
2005         reg_dma_rw_ack_intr    ack_intr = {.data = 1,.in_eop = 1 };
2006
2007         local_irq_save(flags);
2008
2009         /* Clear intr ack. */
2010         REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
2011
2012         /* Disable DMAs. */
2013         REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
2014         REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
2015
2016         /* Disable interrupts. */
2017         REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
2018
2019         local_irq_restore(flags);
2020
2021         free_irq(DMA_IRQ, NULL);
2022
2023         (void)crisv32_free_dma(OUT_DMA);
2024         (void)crisv32_free_dma(IN_DMA);
2025 }
2026
2027
2028 /* Init job queue. */
2029 static int cryptocop_job_queue_init(void)
2030 {
2031         int i;
2032
2033         INIT_LIST_HEAD(&cryptocop_completed_jobs);
2034
2035         for (i = 0; i < cryptocop_prio_no_prios; i++){
2036                 cryptocop_job_queues[i].prio = (cryptocop_queue_priority)i;
2037                 INIT_LIST_HEAD(&cryptocop_job_queues[i].jobs);
2038         }
2039         return 0;
2040 }
2041
2042
2043 static void cryptocop_job_queue_close(void)
2044 {
2045         struct list_head               *node, *tmp;
2046         struct cryptocop_prio_job      *pj = NULL;
2047         unsigned long int              process_flags, flags;
2048         int                            i;
2049
2050         /* FIXME: This is as yet untested code. */
2051
2052         /* Stop strcop from getting an operation to process while we are closing the
2053            module. */
2054         spin_lock_irqsave(&cryptocop_process_lock, process_flags);
2055
2056         /* Empty the job queue. */
2057         for (i = 0; i < cryptocop_prio_no_prios; i++){
2058                 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
2059                         list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
2060                                 pj = list_entry(node, struct cryptocop_prio_job, node);
2061                                 list_del(node);
2062
2063                                 /* Call callback to notify consumer of job removal. */
2064                                 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2065                                 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2066                                 pj->oper->cb(pj->oper, pj->oper->cb_data);
2067
2068                                 delete_internal_operation(pj->iop);
2069                                 kfree(pj);
2070                         }
2071                 }
2072         }
2073         spin_unlock_irqrestore(&cryptocop_process_lock, process_flags);
2074
2075         /* Remove the running job, if any. */
2076         spin_lock_irqsave(&running_job_lock, flags);
2077         if (cryptocop_running_job){
2078                 reg_strcop_rw_cfg rw_cfg;
2079                 reg_dma_rw_cfg    dma_out_cfg, dma_in_cfg;
2080
2081                 /* Stop DMA. */
2082                 dma_out_cfg = REG_RD(dma, OUT_DMA_INST, rw_cfg);
2083                 dma_out_cfg.en = regk_dma_no;
2084                 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_out_cfg);
2085
2086                 dma_in_cfg = REG_RD(dma, IN_DMA_INST, rw_cfg);
2087                 dma_in_cfg.en = regk_dma_no;
2088                 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_in_cfg);
2089
2090                 /* Disble the cryptocop. */
2091                 rw_cfg = REG_RD(strcop, regi_strcop, rw_cfg);
2092                 rw_cfg.en = 0;
2093                 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2094
2095                 pj = cryptocop_running_job;
2096                 cryptocop_running_job = NULL;
2097
2098                 /* Call callback to notify consumer of job removal. */
2099                 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2100                 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2101                 pj->oper->cb(pj->oper, pj->oper->cb_data);
2102
2103                 delete_internal_operation(pj->iop);
2104                 kfree(pj);
2105         }
2106         spin_unlock_irqrestore(&running_job_lock, flags);
2107
2108         /* Remove completed jobs, if any. */
2109         spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
2110
2111         list_for_each_safe(node, tmp, &cryptocop_completed_jobs) {
2112                 pj = list_entry(node, struct cryptocop_prio_job, node);
2113                 list_del(node);
2114                 /* Call callback to notify consumer of job removal. */
2115                 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2116                 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2117                 pj->oper->cb(pj->oper, pj->oper->cb_data);
2118
2119                 delete_internal_operation(pj->iop);
2120                 kfree(pj);
2121         }
2122         spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
2123 }
2124
2125
2126 static void cryptocop_start_job(void)
2127 {
2128         int                          i;
2129         struct cryptocop_prio_job    *pj;
2130         unsigned long int            flags;
2131         unsigned long int            running_job_flags;
2132         reg_strcop_rw_cfg            rw_cfg = {.en = 1, .ignore_sync = 0};
2133
2134         DEBUG(printk("cryptocop_start_job: entering\n"));
2135
2136         spin_lock_irqsave(&running_job_lock, running_job_flags);
2137         if (cryptocop_running_job != NULL){
2138                 /* Already running. */
2139                 DEBUG(printk("cryptocop_start_job: already running, exit\n"));
2140                 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2141                 return;
2142         }
2143         spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
2144
2145         /* Check the queues in priority order. */
2146         for (i = cryptocop_prio_kernel_csum; (i < cryptocop_prio_no_prios) && list_empty(&cryptocop_job_queues[i].jobs); i++);
2147         if (i == cryptocop_prio_no_prios) {
2148                 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2149                 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2150                 DEBUG(printk("cryptocop_start_job: no jobs to run\n"));
2151                 return; /* No jobs to run */
2152         }
2153         DEBUG(printk("starting job for prio %d\n", i));
2154
2155         /* TODO: Do not starve lower priority jobs.  Let in a lower
2156          * prio job for every N-th processed higher prio job or some
2157          * other scheduling policy.  This could reasonably be
2158          * tweakable since the optimal balance would depend on the
2159          * type of load on the system. */
2160
2161         /* Pull the DMA lists from the job and start the DMA client. */
2162         pj = list_entry(cryptocop_job_queues[i].jobs.next, struct cryptocop_prio_job, node);
2163         list_del(&pj->node);
2164         spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2165         cryptocop_running_job = pj;
2166
2167         /* Set config register (3DES and CSUM modes). */
2168         switch (pj->iop->tdes_mode){
2169         case cryptocop_3des_eee:
2170                 rw_cfg.td1 = regk_strcop_e;
2171                 rw_cfg.td2 = regk_strcop_e;
2172                 rw_cfg.td3 = regk_strcop_e;
2173                 break;
2174         case cryptocop_3des_eed:
2175                 rw_cfg.td1 = regk_strcop_e;
2176                 rw_cfg.td2 = regk_strcop_e;
2177                 rw_cfg.td3 = regk_strcop_d;
2178                 break;
2179         case cryptocop_3des_ede:
2180                 rw_cfg.td1 = regk_strcop_e;
2181                 rw_cfg.td2 = regk_strcop_d;
2182                 rw_cfg.td3 = regk_strcop_e;
2183                 break;
2184         case cryptocop_3des_edd:
2185                 rw_cfg.td1 = regk_strcop_e;
2186                 rw_cfg.td2 = regk_strcop_d;
2187                 rw_cfg.td3 = regk_strcop_d;
2188                 break;
2189         case cryptocop_3des_dee:
2190                 rw_cfg.td1 = regk_strcop_d;
2191                 rw_cfg.td2 = regk_strcop_e;
2192                 rw_cfg.td3 = regk_strcop_e;
2193                 break;
2194         case cryptocop_3des_ded:
2195                 rw_cfg.td1 = regk_strcop_d;
2196                 rw_cfg.td2 = regk_strcop_e;
2197                 rw_cfg.td3 = regk_strcop_d;
2198                 break;
2199         case cryptocop_3des_dde:
2200                 rw_cfg.td1 = regk_strcop_d;
2201                 rw_cfg.td2 = regk_strcop_d;
2202                 rw_cfg.td3 = regk_strcop_e;
2203                 break;
2204         case cryptocop_3des_ddd:
2205                 rw_cfg.td1 = regk_strcop_d;
2206                 rw_cfg.td2 = regk_strcop_d;
2207                 rw_cfg.td3 = regk_strcop_d;
2208                 break;
2209         default:
2210                 DEBUG(printk("cryptocop_setup_dma_list: bad 3DES mode\n"));
2211         }
2212         switch (pj->iop->csum_mode){
2213         case cryptocop_csum_le:
2214                 rw_cfg.ipend = regk_strcop_little;
2215                 break;
2216         case cryptocop_csum_be:
2217                 rw_cfg.ipend = regk_strcop_big;
2218                 break;
2219         default:
2220                 DEBUG(printk("cryptocop_setup_dma_list: bad checksum mode\n"));
2221         }
2222         REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2223
2224         DEBUG(printk("cryptocop_start_job: starting DMA, new cryptocop_running_job=0x%p\n"
2225                      "ctx_in: 0x%p, phys: 0x%p\n"
2226                      "ctx_out: 0x%p, phys: 0x%p\n",
2227                      pj,
2228                      &pj->iop->ctx_in, (char*)virt_to_phys(&pj->iop->ctx_in),
2229                      &pj->iop->ctx_out, (char*)virt_to_phys(&pj->iop->ctx_out)));
2230
2231         /* Start input DMA. */
2232         flush_dma_context(&pj->iop->ctx_in);
2233         DMA_START_CONTEXT(IN_DMA_INST, virt_to_phys(&pj->iop->ctx_in));
2234
2235         /* Start output DMA. */
2236         DMA_START_CONTEXT(OUT_DMA_INST, virt_to_phys(&pj->iop->ctx_out));
2237
2238         spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2239         DEBUG(printk("cryptocop_start_job: exiting\n"));
2240 }
2241
2242
2243 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation)
2244 {
2245         int  err;
2246         int  alloc_flag = operation->in_interrupt ? GFP_ATOMIC : GFP_KERNEL;
2247         void *iop_alloc_ptr = NULL;
2248
2249         *pj = kmalloc(sizeof (struct cryptocop_prio_job), alloc_flag);
2250         if (!*pj) return -ENOMEM;
2251
2252         DEBUG(printk("cryptocop_job_setup: operation=0x%p\n", operation));
2253
2254         (*pj)->oper = operation;
2255         DEBUG(printk("cryptocop_job_setup, cb=0x%p cb_data=0x%p\n",  (*pj)->oper->cb, (*pj)->oper->cb_data));
2256
2257         if (operation->use_dmalists) {
2258                 DEBUG(print_user_dma_lists(&operation->list_op));
2259                 if (!operation->list_op.inlist || !operation->list_op.outlist || !operation->list_op.out_data_buf || !operation->list_op.in_data_buf){
2260                         DEBUG_API(printk("cryptocop_job_setup: bad indata (use_dmalists)\n"));
2261                         kfree(*pj);
2262                         return -EINVAL;
2263                 }
2264                 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
2265                 if (!iop_alloc_ptr) {
2266                         DEBUG_API(printk("cryptocop_job_setup: kmalloc cryptocop_int_operation\n"));
2267                         kfree(*pj);
2268                         return -ENOMEM;
2269                 }
2270                 (*pj)->iop = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
2271                 DEBUG(memset((*pj)->iop, 0xff, sizeof(struct cryptocop_int_operation)));
2272                 (*pj)->iop->alloc_ptr = iop_alloc_ptr;
2273                 (*pj)->iop->sid = operation->sid;
2274                 (*pj)->iop->cdesc_out = NULL;
2275                 (*pj)->iop->cdesc_in = NULL;
2276                 (*pj)->iop->tdes_mode = operation->list_op.tdes_mode;
2277                 (*pj)->iop->csum_mode = operation->list_op.csum_mode;
2278                 (*pj)->iop->ddesc_out = operation->list_op.outlist;
2279                 (*pj)->iop->ddesc_in = operation->list_op.inlist;
2280
2281                 /* Setup DMA contexts. */
2282                 (*pj)->iop->ctx_out.next = NULL;
2283                 (*pj)->iop->ctx_out.eol = 1;
2284                 (*pj)->iop->ctx_out.saved_data = operation->list_op.outlist;
2285                 (*pj)->iop->ctx_out.saved_data_buf = operation->list_op.out_data_buf;
2286
2287                 (*pj)->iop->ctx_in.next = NULL;
2288                 (*pj)->iop->ctx_in.eol = 1;
2289                 (*pj)->iop->ctx_in.saved_data = operation->list_op.inlist;
2290                 (*pj)->iop->ctx_in.saved_data_buf = operation->list_op.in_data_buf;
2291         } else {
2292                 if ((err = cryptocop_setup_dma_list(operation, &(*pj)->iop, alloc_flag))) {
2293                         DEBUG_API(printk("cryptocop_job_setup: cryptocop_setup_dma_list failed %d\n", err));
2294                         kfree(*pj);
2295                         return err;
2296                 }
2297         }
2298         DEBUG(print_dma_descriptors((*pj)->iop));
2299
2300         DEBUG(printk("cryptocop_job_setup, DMA list setup successful\n"));
2301
2302         return 0;
2303 }
2304
2305 static int cryptocop_open(struct inode *inode, struct file *filp)
2306 {
2307         int p = iminor(inode);
2308
2309         if (p != CRYPTOCOP_MINOR) return -EINVAL;
2310
2311         filp->private_data = NULL;
2312         return 0;
2313 }
2314
2315
2316 static int cryptocop_release(struct inode *inode, struct file *filp)
2317 {
2318         struct cryptocop_private *dev = filp->private_data;
2319         struct cryptocop_private *dev_next;
2320
2321         while (dev){
2322                 dev_next = dev->next;
2323                 if (dev->sid != CRYPTOCOP_SESSION_ID_NONE) {
2324                         (void)cryptocop_free_session(dev->sid);
2325                 }
2326                 kfree(dev);
2327                 dev = dev_next;
2328         }
2329
2330         return 0;
2331 }
2332
2333
2334 static int cryptocop_ioctl_close_session(struct inode *inode, struct file *filp,
2335                                          unsigned int cmd, unsigned long arg)
2336 {
2337         struct cryptocop_private  *dev = filp->private_data;
2338         struct cryptocop_private  *prev_dev = NULL;
2339         struct strcop_session_op  *sess_op = (struct strcop_session_op *)arg;
2340         struct strcop_session_op  sop;
2341         int                       err;
2342
2343         DEBUG(printk("cryptocop_ioctl_close_session\n"));
2344
2345         if (!access_ok(VERIFY_READ, sess_op, sizeof(struct strcop_session_op)))
2346                 return -EFAULT;
2347         err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2348         if (err) return -EFAULT;
2349
2350         while (dev && (dev->sid != sop.ses_id)) {
2351                 prev_dev = dev;
2352                 dev = dev->next;
2353         }
2354         if (dev){
2355                 if (prev_dev){
2356                         prev_dev->next = dev->next;
2357                 } else {
2358                         filp->private_data = dev->next;
2359                 }
2360                 err = cryptocop_free_session(dev->sid);
2361                 if (err) return -EFAULT;
2362         } else {
2363                 DEBUG_API(printk("cryptocop_ioctl_close_session: session %lld not found\n", sop.ses_id));
2364                 return -EINVAL;
2365         }
2366         return 0;
2367 }
2368
2369
2370 static void ioctl_process_job_callback(struct cryptocop_operation *op, void*cb_data)
2371 {
2372         struct ioctl_job_cb_ctx *jc = (struct ioctl_job_cb_ctx *)cb_data;
2373
2374         DEBUG(printk("ioctl_process_job_callback: op=0x%p, cb_data=0x%p\n", op, cb_data));
2375
2376         jc->processed = 1;
2377         wake_up(&cryptocop_ioc_process_wq);
2378 }
2379
2380
2381 #define CRYPTOCOP_IOCTL_CIPHER_TID  (1)
2382 #define CRYPTOCOP_IOCTL_DIGEST_TID  (2)
2383 #define CRYPTOCOP_IOCTL_CSUM_TID    (3)
2384
2385 static size_t first_cfg_change_ix(struct strcop_crypto_op *crp_op)
2386 {
2387         size_t ch_ix = 0;
2388
2389         if (crp_op->do_cipher) ch_ix = crp_op->cipher_start;
2390         if (crp_op->do_digest && (crp_op->digest_start < ch_ix)) ch_ix = crp_op->digest_start;
2391         if (crp_op->do_csum && (crp_op->csum_start < ch_ix)) ch_ix = crp_op->csum_start;
2392
2393         DEBUG(printk("first_cfg_change_ix: ix=%d\n", ch_ix));
2394         return ch_ix;
2395 }
2396
2397
2398 static size_t next_cfg_change_ix(struct strcop_crypto_op *crp_op, size_t ix)
2399 {
2400         size_t ch_ix = INT_MAX;
2401         size_t tmp_ix = 0;
2402
2403         if (crp_op->do_cipher && ((crp_op->cipher_start + crp_op->cipher_len) > ix)){
2404                 if (crp_op->cipher_start > ix) {
2405                         ch_ix = crp_op->cipher_start;
2406                 } else {
2407                         ch_ix = crp_op->cipher_start + crp_op->cipher_len;
2408                 }
2409         }
2410         if (crp_op->do_digest && ((crp_op->digest_start + crp_op->digest_len) > ix)){
2411                 if (crp_op->digest_start > ix) {
2412                         tmp_ix = crp_op->digest_start;
2413                 } else {
2414                         tmp_ix = crp_op->digest_start + crp_op->digest_len;
2415                 }
2416                 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2417         }
2418         if (crp_op->do_csum && ((crp_op->csum_start + crp_op->csum_len) > ix)){
2419                 if (crp_op->csum_start > ix) {
2420                         tmp_ix = crp_op->csum_start;
2421                 } else {
2422                         tmp_ix = crp_op->csum_start + crp_op->csum_len;
2423                 }
2424                 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2425         }
2426         if (ch_ix == INT_MAX) ch_ix = ix;
2427         DEBUG(printk("next_cfg_change_ix prev ix=%d, next ix=%d\n", ix, ch_ix));
2428         return ch_ix;
2429 }
2430
2431
2432 /* Map map_length bytes from the pages starting on *pageix and *pageoffset to iovecs starting on *iovix.
2433  * Return -1 for ok, 0 for fail. */
2434 static int map_pages_to_iovec(struct iovec *iov, int iovlen, int *iovix, struct page **pages, int nopages, int *pageix, int *pageoffset, int map_length )
2435 {
2436         int tmplen;
2437
2438         assert(iov != NULL);
2439         assert(iovix != NULL);
2440         assert(pages != NULL);
2441         assert(pageix != NULL);
2442         assert(pageoffset != NULL);
2443
2444         DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2445
2446         while (map_length > 0){
2447                 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2448                 if (*iovix >= iovlen){
2449                         DEBUG_API(printk("map_page_to_iovec: *iovix=%d >= iovlen=%d\n", *iovix, iovlen));
2450                         return 0;
2451                 }
2452                 if (*pageix >= nopages){
2453                         DEBUG_API(printk("map_page_to_iovec: *pageix=%d >= nopages=%d\n", *pageix, nopages));
2454                         return 0;
2455                 }
2456                 iov[*iovix].iov_base = (unsigned char*)page_address(pages[*pageix]) + *pageoffset;
2457                 tmplen = PAGE_SIZE - *pageoffset;
2458                 if (tmplen < map_length){
2459                         (*pageoffset) = 0;
2460                         (*pageix)++;
2461                 } else {
2462                         tmplen = map_length;
2463                         (*pageoffset) += map_length;
2464                 }
2465                 DEBUG(printk("mapping %d bytes from page %d (or %d) to iovec %d\n", tmplen, *pageix, *pageix-1, *iovix));
2466                 iov[*iovix].iov_len = tmplen;
2467                 map_length -= tmplen;
2468                 (*iovix)++;
2469         }
2470         DEBUG(printk("map_page_to_iovec, exit, *iovix=%d\n", *iovix));
2471         return -1;
2472 }
2473
2474
2475
2476 static int cryptocop_ioctl_process(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2477 {
2478         int                             i;
2479         struct cryptocop_private        *dev = filp->private_data;
2480         struct strcop_crypto_op         *crp_oper = (struct strcop_crypto_op *)arg;
2481         struct strcop_crypto_op         oper = {0};
2482         int                             err = 0;
2483         struct cryptocop_operation      *cop = NULL;
2484
2485         struct ioctl_job_cb_ctx         *jc = NULL;
2486
2487         struct page                     **inpages = NULL;
2488         struct page                     **outpages = NULL;
2489         int                             noinpages = 0;
2490         int                             nooutpages = 0;
2491
2492         struct cryptocop_desc           descs[5]; /* Max 5 descriptors are needed, there are three transforms that
2493                                                    * can get connected/disconnected on different places in the indata. */
2494         struct cryptocop_desc_cfg       dcfgs[5*3];
2495         int                             desc_ix = 0;
2496         int                             dcfg_ix = 0;
2497         struct cryptocop_tfrm_cfg       ciph_tcfg = {0};
2498         struct cryptocop_tfrm_cfg       digest_tcfg = {0};
2499         struct cryptocop_tfrm_cfg       csum_tcfg = {0};
2500
2501         unsigned char                   *digest_result = NULL;
2502         int                             digest_length = 0;
2503         int                             cblocklen = 0;
2504         unsigned char                   csum_result[CSUM_BLOCK_LENGTH];
2505         struct cryptocop_session        *sess;
2506
2507         int    iovlen = 0;
2508         int    iovix = 0;
2509         int    pageix = 0;
2510         int    pageoffset = 0;
2511
2512         size_t prev_ix = 0;
2513         size_t next_ix;
2514
2515         int    cipher_active, digest_active, csum_active;
2516         int    end_digest, end_csum;
2517         int    digest_done = 0;
2518         int    cipher_done = 0;
2519         int    csum_done = 0;
2520
2521         DEBUG(printk("cryptocop_ioctl_process\n"));
2522
2523         if (!access_ok(VERIFY_WRITE, crp_oper, sizeof(struct strcop_crypto_op))){
2524                 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok crp_oper!\n"));
2525                 return -EFAULT;
2526         }
2527         if (copy_from_user(&oper, crp_oper, sizeof(struct strcop_crypto_op))) {
2528                 DEBUG_API(printk("cryptocop_ioctl_process: copy_from_user\n"));
2529                 return -EFAULT;
2530         }
2531         DEBUG(print_strcop_crypto_op(&oper));
2532
2533         while (dev && dev->sid != oper.ses_id) dev = dev->next;
2534         if (!dev){
2535                 DEBUG_API(printk("cryptocop_ioctl_process: session %lld not found\n", oper.ses_id));
2536                 return -EINVAL;
2537         }
2538
2539         /* Check buffers. */
2540         if (((oper.indata + oper.inlen) < oper.indata) || ((oper.cipher_outdata + oper.cipher_outlen) < oper.cipher_outdata)){
2541                 DEBUG_API(printk("cryptocop_ioctl_process: user buffers wrapped around, bad user!\n"));
2542                 return -EINVAL;
2543         }
2544
2545         if (!access_ok(VERIFY_WRITE, oper.cipher_outdata, oper.cipher_outlen)){
2546                 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok out data!\n"));
2547                 return -EFAULT;
2548         }
2549         if (!access_ok(VERIFY_READ, oper.indata, oper.inlen)){
2550                 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok in data!\n"));
2551                 return -EFAULT;
2552         }
2553
2554         cop = kmalloc(sizeof(struct cryptocop_operation), GFP_KERNEL);
2555         if (!cop) {
2556                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2557                 return -ENOMEM;
2558         }
2559         jc = kmalloc(sizeof(struct ioctl_job_cb_ctx), GFP_KERNEL);
2560         if (!jc) {
2561                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2562                 err = -ENOMEM;
2563                 goto error_cleanup;
2564         }
2565         jc->processed = 0;
2566
2567         cop->cb_data = jc;
2568         cop->cb = ioctl_process_job_callback;
2569         cop->operation_status = 0;
2570         cop->use_dmalists = 0;
2571         cop->in_interrupt = 0;
2572         cop->fast_callback = 0;
2573         cop->tfrm_op.tfrm_cfg = NULL;
2574         cop->tfrm_op.desc = NULL;
2575         cop->tfrm_op.indata = NULL;
2576         cop->tfrm_op.incount = 0;
2577         cop->tfrm_op.inlen = 0;
2578         cop->tfrm_op.outdata = NULL;
2579         cop->tfrm_op.outcount = 0;
2580         cop->tfrm_op.outlen = 0;
2581
2582         sess = get_session(oper.ses_id);
2583         if (!sess){
2584                 DEBUG_API(printk("cryptocop_ioctl_process: bad session id.\n"));
2585                 kfree(cop);
2586                 kfree(jc);
2587                 return -EINVAL;
2588         }
2589
2590         if (oper.do_cipher) {
2591                 unsigned int                    cipher_outlen = 0;
2592                 struct cryptocop_transform_ctx  *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_CIPHER_TID);
2593                 if (!tc) {
2594                         DEBUG_API(printk("cryptocop_ioctl_process: no cipher transform in session.\n"));
2595                         err = -EINVAL;
2596                         goto error_cleanup;
2597                 }
2598                 ciph_tcfg.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2599                 ciph_tcfg.inject_ix = 0;
2600                 ciph_tcfg.flags = 0;
2601                 if ((oper.cipher_start < 0) || (oper.cipher_len <= 0) || (oper.cipher_start > oper.inlen) || ((oper.cipher_start + oper.cipher_len) > oper.inlen)){
2602                         DEBUG_API(printk("cryptocop_ioctl_process: bad cipher length\n"));
2603                         kfree(cop);
2604                         kfree(jc);
2605                         return -EINVAL;
2606                 }
2607                 cblocklen = tc->init.alg == cryptocop_alg_aes ? AES_BLOCK_LENGTH : DES_BLOCK_LENGTH;
2608                 if (oper.cipher_len % cblocklen) {
2609                         kfree(cop);
2610                         kfree(jc);
2611                         DEBUG_API(printk("cryptocop_ioctl_process: cipher inlength not multiple of block length.\n"));
2612                         return -EINVAL;
2613                 }
2614                 cipher_outlen = oper.cipher_len;
2615                 if (tc->init.cipher_mode == cryptocop_cipher_mode_cbc){
2616                         if (oper.cipher_explicit) {
2617                                 ciph_tcfg.flags |= CRYPTOCOP_EXPLICIT_IV;
2618                                 memcpy(ciph_tcfg.iv, oper.cipher_iv, cblocklen);
2619                         } else {
2620                                 cipher_outlen = oper.cipher_len - cblocklen;
2621                         }
2622                 } else {
2623                         if (oper.cipher_explicit){
2624                                 kfree(cop);
2625                                 kfree(jc);
2626                                 DEBUG_API(printk("cryptocop_ioctl_process: explicit_iv when not CBC mode\n"));
2627                                 return -EINVAL;
2628                         }
2629                 }
2630                 if (oper.cipher_outlen != cipher_outlen) {
2631                         kfree(cop);
2632                         kfree(jc);
2633                         DEBUG_API(printk("cryptocop_ioctl_process: cipher_outlen incorrect, should be %d not %d.\n", cipher_outlen, oper.cipher_outlen));
2634                         return -EINVAL;
2635                 }
2636
2637                 if (oper.decrypt){
2638                         ciph_tcfg.flags |= CRYPTOCOP_DECRYPT;
2639                 } else {
2640                         ciph_tcfg.flags |= CRYPTOCOP_ENCRYPT;
2641                 }
2642                 ciph_tcfg.next = cop->tfrm_op.tfrm_cfg;
2643                 cop->tfrm_op.tfrm_cfg = &ciph_tcfg;
2644         }
2645         if (oper.do_digest){
2646                 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_DIGEST_TID);
2647                 if (!tc) {
2648                         DEBUG_API(printk("cryptocop_ioctl_process: no digest transform in session.\n"));
2649                         err = -EINVAL;
2650                         goto error_cleanup;
2651                 }
2652                 digest_length = tc->init.alg == cryptocop_alg_md5 ? 16 : 20;
2653                 digest_result = kmalloc(digest_length, GFP_KERNEL);
2654                 if (!digest_result) {
2655                         DEBUG_API(printk("cryptocop_ioctl_process: kmalloc digest_result\n"));
2656                         err = -EINVAL;
2657                         goto error_cleanup;
2658                 }
2659                 DEBUG(memset(digest_result, 0xff, digest_length));
2660
2661                 digest_tcfg.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2662                 digest_tcfg.inject_ix = 0;
2663                 ciph_tcfg.inject_ix += digest_length;
2664                 if ((oper.digest_start < 0) || (oper.digest_len <= 0) || (oper.digest_start > oper.inlen) || ((oper.digest_start + oper.digest_len) > oper.inlen)){
2665                         DEBUG_API(printk("cryptocop_ioctl_process: bad digest length\n"));
2666                         err = -EINVAL;
2667                         goto error_cleanup;
2668                 }
2669
2670                 digest_tcfg.next = cop->tfrm_op.tfrm_cfg;
2671                 cop->tfrm_op.tfrm_cfg = &digest_tcfg;
2672         }
2673         if (oper.do_csum){
2674                 csum_tcfg.tid = CRYPTOCOP_IOCTL_CSUM_TID;
2675                 csum_tcfg.inject_ix = digest_length;
2676                 ciph_tcfg.inject_ix += 2;
2677
2678                 if ((oper.csum_start < 0) || (oper.csum_len <= 0) || (oper.csum_start > oper.inlen) || ((oper.csum_start + oper.csum_len) > oper.inlen)){
2679                         DEBUG_API(printk("cryptocop_ioctl_process: bad csum length\n"));
2680                         kfree(cop);
2681                         kfree(jc);
2682                         return -EINVAL;
2683                 }
2684
2685                 csum_tcfg.next = cop->tfrm_op.tfrm_cfg;
2686                 cop->tfrm_op.tfrm_cfg = &csum_tcfg;
2687         }
2688
2689         prev_ix = first_cfg_change_ix(&oper);
2690         if (prev_ix > oper.inlen) {
2691                 DEBUG_API(printk("cryptocop_ioctl_process: length mismatch\n"));
2692                 nooutpages = noinpages = 0;
2693                 err = -EINVAL;
2694                 goto error_cleanup;
2695         }
2696         DEBUG(printk("cryptocop_ioctl_process: inlen=%d, cipher_outlen=%d\n", oper.inlen, oper.cipher_outlen));
2697
2698         /* Map user pages for in and out data of the operation. */
2699         noinpages = (((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK) + oper.inlen - 1 - prev_ix + ~PAGE_MASK) >> PAGE_SHIFT;
2700         DEBUG(printk("cryptocop_ioctl_process: noinpages=%d\n", noinpages));
2701         inpages = kmalloc(noinpages * sizeof(struct page*), GFP_KERNEL);
2702         if (!inpages){
2703                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc inpages\n"));
2704                 nooutpages = noinpages = 0;
2705                 err = -ENOMEM;
2706                 goto error_cleanup;
2707         }
2708         if (oper.do_cipher){
2709                 nooutpages = (((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) + oper.cipher_outlen - 1 + ~PAGE_MASK) >> PAGE_SHIFT;
2710                 DEBUG(printk("cryptocop_ioctl_process: nooutpages=%d\n", nooutpages));
2711                 outpages = kmalloc(nooutpages * sizeof(struct page*), GFP_KERNEL);
2712                 if (!outpages){
2713                         DEBUG_API(printk("cryptocop_ioctl_process: kmalloc outpages\n"));
2714                         nooutpages = noinpages = 0;
2715                         err = -ENOMEM;
2716                         goto error_cleanup;
2717                 }
2718         }
2719
2720         /* Acquire the mm page semaphore. */
2721         down_read(&current->mm->mmap_sem);
2722
2723         err = get_user_pages(current,
2724                              current->mm,
2725                              (unsigned long int)(oper.indata + prev_ix),
2726                              noinpages,
2727                              0,  /* read access only for in data */
2728                              0, /* no force */
2729                              inpages,
2730                              NULL);
2731
2732         if (err < 0) {
2733                 up_read(&current->mm->mmap_sem);
2734                 nooutpages = noinpages = 0;
2735                 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages indata\n"));
2736                 goto error_cleanup;
2737         }
2738         noinpages = err;
2739         if (oper.do_cipher){
2740                 err = get_user_pages(current,
2741                                      current->mm,
2742                                      (unsigned long int)oper.cipher_outdata,
2743                                      nooutpages,
2744                                      1, /* write access for out data */
2745                                      0, /* no force */
2746                                      outpages,
2747                                      NULL);
2748                 up_read(&current->mm->mmap_sem);
2749                 if (err < 0) {
2750                         nooutpages = 0;
2751                         DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages outdata\n"));
2752                         goto error_cleanup;
2753                 }
2754                 nooutpages = err;
2755         } else {
2756                 up_read(&current->mm->mmap_sem);
2757         }
2758
2759         /* Add 6 to nooutpages to make room for possibly inserted buffers for storing digest and
2760          * csum output and splits when units are (dis-)connected. */
2761         cop->tfrm_op.indata = kmalloc((noinpages) * sizeof(struct iovec), GFP_KERNEL);
2762         cop->tfrm_op.outdata = kmalloc((6 + nooutpages) * sizeof(struct iovec), GFP_KERNEL);
2763         if (!cop->tfrm_op.indata || !cop->tfrm_op.outdata) {
2764                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc iovecs\n"));
2765                 err = -ENOMEM;
2766                 goto error_cleanup;
2767         }
2768
2769         cop->tfrm_op.inlen = oper.inlen - prev_ix;
2770         cop->tfrm_op.outlen = 0;
2771         if (oper.do_cipher) cop->tfrm_op.outlen += oper.cipher_outlen;
2772         if (oper.do_digest) cop->tfrm_op.outlen += digest_length;
2773         if (oper.do_csum) cop->tfrm_op.outlen += 2;
2774
2775         /* Setup the in iovecs. */
2776         cop->tfrm_op.incount = noinpages;
2777         if (noinpages > 1){
2778                 size_t tmplen = cop->tfrm_op.inlen;
2779
2780                 cop->tfrm_op.indata[0].iov_len = PAGE_SIZE - ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2781                 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2782                 tmplen -= cop->tfrm_op.indata[0].iov_len;
2783                 for (i = 1; i<noinpages; i++){
2784                         cop->tfrm_op.indata[i].iov_len = tmplen < PAGE_SIZE ? tmplen : PAGE_SIZE;
2785                         cop->tfrm_op.indata[i].iov_base = (unsigned char*)page_address(inpages[i]);
2786                         tmplen -= PAGE_SIZE;
2787                 }
2788         } else {
2789                 cop->tfrm_op.indata[0].iov_len = oper.inlen - prev_ix;
2790                 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2791         }
2792
2793         iovlen = nooutpages + 6;
2794         pageoffset = oper.do_cipher ? ((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) : 0;
2795
2796         next_ix = next_cfg_change_ix(&oper, prev_ix);
2797         if (prev_ix == next_ix){
2798                 DEBUG_API(printk("cryptocop_ioctl_process: length configuration broken.\n"));
2799                 err = -EINVAL;  /* This should be impossible barring bugs. */
2800                 goto error_cleanup;
2801         }
2802         while (prev_ix != next_ix){
2803                 end_digest = end_csum = cipher_active = digest_active = csum_active = 0;
2804                 descs[desc_ix].cfg = NULL;
2805                 descs[desc_ix].length = next_ix - prev_ix;
2806
2807                 if (oper.do_cipher && (oper.cipher_start < next_ix) && (prev_ix < (oper.cipher_start + oper.cipher_len))) {
2808                         dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2809                         dcfgs[dcfg_ix].src = cryptocop_source_dma;
2810                         cipher_active = 1;
2811
2812                         if (next_ix == (oper.cipher_start + oper.cipher_len)){
2813                                 cipher_done = 1;
2814                                 dcfgs[dcfg_ix].last = 1;
2815                         } else {
2816                                 dcfgs[dcfg_ix].last = 0;
2817                         }
2818                         dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2819                         descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2820                         ++dcfg_ix;
2821                 }
2822                 if (oper.do_digest && (oper.digest_start < next_ix) && (prev_ix < (oper.digest_start + oper.digest_len))) {
2823                         digest_active = 1;
2824                         dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2825                         dcfgs[dcfg_ix].src = cryptocop_source_dma;
2826                         if (next_ix == (oper.digest_start + oper.digest_len)){
2827                                 assert(!digest_done);
2828                                 digest_done = 1;
2829                                 dcfgs[dcfg_ix].last = 1;
2830                         } else {
2831                                 dcfgs[dcfg_ix].last = 0;
2832                         }
2833                         dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2834                         descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2835                         ++dcfg_ix;
2836                 }
2837                 if (oper.do_csum && (oper.csum_start < next_ix) && (prev_ix < (oper.csum_start + oper.csum_len))){
2838                         csum_active = 1;
2839                         dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CSUM_TID;
2840                         dcfgs[dcfg_ix].src = cryptocop_source_dma;
2841                         if (next_ix == (oper.csum_start + oper.csum_len)){
2842                                 csum_done = 1;
2843                                 dcfgs[dcfg_ix].last = 1;
2844                         } else {
2845                                 dcfgs[dcfg_ix].last = 0;
2846                         }
2847                         dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2848                         descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2849                         ++dcfg_ix;
2850                 }
2851                 if (!descs[desc_ix].cfg){
2852                         DEBUG_API(printk("cryptocop_ioctl_process: data segment %d (%d to %d) had no active transforms\n", desc_ix, prev_ix, next_ix));
2853                         err = -EINVAL;
2854                         goto error_cleanup;
2855                 }
2856                 descs[desc_ix].next = &(descs[desc_ix]) + 1;
2857                 ++desc_ix;
2858                 prev_ix = next_ix;
2859                 next_ix = next_cfg_change_ix(&oper, prev_ix);
2860         }
2861         if (desc_ix > 0){
2862                 descs[desc_ix-1].next = NULL;
2863         } else {
2864                 descs[0].next = NULL;
2865         }
2866         if (oper.do_digest) {
2867                 DEBUG(printk("cryptocop_ioctl_process: mapping %d byte digest output to iovec %d\n", digest_length, iovix));
2868                 /* Add outdata iovec, length == <length of type of digest> */
2869                 cop->tfrm_op.outdata[iovix].iov_base = digest_result;
2870                 cop->tfrm_op.outdata[iovix].iov_len = digest_length;
2871                 ++iovix;
2872         }
2873         if (oper.do_csum) {
2874                 /* Add outdata iovec, length == 2, the length of csum. */
2875                 DEBUG(printk("cryptocop_ioctl_process: mapping 2 byte csum output to iovec %d\n", iovix));
2876                 /* Add outdata iovec, length == <length of type of digest> */
2877                 cop->tfrm_op.outdata[iovix].iov_base = csum_result;
2878                 cop->tfrm_op.outdata[iovix].iov_len = 2;
2879                 ++iovix;
2880         }
2881         if (oper.do_cipher) {
2882                 if (!map_pages_to_iovec(cop->tfrm_op.outdata, iovlen, &iovix, outpages, nooutpages, &pageix, &pageoffset, oper.cipher_outlen)){
2883                         DEBUG_API(printk("cryptocop_ioctl_process: failed to map pages to iovec.\n"));
2884                         err = -ENOSYS; /* This should be impossible barring bugs. */
2885                         goto error_cleanup;
2886                 }
2887         }
2888         DEBUG(printk("cryptocop_ioctl_process: setting cop->tfrm_op.outcount %d\n", iovix));
2889         cop->tfrm_op.outcount = iovix;
2890         assert(iovix <= (nooutpages + 6));
2891
2892         cop->sid = oper.ses_id;
2893         cop->tfrm_op.desc = &descs[0];
2894
2895         DEBUG(printk("cryptocop_ioctl_process: inserting job, cb_data=0x%p\n", cop->cb_data));
2896
2897         if ((err = cryptocop_job_queue_insert_user_job(cop)) != 0) {
2898                 DEBUG_API(printk("cryptocop_ioctl_process: insert job %d\n", err));
2899                 err = -EINVAL;
2900                 goto error_cleanup;
2901         }
2902
2903         DEBUG(printk("cryptocop_ioctl_process: begin wait for result\n"));
2904
2905         wait_event(cryptocop_ioc_process_wq, (jc->processed != 0));
2906         DEBUG(printk("cryptocop_ioctl_process: end wait for result\n"));
2907         if (!jc->processed){
2908                 printk(KERN_WARNING "cryptocop_ioctl_process: job not processed at completion\n");
2909                 err = -EIO;
2910                 goto error_cleanup;
2911         }
2912
2913         /* Job process done.  Cipher output should already be correct in job so no post processing of outdata. */
2914         DEBUG(printk("cryptocop_ioctl_process: operation_status = %d\n", cop->operation_status));
2915         if (cop->operation_status == 0){
2916                 if (oper.do_digest){
2917                         DEBUG(printk("cryptocop_ioctl_process: copy %d bytes digest to user\n", digest_length));
2918                         err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, digest), digest_result, digest_length);
2919                         if (0 != err){
2920                                 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, digest length %d, err %d\n", digest_length, err));
2921                                 err = -EFAULT;
2922                                 goto error_cleanup;
2923                         }
2924                 }
2925                 if (oper.do_csum){
2926                         DEBUG(printk("cryptocop_ioctl_process: copy 2 bytes checksum to user\n"));
2927                         err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, csum), csum_result, 2);
2928                         if (0 != err){
2929                                 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, csum, err %d\n", err));
2930                                 err = -EFAULT;
2931                                 goto error_cleanup;
2932                         }
2933                 }
2934                 err = 0;
2935         } else {
2936                 DEBUG(printk("cryptocop_ioctl_process: returning err = operation_status = %d\n", cop->operation_status));
2937                 err = cop->operation_status;
2938         }
2939
2940  error_cleanup:
2941         /* Release page caches. */
2942         for (i = 0; i < noinpages; i++){
2943                 put_page(inpages[i]);
2944         }
2945         for (i = 0; i < nooutpages; i++){
2946                 int spdl_err;
2947                 /* Mark output pages dirty. */
2948                 spdl_err = set_page_dirty_lock(outpages[i]);
2949                 DEBUG(if (spdl_err < 0)printk("cryptocop_ioctl_process: set_page_dirty_lock returned %d\n", spdl_err));
2950         }
2951         for (i = 0; i < nooutpages; i++){
2952                 put_page(outpages[i]);
2953         }
2954
2955         kfree(digest_result);
2956         kfree(inpages);
2957         kfree(outpages);
2958         if (cop){
2959                 kfree(cop->tfrm_op.indata);
2960                 kfree(cop->tfrm_op.outdata);
2961                 kfree(cop);
2962         }
2963         kfree(jc);
2964
2965         DEBUG(print_lock_status());
2966
2967         return err;
2968 }
2969
2970
2971 static int cryptocop_ioctl_create_session(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2972 {
2973         cryptocop_session_id             sid;
2974         int                              err;
2975         struct cryptocop_private         *dev;
2976         struct strcop_session_op         *sess_op = (struct strcop_session_op *)arg;
2977         struct strcop_session_op         sop;
2978         struct cryptocop_transform_init  *tis = NULL;
2979         struct cryptocop_transform_init  ti_cipher = {0};
2980         struct cryptocop_transform_init  ti_digest = {0};
2981         struct cryptocop_transform_init  ti_csum = {0};
2982
2983         if (!access_ok(VERIFY_WRITE, sess_op, sizeof(struct strcop_session_op)))
2984                 return -EFAULT;
2985         err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2986         if (err) return -EFAULT;
2987         if (sop.cipher != cryptocop_cipher_none) {
2988                 if (!access_ok(VERIFY_READ, sop.key, sop.keylen)) return -EFAULT;
2989         }
2990         DEBUG(printk("cryptocop_ioctl_create_session, sess_op:\n"));
2991
2992         DEBUG(printk("\tcipher:%d\n"
2993                      "\tcipher_mode:%d\n"
2994                      "\tdigest:%d\n"
2995                      "\tcsum:%d\n",
2996                      (int)sop.cipher,
2997                      (int)sop.cmode,
2998                      (int)sop.digest,
2999                      (int)sop.csum));
3000
3001         if (sop.cipher != cryptocop_cipher_none){
3002                 /* Init the cipher. */
3003                 switch (sop.cipher){
3004                 case cryptocop_cipher_des:
3005                         ti_cipher.alg = cryptocop_alg_des;
3006                         break;
3007                 case cryptocop_cipher_3des:
3008                         ti_cipher.alg = cryptocop_alg_3des;
3009                         break;
3010                 case cryptocop_cipher_aes:
3011                         ti_cipher.alg = cryptocop_alg_aes;
3012                         break;
3013                 default:
3014                         DEBUG_API(printk("create session, bad cipher algorithm %d\n", sop.cipher));
3015                         return -EINVAL;
3016                 };
3017                 DEBUG(printk("setting cipher transform %d\n", ti_cipher.alg));
3018                 copy_from_user(ti_cipher.key, sop.key, sop.keylen/8);
3019                 ti_cipher.keylen = sop.keylen;
3020                 switch (sop.cmode){
3021                 case cryptocop_cipher_mode_cbc:
3022                 case cryptocop_cipher_mode_ecb:
3023                         ti_cipher.cipher_mode = sop.cmode;
3024                         break;
3025                 default:
3026                         DEBUG_API(printk("create session, bad cipher mode %d\n", sop.cmode));
3027                         return -EINVAL;
3028                 }
3029                 DEBUG(printk("cryptocop_ioctl_create_session: setting CBC mode %d\n", ti_cipher.cipher_mode));
3030                 switch (sop.des3_mode){
3031                 case cryptocop_3des_eee:
3032                 case cryptocop_3des_eed:
3033                 case cryptocop_3des_ede:
3034                 case cryptocop_3des_edd:
3035                 case cryptocop_3des_dee:
3036                 case cryptocop_3des_ded:
3037                 case cryptocop_3des_dde:
3038                 case cryptocop_3des_ddd:
3039                         ti_cipher.tdes_mode = sop.des3_mode;
3040                         break;
3041                 default:
3042                         DEBUG_API(printk("create session, bad 3DES mode %d\n", sop.des3_mode));
3043                         return -EINVAL;
3044                 }
3045                 ti_cipher.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
3046                 ti_cipher.next = tis;
3047                 tis = &ti_cipher;
3048         } /* if (sop.cipher != cryptocop_cipher_none) */
3049         if (sop.digest != cryptocop_digest_none){
3050                 DEBUG(printk("setting digest transform\n"));
3051                 switch (sop.digest){
3052                 case cryptocop_digest_md5:
3053                         ti_digest.alg = cryptocop_alg_md5;
3054                         break;
3055                 case cryptocop_digest_sha1:
3056                         ti_digest.alg = cryptocop_alg_sha1;
3057                         break;
3058                 default:
3059                         DEBUG_API(printk("create session, bad digest algorithm %d\n", sop.digest));
3060                         return -EINVAL;
3061                 }
3062                 ti_digest.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
3063                 ti_digest.next = tis;
3064                 tis = &ti_digest;
3065         } /* if (sop.digest != cryptocop_digest_none) */
3066         if (sop.csum != cryptocop_csum_none){
3067                 DEBUG(printk("setting csum transform\n"));
3068                 switch (sop.csum){
3069                 case cryptocop_csum_le:
3070                 case cryptocop_csum_be:
3071                         ti_csum.csum_mode = sop.csum;
3072                         break;
3073                 default:
3074                         DEBUG_API(printk("create session, bad checksum algorithm %d\n", sop.csum));
3075                         return -EINVAL;
3076                 }
3077                 ti_csum.alg = cryptocop_alg_csum;
3078                 ti_csum.tid = CRYPTOCOP_IOCTL_CSUM_TID;
3079                 ti_csum.next = tis;
3080                 tis = &ti_csum;
3081         } /* (sop.csum != cryptocop_csum_none) */
3082         dev = kmalloc(sizeof(struct cryptocop_private), GFP_KERNEL);
3083         if (!dev){
3084                 DEBUG_API(printk("create session, alloc dev\n"));
3085                 return -ENOMEM;
3086         }
3087
3088         err = cryptocop_new_session(&sid, tis, GFP_KERNEL);
3089         DEBUG({ if (err) printk("create session, cryptocop_new_session %d\n", err);});
3090
3091         if (err) {
3092                 kfree(dev);
3093                 return err;
3094         }
3095         sess_op->ses_id = sid;
3096         dev->sid = sid;
3097         dev->next = filp->private_data;
3098         filp->private_data = dev;
3099
3100         return 0;
3101 }
3102
3103 static long cryptocop_ioctl_unlocked(struct inode *inode,
3104         struct file *filp, unsigned int cmd, unsigned long arg)
3105 {
3106         int err = 0;
3107         if (_IOC_TYPE(cmd) != ETRAXCRYPTOCOP_IOCTYPE) {
3108                 DEBUG_API(printk("cryptocop_ioctl: wrong type\n"));
3109                 return -ENOTTY;
3110         }
3111         if (_IOC_NR(cmd) > CRYPTOCOP_IO_MAXNR){
3112                 return -ENOTTY;
3113         }
3114         /* Access check of the argument.  Some commands, e.g. create session and process op,
3115            needs additional checks.  Those are handled in the command handling functions. */
3116         if (_IOC_DIR(cmd) & _IOC_READ)
3117                 err = !access_ok(VERIFY_WRITE, (void *)arg, _IOC_SIZE(cmd));
3118         else if (_IOC_DIR(cmd) & _IOC_WRITE)
3119                 err = !access_ok(VERIFY_READ, (void *)arg, _IOC_SIZE(cmd));
3120         if (err) return -EFAULT;
3121
3122         switch (cmd) {
3123         case CRYPTOCOP_IO_CREATE_SESSION:
3124                 return cryptocop_ioctl_create_session(inode, filp, cmd, arg);
3125         case CRYPTOCOP_IO_CLOSE_SESSION:
3126                 return cryptocop_ioctl_close_session(inode, filp, cmd, arg);
3127         case CRYPTOCOP_IO_PROCESS_OP:
3128                 return cryptocop_ioctl_process(inode, filp, cmd, arg);
3129         default:
3130                 DEBUG_API(printk("cryptocop_ioctl: unknown command\n"));
3131                 return -ENOTTY;
3132         }
3133         return 0;
3134 }
3135
3136 static long
3137 cryptocop_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
3138 {
3139        struct inode *inode = file->f_path.dentry->d_inode;
3140        long ret;
3141
3142        lock_kernel();
3143        ret = cryptocop_ioctl_unlocked(inode, filp, cmd, arg);
3144        unlock_kernel();
3145
3146        return ret;
3147 }
3148
3149
3150 #ifdef LDEBUG
3151 static void print_dma_descriptors(struct cryptocop_int_operation *iop)
3152 {
3153         struct cryptocop_dma_desc *cdesc_out = iop->cdesc_out;
3154         struct cryptocop_dma_desc *cdesc_in = iop->cdesc_in;
3155         int                       i;
3156
3157         printk("print_dma_descriptors start\n");
3158
3159         printk("iop:\n");
3160         printk("\tsid: 0x%lld\n", iop->sid);
3161
3162         printk("\tcdesc_out: 0x%p\n", iop->cdesc_out);
3163         printk("\tcdesc_in: 0x%p\n", iop->cdesc_in);
3164         printk("\tddesc_out: 0x%p\n", iop->ddesc_out);
3165         printk("\tddesc_in: 0x%p\n", iop->ddesc_in);
3166
3167         printk("\niop->ctx_out: 0x%p phys: 0x%p\n", &iop->ctx_out, (char*)virt_to_phys(&iop->ctx_out));
3168         printk("\tnext: 0x%p\n"
3169                "\tsaved_data: 0x%p\n"
3170                "\tsaved_data_buf: 0x%p\n",
3171                iop->ctx_out.next,
3172                iop->ctx_out.saved_data,
3173                iop->ctx_out.saved_data_buf);
3174
3175         printk("\niop->ctx_in: 0x%p phys: 0x%p\n", &iop->ctx_in, (char*)virt_to_phys(&iop->ctx_in));
3176         printk("\tnext: 0x%p\n"
3177                "\tsaved_data: 0x%p\n"
3178                "\tsaved_data_buf: 0x%p\n",
3179                iop->ctx_in.next,
3180                iop->ctx_in.saved_data,
3181                iop->ctx_in.saved_data_buf);
3182
3183         i = 0;
3184         while (cdesc_out) {
3185                 dma_descr_data *td;
3186                 printk("cdesc_out %d, desc=0x%p\n", i, cdesc_out->dma_descr);
3187                 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_out->dma_descr));
3188                 td = cdesc_out->dma_descr;
3189                 printk("\n\tbuf: 0x%p\n"
3190                        "\tafter: 0x%p\n"
3191                        "\tmd: 0x%04x\n"
3192                        "\tnext: 0x%p\n",
3193                        td->buf,
3194                        td->after,
3195                        td->md,
3196                        td->next);
3197                 printk("flags:\n"
3198                        "\twait:\t%d\n"
3199                        "\teol:\t%d\n"
3200                        "\touteop:\t%d\n"
3201                        "\tineop:\t%d\n"
3202                        "\tintr:\t%d\n",
3203                        td->wait,
3204                        td->eol,
3205                        td->out_eop,
3206                        td->in_eop,
3207                        td->intr);
3208                 cdesc_out = cdesc_out->next;
3209                 i++;
3210         }
3211         i = 0;
3212         while (cdesc_in) {
3213                 dma_descr_data *td;
3214                 printk("cdesc_in %d, desc=0x%p\n", i, cdesc_in->dma_descr);
3215                 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_in->dma_descr));
3216                 td = cdesc_in->dma_descr;
3217                 printk("\n\tbuf: 0x%p\n"
3218                        "\tafter: 0x%p\n"
3219                        "\tmd: 0x%04x\n"
3220                        "\tnext: 0x%p\n",
3221                        td->buf,
3222                        td->after,
3223                        td->md,
3224                        td->next);
3225                 printk("flags:\n"
3226                        "\twait:\t%d\n"
3227                        "\teol:\t%d\n"
3228                        "\touteop:\t%d\n"
3229                        "\tineop:\t%d\n"
3230                        "\tintr:\t%d\n",
3231                        td->wait,
3232                        td->eol,
3233                        td->out_eop,
3234                        td->in_eop,
3235                        td->intr);
3236                 cdesc_in = cdesc_in->next;
3237                 i++;
3238         }
3239
3240         printk("print_dma_descriptors end\n");
3241 }
3242
3243
3244 static void print_strcop_crypto_op(struct strcop_crypto_op *cop)
3245 {
3246         printk("print_strcop_crypto_op, 0x%p\n", cop);
3247
3248         /* Indata. */
3249         printk("indata=0x%p\n"
3250                "inlen=%d\n"
3251                "do_cipher=%d\n"
3252                "decrypt=%d\n"
3253                "cipher_explicit=%d\n"
3254                "cipher_start=%d\n"
3255                "cipher_len=%d\n"
3256                "outdata=0x%p\n"
3257                "outlen=%d\n",
3258                cop->indata,
3259                cop->inlen,
3260                cop->do_cipher,
3261                cop->decrypt,
3262                cop->cipher_explicit,
3263                cop->cipher_start,
3264                cop->cipher_len,
3265                cop->cipher_outdata,
3266                cop->cipher_outlen);
3267
3268         printk("do_digest=%d\n"
3269                "digest_start=%d\n"
3270                "digest_len=%d\n",
3271                cop->do_digest,
3272                cop->digest_start,
3273                cop->digest_len);
3274
3275         printk("do_csum=%d\n"
3276                "csum_start=%d\n"
3277                "csum_len=%d\n",
3278                cop->do_csum,
3279                cop->csum_start,
3280                cop->csum_len);
3281 }
3282
3283 static void print_cryptocop_operation(struct cryptocop_operation *cop)
3284 {
3285         struct cryptocop_desc      *d;
3286         struct cryptocop_tfrm_cfg  *tc;
3287         struct cryptocop_desc_cfg  *dc;
3288         int                        i;
3289
3290         printk("print_cryptocop_operation, cop=0x%p\n\n", cop);
3291         printk("sid: %lld\n", cop->sid);
3292         printk("operation_status=%d\n"
3293                "use_dmalists=%d\n"
3294                "in_interrupt=%d\n"
3295                "fast_callback=%d\n",
3296                cop->operation_status,
3297                cop->use_dmalists,
3298                cop->in_interrupt,
3299                cop->fast_callback);
3300
3301         if (cop->use_dmalists){
3302                 print_user_dma_lists(&cop->list_op);
3303         } else {
3304                 printk("cop->tfrm_op\n"
3305                        "tfrm_cfg=0x%p\n"
3306                        "desc=0x%p\n"
3307                        "indata=0x%p\n"
3308                        "incount=%d\n"
3309                        "inlen=%d\n"
3310                        "outdata=0x%p\n"
3311                        "outcount=%d\n"
3312                        "outlen=%d\n\n",
3313                        cop->tfrm_op.tfrm_cfg,
3314                        cop->tfrm_op.desc,
3315                        cop->tfrm_op.indata,
3316                        cop->tfrm_op.incount,
3317                        cop->tfrm_op.inlen,
3318                        cop->tfrm_op.outdata,
3319                        cop->tfrm_op.outcount,
3320                        cop->tfrm_op.outlen);
3321
3322                 tc = cop->tfrm_op.tfrm_cfg;
3323                 while (tc){
3324                         printk("tfrm_cfg, 0x%p\n"
3325                                "tid=%d\n"
3326                                "flags=%d\n"
3327                                "inject_ix=%d\n"
3328                                "next=0x%p\n",
3329                                tc,
3330                                tc->tid,
3331                                tc->flags,
3332                                tc->inject_ix,
3333                                tc->next);
3334                         tc = tc->next;
3335                 }
3336                 d = cop->tfrm_op.desc;
3337                 while (d){
3338                         printk("\n======================desc, 0x%p\n"
3339                                "length=%d\n"
3340                                "cfg=0x%p\n"
3341                                "next=0x%p\n",
3342                                d,
3343                                d->length,
3344                                d->cfg,
3345                                d->next);
3346                         dc = d->cfg;
3347                         while (dc){
3348                                 printk("=========desc_cfg, 0x%p\n"
3349                                        "tid=%d\n"
3350                                        "src=%d\n"
3351                                        "last=%d\n"
3352                                        "next=0x%p\n",
3353                                        dc,
3354                                        dc->tid,
3355                                        dc->src,
3356                                        dc->last,
3357                                        dc->next);
3358                                 dc = dc->next;
3359                         }
3360                         d = d->next;
3361                 }
3362                 printk("\n====iniov\n");
3363                 for (i = 0; i < cop->tfrm_op.incount; i++){
3364                         printk("indata[%d]\n"
3365                                "base=0x%p\n"
3366                                "len=%d\n",
3367                                i,
3368                                cop->tfrm_op.indata[i].iov_base,
3369                                cop->tfrm_op.indata[i].iov_len);
3370                 }
3371                 printk("\n====outiov\n");
3372                 for (i = 0; i < cop->tfrm_op.outcount; i++){
3373                         printk("outdata[%d]\n"
3374                                "base=0x%p\n"
3375                                "len=%d\n",
3376                                i,
3377                                cop->tfrm_op.outdata[i].iov_base,
3378                                cop->tfrm_op.outdata[i].iov_len);
3379                 }
3380         }
3381         printk("------------end print_cryptocop_operation\n");
3382 }
3383
3384
3385 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op)
3386 {
3387         dma_descr_data *dd;
3388         int i;
3389
3390         printk("print_user_dma_lists, dma_op=0x%p\n", dma_op);
3391
3392         printk("out_data_buf = 0x%p, phys_to_virt(out_data_buf) = 0x%p\n", dma_op->out_data_buf, phys_to_virt((unsigned long int)dma_op->out_data_buf));
3393         printk("in_data_buf = 0x%p, phys_to_virt(in_data_buf) = 0x%p\n", dma_op->in_data_buf, phys_to_virt((unsigned long int)dma_op->in_data_buf));
3394
3395         printk("##############outlist\n");
3396         dd = phys_to_virt((unsigned long int)dma_op->outlist);
3397         i = 0;
3398         while (dd != NULL) {
3399                 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3400                 printk("\n\tbuf: 0x%p\n"
3401                        "\tafter: 0x%p\n"
3402                        "\tmd: 0x%04x\n"
3403                        "\tnext: 0x%p\n",
3404                        dd->buf,
3405                        dd->after,
3406                        dd->md,
3407                        dd->next);
3408                 printk("flags:\n"
3409                        "\twait:\t%d\n"
3410                        "\teol:\t%d\n"
3411                        "\touteop:\t%d\n"
3412                        "\tineop:\t%d\n"
3413                        "\tintr:\t%d\n",
3414                        dd->wait,
3415                        dd->eol,
3416                        dd->out_eop,
3417                        dd->in_eop,
3418                        dd->intr);
3419                 if (dd->eol)
3420                         dd = NULL;
3421                 else
3422                         dd = phys_to_virt((unsigned long int)dd->next);
3423                 ++i;
3424         }
3425
3426         printk("##############inlist\n");
3427         dd = phys_to_virt((unsigned long int)dma_op->inlist);
3428         i = 0;
3429         while (dd != NULL) {
3430                 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3431                 printk("\n\tbuf: 0x%p\n"
3432                        "\tafter: 0x%p\n"
3433                        "\tmd: 0x%04x\n"
3434                        "\tnext: 0x%p\n",
3435                        dd->buf,
3436                        dd->after,
3437                        dd->md,
3438                        dd->next);
3439                 printk("flags:\n"
3440                        "\twait:\t%d\n"
3441                        "\teol:\t%d\n"
3442                        "\touteop:\t%d\n"
3443                        "\tineop:\t%d\n"
3444                        "\tintr:\t%d\n",
3445                        dd->wait,
3446                        dd->eol,
3447                        dd->out_eop,
3448                        dd->in_eop,
3449                        dd->intr);
3450                 if (dd->eol)
3451                         dd = NULL;
3452                 else
3453                         dd = phys_to_virt((unsigned long int)dd->next);
3454                 ++i;
3455         }
3456 }
3457
3458
3459 static void print_lock_status(void)
3460 {
3461         printk("**********************print_lock_status\n");
3462         printk("cryptocop_completed_jobs_lock %d\n", spin_is_locked(&cryptocop_completed_jobs_lock));
3463         printk("cryptocop_job_queue_lock %d\n", spin_is_locked(&cryptocop_job_queue_lock));
3464         printk("descr_pool_lock %d\n", spin_is_locked(&descr_pool_lock));
3465         printk("cryptocop_sessions_lock %d\n", spin_is_locked(cryptocop_sessions_lock));
3466         printk("running_job_lock %d\n", spin_is_locked(running_job_lock));
3467         printk("cryptocop_process_lock %d\n", spin_is_locked(cryptocop_process_lock));
3468 }
3469 #endif /* LDEBUG */
3470
3471
3472 static const char cryptocop_name[] = "ETRAX FS stream co-processor";
3473
3474 static int init_stream_coprocessor(void)
3475 {
3476         int err;
3477         int i;
3478         static int initialized = 0;
3479
3480         if (initialized)
3481                 return 0;
3482
3483         initialized = 1;
3484
3485         printk("ETRAX FS stream co-processor driver v0.01, (c) 2003 Axis Communications AB\n");
3486
3487         err = register_chrdev(CRYPTOCOP_MAJOR, cryptocop_name, &cryptocop_fops);
3488         if (err < 0) {
3489                 printk(KERN_ERR "stream co-processor: could not get major number.\n");
3490                 return err;
3491         }
3492
3493         err = init_cryptocop();
3494         if (err) {
3495                 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3496                 return err;
3497         }
3498         err = cryptocop_job_queue_init();
3499         if (err) {
3500                 release_cryptocop();
3501                 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3502                 return err;
3503         }
3504         /* Init the descriptor pool. */
3505         for (i = 0; i < CRYPTOCOP_DESCRIPTOR_POOL_SIZE - 1; i++) {
3506                 descr_pool[i].from_pool = 1;
3507                 descr_pool[i].next = &descr_pool[i + 1];
3508         }
3509         descr_pool[i].from_pool = 1;
3510         descr_pool[i].next = NULL;
3511         descr_pool_free_list = &descr_pool[0];
3512         descr_pool_no_free = CRYPTOCOP_DESCRIPTOR_POOL_SIZE;
3513
3514         spin_lock_init(&cryptocop_completed_jobs_lock);
3515         spin_lock_init(&cryptocop_job_queue_lock);
3516         spin_lock_init(&descr_pool_lock);
3517         spin_lock_init(&cryptocop_sessions_lock);
3518         spin_lock_init(&running_job_lock);
3519         spin_lock_init(&cryptocop_process_lock);
3520
3521         cryptocop_sessions = NULL;
3522         next_sid = 1;
3523
3524         cryptocop_running_job = NULL;
3525
3526         printk("stream co-processor: init done.\n");
3527         return 0;
3528 }
3529
3530 static void __exit exit_stream_coprocessor(void)
3531 {
3532         release_cryptocop();
3533         cryptocop_job_queue_close();
3534 }
3535
3536 module_init(init_stream_coprocessor);
3537 module_exit(exit_stream_coprocessor);
3538