arch/cris/arch-v32/drivers/cryptocop.c

   1 /*
   2  * Stream co-processor driver for the ETRAX FS
   3  *
   4  *    Copyright (C) 2003-2007  Axis Communications AB
   5  */
   6
   7 #include <linux/init.h>
   8 #include <linux/sched.h>
   9 #include <linux/module.h>
  10 #include <linux/slab.h>
  11 #include <linux/string.h>
  12 #include <linux/fs.h>
  13 #include <linux/mm.h>
  14 #include <linux/spinlock.h>
  15 #include <linux/stddef.h>
  16
  17 #include <asm/uaccess.h>
  18 #include <asm/io.h>
  19 #include <asm/atomic.h>
  20
  21 #include <linux/list.h>
  22 #include <linux/interrupt.h>
  23
  24 #include <asm/signal.h>
  25 #include <asm/irq.h>
  26
  27 #include <dma.h>
  28 #include <hwregs/dma.h>
  29 #include <hwregs/reg_map.h>
  30 #include <hwregs/reg_rdwr.h>
  31 #include <hwregs/intr_vect_defs.h>
  32
  33 #include <hwregs/strcop.h>
  34 #include <hwregs/strcop_defs.h>
  35 #include <cryptocop.h>
  36
  37 #ifdef CONFIG_ETRAXFS
  38 #define IN_DMA 9
  39 #define OUT_DMA 8
  40 #define IN_DMA_INST regi_dma9
  41 #define OUT_DMA_INST regi_dma8
  42 #define DMA_IRQ DMA9_INTR_VECT
  43 #else
  44 #define IN_DMA 3
  45 #define OUT_DMA 2
  46 #define IN_DMA_INST regi_dma3
  47 #define OUT_DMA_INST regi_dma2
  48 #define DMA_IRQ DMA3_INTR_VECT
  49 #endif
  50
  51 #define DESCR_ALLOC_PAD  (31)
  52
  53 struct cryptocop_dma_desc {
  54         char *free_buf; /* If non-null will be kfreed in free_cdesc() */
  55         dma_descr_data *dma_descr;
  56
  57         unsigned char dma_descr_buf[sizeof(dma_descr_data) + DESCR_ALLOC_PAD];
  58
  59         unsigned int from_pool:1; /* If 1 'allocated' from the descriptor pool. */
  60         struct cryptocop_dma_desc *next;
  61 };
  62
  63
  64 struct cryptocop_int_operation{
  65         void                        *alloc_ptr;
  66         cryptocop_session_id        sid;
  67
  68         dma_descr_context           ctx_out;
  69         dma_descr_context           ctx_in;
  70
  71         /* DMA descriptors allocated by driver. */
  72         struct cryptocop_dma_desc   *cdesc_out;
  73         struct cryptocop_dma_desc   *cdesc_in;
  74
  75         /* Strcop config to use. */
  76         cryptocop_3des_mode         tdes_mode;
  77         cryptocop_csum_type         csum_mode;
  78
  79         /* DMA descrs provided by consumer. */
  80         dma_descr_data              *ddesc_out;
  81         dma_descr_data              *ddesc_in;
  82 };
  83
  84
  85 struct cryptocop_tfrm_ctx {
  86         cryptocop_tfrm_id tid;
  87         unsigned int blocklength;
  88
  89         unsigned int start_ix;
  90
  91         struct cryptocop_tfrm_cfg *tcfg;
  92         struct cryptocop_transform_ctx *tctx;
  93
  94         unsigned char previous_src;
  95         unsigned char current_src;
  96
  97         /* Values to use in metadata out. */
  98         unsigned char hash_conf;
  99         unsigned char hash_mode;
 100         unsigned char ciph_conf;
 101         unsigned char cbcmode;
 102         unsigned char decrypt;
 103
 104         unsigned int requires_padding:1;
 105         unsigned int strict_block_length:1;
 106         unsigned int active:1;
 107         unsigned int done:1;
 108         size_t consumed;
 109         size_t produced;
 110
 111         /* Pad (input) descriptors to put in the DMA out list when the transform
 112          * output is put on the DMA in list. */
 113         struct cryptocop_dma_desc *pad_descs;
 114
 115         struct cryptocop_tfrm_ctx *prev_src;
 116         struct cryptocop_tfrm_ctx *curr_src;
 117
 118         /* Mapping to HW. */
 119         unsigned char unit_no;
 120 };
 121
 122
 123 struct cryptocop_private{
 124         cryptocop_session_id sid;
 125         struct cryptocop_private *next;
 126 };
 127
 128 /* Session list. */
 129
 130 struct cryptocop_transform_ctx{
 131         struct cryptocop_transform_init init;
 132         unsigned char dec_key[CRYPTOCOP_MAX_KEY_LENGTH];
 133         unsigned int dec_key_set:1;
 134
 135         struct cryptocop_transform_ctx *next;
 136 };
 137
 138
 139 struct cryptocop_session{
 140         cryptocop_session_id sid;
 141
 142         struct cryptocop_transform_ctx *tfrm_ctx;
 143
 144         struct cryptocop_session *next;
 145 };
 146
 147 /* Priority levels for jobs sent to the cryptocop.  Checksum operations from
 148    kernel have highest priority since TCPIP stack processing must not
 149    be a bottleneck. */
 150 typedef enum {
 151         cryptocop_prio_kernel_csum = 0,
 152         cryptocop_prio_kernel = 1,
 153         cryptocop_prio_user = 2,
 154         cryptocop_prio_no_prios = 3
 155 } cryptocop_queue_priority;
 156
 157 struct cryptocop_prio_queue{
 158         struct list_head jobs;
 159         cryptocop_queue_priority prio;
 160 };
 161
 162 struct cryptocop_prio_job{
 163         struct list_head node;
 164         cryptocop_queue_priority prio;
 165
 166         struct cryptocop_operation *oper;
 167         struct cryptocop_int_operation *iop;
 168 };
 169
 170 struct ioctl_job_cb_ctx {
 171         unsigned int processed:1;
 172 };
 173
 174
 175 static struct cryptocop_session *cryptocop_sessions = NULL;
 176 spinlock_t cryptocop_sessions_lock;
 177
 178 /* Next Session ID to assign. */
 179 static cryptocop_session_id next_sid = 1;
 180
 181 /* Pad for checksum. */
 182 static const char csum_zero_pad[1] = {0x00};
 183
 184 /* Trash buffer for mem2mem operations. */
 185 #define MEM2MEM_DISCARD_BUF_LENGTH  (512)
 186 static unsigned char mem2mem_discard_buf[MEM2MEM_DISCARD_BUF_LENGTH];
 187
 188 /* Descriptor pool. */
 189 /* FIXME Tweak this value. */
 190 #define CRYPTOCOP_DESCRIPTOR_POOL_SIZE   (100)
 191 static struct cryptocop_dma_desc descr_pool[CRYPTOCOP_DESCRIPTOR_POOL_SIZE];
 192 static struct cryptocop_dma_desc *descr_pool_free_list;
 193 static int descr_pool_no_free;
 194 static spinlock_t descr_pool_lock;
 195
 196 /* Lock to stop cryptocop to start processing of a new operation. The holder
 197    of this lock MUST call cryptocop_start_job() after it is unlocked. */
 198 spinlock_t cryptocop_process_lock;
 199
 200 static struct cryptocop_prio_queue cryptocop_job_queues[cryptocop_prio_no_prios];
 201 static spinlock_t cryptocop_job_queue_lock;
 202 static struct cryptocop_prio_job *cryptocop_running_job = NULL;
 203 static spinlock_t running_job_lock;
 204
 205 /* The interrupt handler appends completed jobs to this list. The scehduled
 206  * tasklet removes them upon sending the response to the crypto consumer. */
 207 static struct list_head cryptocop_completed_jobs;
 208 static spinlock_t cryptocop_completed_jobs_lock;
 209
 210 DECLARE_WAIT_QUEUE_HEAD(cryptocop_ioc_process_wq);
 211
 212
 213 /** Local functions. **/
 214
 215 static int cryptocop_open(struct inode *, struct file *);
 216
 217 static int cryptocop_release(struct inode *, struct file *);
 218
 219 static long cryptocop_ioctl(struct file *file,
 220                            unsigned int cmd, unsigned long arg);
 221
 222 static void cryptocop_start_job(void);
 223
 224 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation);
 225 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation);
 226
 227 static int cryptocop_job_queue_init(void);
 228 static void cryptocop_job_queue_close(void);
 229
 230 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
 231
 232 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
 233
 234 static int transform_ok(struct cryptocop_transform_init *tinit);
 235
 236 static struct cryptocop_session *get_session(cryptocop_session_id sid);
 237
 238 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid);
 239
 240 static void delete_internal_operation(struct cryptocop_int_operation *iop);
 241
 242 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned  char *key, unsigned int keylength);
 243
 244 static int init_stream_coprocessor(void);
 245
 246 static void __exit exit_stream_coprocessor(void);
 247
 248 /*#define LDEBUG*/
 249 #ifdef LDEBUG
 250 #define DEBUG(s) s
 251 #define DEBUG_API(s) s
 252 static void print_cryptocop_operation(struct cryptocop_operation *cop);
 253 static void print_dma_descriptors(struct cryptocop_int_operation *iop);
 254 static void print_strcop_crypto_op(struct strcop_crypto_op *cop);
 255 static void print_lock_status(void);
 256 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op);
 257 #define assert(s) do{if (!(s)) panic(#s);} while(0);
 258 #else
 259 #define DEBUG(s)
 260 #define DEBUG_API(s)
 261 #define assert(s)
 262 #endif
 263
 264
 265 /* Transform constants. */
 266 #define DES_BLOCK_LENGTH   (8)
 267 #define AES_BLOCK_LENGTH   (16)
 268 #define MD5_BLOCK_LENGTH   (64)
 269 #define SHA1_BLOCK_LENGTH  (64)
 270 #define CSUM_BLOCK_LENGTH  (2)
 271 #define MD5_STATE_LENGTH   (16)
 272 #define SHA1_STATE_LENGTH  (20)
 273
 274 /* The device number. */
 275 #define CRYPTOCOP_MAJOR    (254)
 276 #define CRYPTOCOP_MINOR    (0)
 277
 278
 279
 280 const struct file_operations cryptocop_fops = {
 281         .owner          = THIS_MODULE,
 282         .open           = cryptocop_open,
 283         .release        = cryptocop_release,
 284         .unlocked_ioctl = cryptocop_ioctl,
 285         .llseek         = noop_llseek,
 286 };
 287
 288
 289 static void free_cdesc(struct cryptocop_dma_desc *cdesc)
 290 {
 291         DEBUG(printk("free_cdesc: cdesc 0x%p, from_pool=%d\n", cdesc, cdesc->from_pool));
 292         kfree(cdesc->free_buf);
 293
 294         if (cdesc->from_pool) {
 295                 unsigned long int flags;
 296                 spin_lock_irqsave(&descr_pool_lock, flags);
 297                 cdesc->next = descr_pool_free_list;
 298                 descr_pool_free_list = cdesc;
 299                 ++descr_pool_no_free;
 300                 spin_unlock_irqrestore(&descr_pool_lock, flags);
 301         } else {
 302                 kfree(cdesc);
 303         }
 304 }
 305
 306
 307 static struct cryptocop_dma_desc *alloc_cdesc(int alloc_flag)
 308 {
 309         int use_pool = (alloc_flag & GFP_ATOMIC) ? 1 : 0;
 310         struct cryptocop_dma_desc *cdesc;
 311
 312         if (use_pool) {
 313                 unsigned long int flags;
 314                 spin_lock_irqsave(&descr_pool_lock, flags);
 315                 if (!descr_pool_free_list) {
 316                         spin_unlock_irqrestore(&descr_pool_lock, flags);
 317                         DEBUG_API(printk("alloc_cdesc: pool is empty\n"));
 318                         return NULL;
 319                 }
 320                 cdesc = descr_pool_free_list;
 321                 descr_pool_free_list = descr_pool_free_list->next;
 322                 --descr_pool_no_free;
 323                 spin_unlock_irqrestore(&descr_pool_lock, flags);
 324                 cdesc->from_pool = 1;
 325         } else {
 326                 cdesc = kmalloc(sizeof(struct cryptocop_dma_desc), alloc_flag);
 327                 if (!cdesc) {
 328                         DEBUG_API(printk("alloc_cdesc: kmalloc\n"));
 329                         return NULL;
 330                 }
 331                 cdesc->from_pool = 0;
 332         }
 333         cdesc->dma_descr = (dma_descr_data*)(((unsigned long int)cdesc + offsetof(struct cryptocop_dma_desc, dma_descr_buf) + DESCR_ALLOC_PAD) & ~0x0000001F);
 334
 335         cdesc->next = NULL;
 336
 337         cdesc->free_buf = NULL;
 338         cdesc->dma_descr->out_eop = 0;
 339         cdesc->dma_descr->in_eop = 0;
 340         cdesc->dma_descr->intr = 0;
 341         cdesc->dma_descr->eol = 0;
 342         cdesc->dma_descr->wait = 0;
 343         cdesc->dma_descr->buf = NULL;
 344         cdesc->dma_descr->after = NULL;
 345
 346         DEBUG_API(printk("alloc_cdesc: return 0x%p, cdesc->dma_descr=0x%p, from_pool=%d\n", cdesc, cdesc->dma_descr, cdesc->from_pool));
 347         return cdesc;
 348 }
 349
 350
 351 static void setup_descr_chain(struct cryptocop_dma_desc *cd)
 352 {
 353         DEBUG(printk("setup_descr_chain: entering\n"));
 354         while (cd) {
 355                 if (cd->next) {
 356                         cd->dma_descr->next = (dma_descr_data*)virt_to_phys(cd->next->dma_descr);
 357                 } else {
 358                         cd->dma_descr->next = NULL;
 359                 }
 360                 cd = cd->next;
 361         }
 362         DEBUG(printk("setup_descr_chain: exit\n"));
 363 }
 364
 365
 366 /* Create a pad descriptor for the transform.
 367  * Return -1 for error, 0 if pad created. */
 368 static int create_pad_descriptor(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **pad_desc, int alloc_flag)
 369 {
 370         struct cryptocop_dma_desc        *cdesc = NULL;
 371         int                              error = 0;
 372         struct strcop_meta_out           mo = {
 373                 .ciphsel = src_none,
 374                 .hashsel = src_none,
 375                 .csumsel = src_none
 376         };
 377         char                             *pad;
 378         size_t                           plen;
 379
 380         DEBUG(printk("create_pad_descriptor: start.\n"));
 381         /* Setup pad descriptor. */
 382
 383         DEBUG(printk("create_pad_descriptor: setting up padding.\n"));
 384         cdesc = alloc_cdesc(alloc_flag);
 385         if (!cdesc){
 386                 DEBUG_API(printk("create_pad_descriptor: alloc pad desc\n"));
 387                 goto error_cleanup;
 388         }
 389         switch (tc->unit_no) {
 390         case src_md5:
 391                 error = create_md5_pad(alloc_flag, tc->consumed, &pad, &plen);
 392                 if (error){
 393                         DEBUG_API(printk("create_pad_descriptor: create_md5_pad_failed\n"));
 394                         goto error_cleanup;
 395                 }
 396                 cdesc->free_buf = pad;
 397                 mo.hashsel = src_dma;
 398                 mo.hashconf = tc->hash_conf;
 399                 mo.hashmode = tc->hash_mode;
 400                 break;
 401         case src_sha1:
 402                 error = create_sha1_pad(alloc_flag, tc->consumed, &pad, &plen);
 403                 if (error){
 404                         DEBUG_API(printk("create_pad_descriptor: create_sha1_pad_failed\n"));
 405                         goto error_cleanup;
 406                 }
 407                 cdesc->free_buf = pad;
 408                 mo.hashsel = src_dma;
 409                 mo.hashconf = tc->hash_conf;
 410                 mo.hashmode = tc->hash_mode;
 411                 break;
 412         case src_csum:
 413                 if (tc->consumed % tc->blocklength){
 414                         pad = (char*)csum_zero_pad;
 415                         plen = 1;
 416                 } else {
 417                         pad = (char*)cdesc; /* Use any pointer. */
 418                         plen = 0;
 419                 }
 420                 mo.csumsel = src_dma;
 421                 break;
 422         }
 423         cdesc->dma_descr->wait = 1;
 424         cdesc->dma_descr->out_eop = 1; /* Since this is a pad output is pushed.  EOP is ok here since the padded unit is the only one active. */
 425         cdesc->dma_descr->buf = (char*)virt_to_phys((char*)pad);
 426         cdesc->dma_descr->after = cdesc->dma_descr->buf + plen;
 427
 428         cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
 429         *pad_desc = cdesc;
 430
 431         return 0;
 432
 433  error_cleanup:
 434         if (cdesc) free_cdesc(cdesc);
 435         return -1;
 436 }
 437
 438
 439 static int setup_key_dl_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **kd, int alloc_flag)
 440 {
 441         struct cryptocop_dma_desc  *key_desc = alloc_cdesc(alloc_flag);
 442         struct strcop_meta_out     mo = {0};
 443
 444         DEBUG(printk("setup_key_dl_desc\n"));
 445
 446         if (!key_desc) {
 447                 DEBUG_API(printk("setup_key_dl_desc: failed descriptor allocation.\n"));
 448                 return -ENOMEM;
 449         }
 450
 451         /* Download key. */
 452         if ((tc->tctx->init.alg == cryptocop_alg_aes) && (tc->tcfg->flags & CRYPTOCOP_DECRYPT)) {
 453                 /* Precook the AES decrypt key. */
 454                 if (!tc->tctx->dec_key_set){
 455                         get_aes_decrypt_key(tc->tctx->dec_key, tc->tctx->init.key, tc->tctx->init.keylen);
 456                         tc->tctx->dec_key_set = 1;
 457                 }
 458                 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->dec_key);
 459                 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
 460         } else {
 461                 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->init.key);
 462                 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
 463         }
 464         /* Setup metadata. */
 465         mo.dlkey = 1;
 466         switch (tc->tctx->init.keylen) {
 467         case 64:
 468                 mo.decrypt = 0;
 469                 mo.hashmode = 0;
 470                 break;
 471         case 128:
 472                 mo.decrypt = 0;
 473                 mo.hashmode = 1;
 474                 break;
 475         case 192:
 476                 mo.decrypt = 1;
 477                 mo.hashmode = 0;
 478                 break;
 479         case 256:
 480                 mo.decrypt = 1;
 481                 mo.hashmode = 1;
 482                 break;
 483         default:
 484                 break;
 485         }
 486         mo.ciphsel = mo.hashsel = mo.csumsel = src_none;
 487         key_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
 488
 489         key_desc->dma_descr->out_eop = 1;
 490         key_desc->dma_descr->wait = 1;
 491         key_desc->dma_descr->intr = 0;
 492
 493         *kd = key_desc;
 494         return 0;
 495 }
 496
 497 static int setup_cipher_iv_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
 498 {
 499         struct cryptocop_dma_desc  *iv_desc = alloc_cdesc(alloc_flag);
 500         struct strcop_meta_out     mo = {0};
 501
 502         DEBUG(printk("setup_cipher_iv_desc\n"));
 503
 504         if (!iv_desc) {
 505                 DEBUG_API(printk("setup_cipher_iv_desc: failed CBC IV descriptor allocation.\n"));
 506                 return -ENOMEM;
 507         }
 508         /* Download IV. */
 509         iv_desc->dma_descr->buf = (char*)virt_to_phys(tc->tcfg->iv);
 510         iv_desc->dma_descr->after = iv_desc->dma_descr->buf + tc->blocklength;
 511
 512         /* Setup metadata. */
 513         mo.hashsel = mo.csumsel = src_none;
 514         mo.ciphsel = src_dma;
 515         mo.ciphconf = tc->ciph_conf;
 516         mo.cbcmode = tc->cbcmode;
 517
 518         iv_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
 519
 520         iv_desc->dma_descr->out_eop = 0;
 521         iv_desc->dma_descr->wait = 1;
 522         iv_desc->dma_descr->intr = 0;
 523
 524         *id = iv_desc;
 525         return 0;
 526 }
 527
 528 /* Map the ouput length of the transform to operation output starting on the inject index. */
 529 static int create_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
 530 {
 531         int                        err = 0;
 532         struct cryptocop_dma_desc  head = {0};
 533         struct cryptocop_dma_desc  *outdesc = &head;
 534         size_t                     iov_offset = 0;
 535         size_t                     out_ix = 0;
 536         int                        outiov_ix = 0;
 537         struct strcop_meta_in      mi = {0};
 538
 539         size_t                     out_length = tc->produced;
 540         int                        rem_length;
 541         int                        dlength;
 542
 543         assert(out_length != 0);
 544         if (((tc->produced + tc->tcfg->inject_ix) > operation->tfrm_op.outlen) || (tc->produced && (operation->tfrm_op.outlen == 0))) {
 545                 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
 546                 return -EINVAL;
 547         }
 548         /* Traverse the out iovec until the result inject index is reached. */
 549         while ((outiov_ix < operation->tfrm_op.outcount) && ((out_ix + operation->tfrm_op.outdata[outiov_ix].iov_len) <= tc->tcfg->inject_ix)){
 550                 out_ix += operation->tfrm_op.outdata[outiov_ix].iov_len;
 551                 outiov_ix++;
 552         }
 553         if (outiov_ix >= operation->tfrm_op.outcount){
 554                 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
 555                 return -EINVAL;
 556         }
 557         iov_offset = tc->tcfg->inject_ix - out_ix;
 558         mi.dmasel = tc->unit_no;
 559
 560         /* Setup the output descriptors. */
 561         while ((out_length > 0) && (outiov_ix < operation->tfrm_op.outcount)) {
 562                 outdesc->next = alloc_cdesc(alloc_flag);
 563                 if (!outdesc->next) {
 564                         DEBUG_API(printk("create_input_descriptors: alloc_cdesc\n"));
 565                         err = -ENOMEM;
 566                         goto error_cleanup;
 567                 }
 568                 outdesc = outdesc->next;
 569                 rem_length = operation->tfrm_op.outdata[outiov_ix].iov_len - iov_offset;
 570                 dlength = (out_length < rem_length) ? out_length : rem_length;
 571
 572                 DEBUG(printk("create_input_descriptors:\n"
 573                              "outiov_ix=%d, rem_length=%d, dlength=%d\n"
 574                              "iov_offset=%d, outdata[outiov_ix].iov_len=%d\n"
 575                              "outcount=%d, outiov_ix=%d\n",
 576                              outiov_ix, rem_length, dlength, iov_offset, operation->tfrm_op.outdata[outiov_ix].iov_len, operation->tfrm_op.outcount, outiov_ix));
 577
 578                 outdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.outdata[outiov_ix].iov_base + iov_offset);
 579                 outdesc->dma_descr->after = outdesc->dma_descr->buf + dlength;
 580                 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 581
 582                 out_length -= dlength;
 583                 iov_offset += dlength;
 584                 if (iov_offset >= operation->tfrm_op.outdata[outiov_ix].iov_len) {
 585                         iov_offset = 0;
 586                         ++outiov_ix;
 587                 }
 588         }
 589         if (out_length > 0){
 590                 DEBUG_API(printk("create_input_descriptors: not enough room for output, %d remained\n", out_length));
 591                 err = -EINVAL;
 592                 goto error_cleanup;
 593         }
 594         /* Set sync in last descriptor. */
 595         mi.sync = 1;
 596         outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 597
 598         *id = head.next;
 599         return 0;
 600
 601  error_cleanup:
 602         while (head.next) {
 603                 outdesc = head.next->next;
 604                 free_cdesc(head.next);
 605                 head.next = outdesc;
 606         }
 607         return err;
 608 }
 609
 610
 611 static int create_output_descriptors(struct cryptocop_operation *operation, int *iniov_ix, int *iniov_offset, size_t desc_len, struct cryptocop_dma_desc **current_out_cdesc, struct strcop_meta_out *meta_out, int alloc_flag)
 612 {
 613         while (desc_len != 0) {
 614                 struct cryptocop_dma_desc  *cdesc;
 615                 int                        rem_length = operation->tfrm_op.indata[*iniov_ix].iov_len - *iniov_offset;
 616                 int                        dlength = (desc_len < rem_length) ? desc_len : rem_length;
 617
 618                 cdesc = alloc_cdesc(alloc_flag);
 619                 if (!cdesc) {
 620                         DEBUG_API(printk("create_output_descriptors: alloc_cdesc\n"));
 621                         return -ENOMEM;
 622                 }
 623                 (*current_out_cdesc)->next = cdesc;
 624                 (*current_out_cdesc) = cdesc;
 625
 626                 cdesc->free_buf = NULL;
 627
 628                 cdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.indata[*iniov_ix].iov_base + *iniov_offset);
 629                 cdesc->dma_descr->after = cdesc->dma_descr->buf + dlength;
 630
 631                 assert(desc_len >= dlength);
 632                 desc_len -= dlength;
 633                 *iniov_offset += dlength;
 634                 if (*iniov_offset >= operation->tfrm_op.indata[*iniov_ix].iov_len) {
 635                         *iniov_offset = 0;
 636                         ++(*iniov_ix);
 637                         if (*iniov_ix > operation->tfrm_op.incount) {
 638                                 DEBUG_API(printk("create_output_descriptors: not enough indata in operation."));
 639                                 return  -EINVAL;
 640                         }
 641                 }
 642                 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, (*meta_out));
 643         } /* while (desc_len != 0) */
 644         /* Last DMA descriptor gets a 'wait' bit to signal expected change in metadata. */
 645         (*current_out_cdesc)->dma_descr->wait = 1; /* This will set extraneous WAIT in some situations, e.g. when padding hashes and checksums. */
 646
 647         return 0;
 648 }
 649
 650
 651 static int append_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_dma_desc **current_in_cdesc, struct cryptocop_dma_desc **current_out_cdesc, struct cryptocop_tfrm_ctx *tc, int alloc_flag)
 652 {
 653         DEBUG(printk("append_input_descriptors, tc=0x%p, unit_no=%d\n", tc, tc->unit_no));
 654         if (tc->tcfg) {
 655                 int                        failed = 0;
 656                 struct cryptocop_dma_desc  *idescs = NULL;
 657                 DEBUG(printk("append_input_descriptors: pushing output, consumed %d produced %d bytes.\n", tc->consumed, tc->produced));
 658                 if (tc->pad_descs) {
 659                         DEBUG(printk("append_input_descriptors: append pad descriptors to DMA out list.\n"));
 660                         while (tc->pad_descs) {
 661                                 DEBUG(printk("append descriptor 0x%p\n", tc->pad_descs));
 662                                 (*current_out_cdesc)->next = tc->pad_descs;
 663                                 tc->pad_descs = tc->pad_descs->next;
 664                                 (*current_out_cdesc) = (*current_out_cdesc)->next;
 665                         }
 666                 }
 667
 668                 /* Setup and append output descriptors to DMA in list. */
 669                 if (tc->unit_no == src_dma){
 670                         /* mem2mem.  Setup DMA in descriptors to discard all input prior to the requested mem2mem data. */
 671                         struct strcop_meta_in mi = {.sync = 0, .dmasel = src_dma};
 672                         unsigned int start_ix = tc->start_ix;
 673                         while (start_ix){
 674                                 unsigned int desclen = start_ix < MEM2MEM_DISCARD_BUF_LENGTH ? start_ix : MEM2MEM_DISCARD_BUF_LENGTH;
 675                                 (*current_in_cdesc)->next = alloc_cdesc(alloc_flag);
 676                                 if (!(*current_in_cdesc)->next){
 677                                         DEBUG_API(printk("append_input_descriptors: alloc_cdesc mem2mem discard failed\n"));
 678                                         return -ENOMEM;
 679                                 }
 680                                 (*current_in_cdesc) = (*current_in_cdesc)->next;
 681                                 (*current_in_cdesc)->dma_descr->buf = (char*)virt_to_phys(mem2mem_discard_buf);
 682                                 (*current_in_cdesc)->dma_descr->after = (*current_in_cdesc)->dma_descr->buf + desclen;
 683                                 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 684                                 start_ix -= desclen;
 685                         }
 686                         mi.sync = 1;
 687                         (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 688                 }
 689
 690                 failed = create_input_descriptors(operation, tc, &idescs, alloc_flag);
 691                 if (failed){
 692                         DEBUG_API(printk("append_input_descriptors: output descriptor setup failed\n"));
 693                         return failed;
 694                 }
 695                 DEBUG(printk("append_input_descriptors: append output descriptors to DMA in list.\n"));
 696                 while (idescs) {
 697                         DEBUG(printk("append descriptor 0x%p\n", idescs));
 698                         (*current_in_cdesc)->next = idescs;
 699                         idescs = idescs->next;
 700                         (*current_in_cdesc) = (*current_in_cdesc)->next;
 701                 }
 702         }
 703         return 0;
 704 }
 705
 706
 707
 708 static int cryptocop_setup_dma_list(struct cryptocop_operation *operation, struct cryptocop_int_operation **int_op, int alloc_flag)
 709 {
 710         struct cryptocop_session *sess;
 711         struct cryptocop_transform_ctx *tctx;
 712
 713         struct cryptocop_tfrm_ctx digest_ctx = {
 714                 .previous_src = src_none,
 715                 .current_src = src_none,
 716                 .start_ix = 0,
 717                 .requires_padding = 1,
 718                 .strict_block_length = 0,
 719                 .hash_conf = 0,
 720                 .hash_mode = 0,
 721                 .ciph_conf = 0,
 722                 .cbcmode = 0,
 723                 .decrypt = 0,
 724                 .consumed = 0,
 725                 .produced = 0,
 726                 .pad_descs = NULL,
 727                 .active = 0,
 728                 .done = 0,
 729                 .prev_src = NULL,
 730                 .curr_src = NULL,
 731                 .tcfg = NULL};
 732         struct cryptocop_tfrm_ctx cipher_ctx = {
 733                 .previous_src = src_none,
 734                 .current_src = src_none,
 735                 .start_ix = 0,
 736                 .requires_padding = 0,
 737                 .strict_block_length = 1,
 738                 .hash_conf = 0,
 739                 .hash_mode = 0,
 740                 .ciph_conf = 0,
 741                 .cbcmode = 0,
 742                 .decrypt = 0,
 743                 .consumed = 0,
 744                 .produced = 0,
 745                 .pad_descs = NULL,
 746                 .active = 0,
 747                 .done = 0,
 748                 .prev_src = NULL,
 749                 .curr_src = NULL,
 750                 .tcfg = NULL};
 751         struct cryptocop_tfrm_ctx csum_ctx = {
 752                 .previous_src = src_none,
 753                 .current_src = src_none,
 754                 .start_ix = 0,
 755                 .blocklength = 2,
 756                 .requires_padding = 1,
 757                 .strict_block_length = 0,
 758                 .hash_conf = 0,
 759                 .hash_mode = 0,
 760                 .ciph_conf = 0,
 761                 .cbcmode = 0,
 762                 .decrypt = 0,
 763                 .consumed = 0,
 764                 .produced = 0,
 765                 .pad_descs = NULL,
 766                 .active = 0,
 767                 .done = 0,
 768                 .tcfg = NULL,
 769                 .prev_src = NULL,
 770                 .curr_src = NULL,
 771                 .unit_no = src_csum};
 772         struct cryptocop_tfrm_cfg *tcfg = operation->tfrm_op.tfrm_cfg;
 773
 774         unsigned int indata_ix = 0;
 775
 776         /* iovec accounting. */
 777         int iniov_ix = 0;
 778         int iniov_offset = 0;
 779
 780         /* Operation descriptor cfg traversal pointer. */
 781         struct cryptocop_desc *odsc;
 782
 783         int failed = 0;
 784         /* List heads for allocated descriptors. */
 785         struct cryptocop_dma_desc out_cdesc_head = {0};
 786         struct cryptocop_dma_desc in_cdesc_head = {0};
 787
 788         struct cryptocop_dma_desc *current_out_cdesc = &out_cdesc_head;
 789         struct cryptocop_dma_desc *current_in_cdesc = &in_cdesc_head;
 790
 791         struct cryptocop_tfrm_ctx *output_tc = NULL;
 792         void                      *iop_alloc_ptr;
 793
 794         assert(operation != NULL);
 795         assert(int_op != NULL);
 796
 797         DEBUG(printk("cryptocop_setup_dma_list: start\n"));
 798         DEBUG(print_cryptocop_operation(operation));
 799
 800         sess = get_session(operation->sid);
 801         if (!sess) {
 802                 DEBUG_API(printk("cryptocop_setup_dma_list: no session found for operation.\n"));
 803                 failed = -EINVAL;
 804                 goto error_cleanup;
 805         }
 806         iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
 807         if (!iop_alloc_ptr) {
 808                 DEBUG_API(printk("cryptocop_setup_dma_list:  kmalloc cryptocop_int_operation\n"));
 809                 failed = -ENOMEM;
 810                 goto error_cleanup;
 811         }
 812         (*int_op) = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
 813         DEBUG(memset((*int_op), 0xff, sizeof(struct cryptocop_int_operation)));
 814         (*int_op)->alloc_ptr = iop_alloc_ptr;
 815         DEBUG(printk("cryptocop_setup_dma_list: *int_op=0x%p, alloc_ptr=0x%p\n", *int_op, (*int_op)->alloc_ptr));
 816
 817         (*int_op)->sid = operation->sid;
 818         (*int_op)->cdesc_out = NULL;
 819         (*int_op)->cdesc_in = NULL;
 820         (*int_op)->tdes_mode = cryptocop_3des_ede;
 821         (*int_op)->csum_mode = cryptocop_csum_le;
 822         (*int_op)->ddesc_out = NULL;
 823         (*int_op)->ddesc_in = NULL;
 824
 825         /* Scan operation->tfrm_op.tfrm_cfg for bad configuration and set up the local contexts. */
 826         if (!tcfg) {
 827                 DEBUG_API(printk("cryptocop_setup_dma_list: no configured transforms in operation.\n"));
 828                 failed = -EINVAL;
 829                 goto error_cleanup;
 830         }
 831         while (tcfg) {
 832                 tctx = get_transform_ctx(sess, tcfg->tid);
 833                 if (!tctx) {
 834                         DEBUG_API(printk("cryptocop_setup_dma_list: no transform id %d in session.\n", tcfg->tid));
 835                         failed = -EINVAL;
 836                         goto error_cleanup;
 837                 }
 838                 if (tcfg->inject_ix > operation->tfrm_op.outlen){
 839                         DEBUG_API(printk("cryptocop_setup_dma_list: transform id %d inject_ix (%d) > operation->tfrm_op.outlen(%d)", tcfg->tid, tcfg->inject_ix, operation->tfrm_op.outlen));
 840                         failed = -EINVAL;
 841                         goto error_cleanup;
 842                 }
 843                 switch (tctx->init.alg){
 844                 case cryptocop_alg_mem2mem:
 845                         if (cipher_ctx.tcfg != NULL){
 846                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
 847                                 failed = -EINVAL;
 848                                 goto error_cleanup;
 849                         }
 850                         /* mem2mem is handled as a NULL cipher. */
 851                         cipher_ctx.cbcmode = 0;
 852                         cipher_ctx.decrypt = 0;
 853                         cipher_ctx.blocklength = 1;
 854                         cipher_ctx.ciph_conf = 0;
 855                         cipher_ctx.unit_no = src_dma;
 856                         cipher_ctx.tcfg = tcfg;
 857                         cipher_ctx.tctx = tctx;
 858                         break;
 859                 case cryptocop_alg_des:
 860                 case cryptocop_alg_3des:
 861                 case cryptocop_alg_aes:
 862                         /* cipher */
 863                         if (cipher_ctx.tcfg != NULL){
 864                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
 865                                 failed = -EINVAL;
 866                                 goto error_cleanup;
 867                         }
 868                         cipher_ctx.tcfg = tcfg;
 869                         cipher_ctx.tctx = tctx;
 870                         if (cipher_ctx.tcfg->flags & CRYPTOCOP_DECRYPT){
 871                                 cipher_ctx.decrypt = 1;
 872                         }
 873                         switch (tctx->init.cipher_mode) {
 874                         case cryptocop_cipher_mode_ecb:
 875                                 cipher_ctx.cbcmode = 0;
 876                                 break;
 877                         case cryptocop_cipher_mode_cbc:
 878                                 cipher_ctx.cbcmode = 1;
 879                                 break;
 880                         default:
 881                                 DEBUG_API(printk("cryptocop_setup_dma_list: cipher_ctx, bad cipher mode==%d\n", tctx->init.cipher_mode));
 882                                 failed = -EINVAL;
 883                                 goto error_cleanup;
 884                         }
 885                         DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx, set CBC mode==%d\n", cipher_ctx.cbcmode));
 886                         switch (tctx->init.alg){
 887                         case cryptocop_alg_des:
 888                                 cipher_ctx.ciph_conf = 0;
 889                                 cipher_ctx.unit_no = src_des;
 890                                 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
 891                                 break;
 892                         case cryptocop_alg_3des:
 893                                 cipher_ctx.ciph_conf = 1;
 894                                 cipher_ctx.unit_no = src_des;
 895                                 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
 896                                 break;
 897                         case cryptocop_alg_aes:
 898                                 cipher_ctx.ciph_conf = 2;
 899                                 cipher_ctx.unit_no = src_aes;
 900                                 cipher_ctx.blocklength = AES_BLOCK_LENGTH;
 901                                 break;
 902                         default:
 903                                 panic("cryptocop_setup_dma_list: impossible algorithm %d\n", tctx->init.alg);
 904                         }
 905                         (*int_op)->tdes_mode = tctx->init.tdes_mode;
 906                         break;
 907                 case cryptocop_alg_md5:
 908                 case cryptocop_alg_sha1:
 909                         /* digest */
 910                         if (digest_ctx.tcfg != NULL){
 911                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple digests in operation.\n"));
 912                                 failed = -EINVAL;
 913                                 goto error_cleanup;
 914                         }
 915                         digest_ctx.tcfg = tcfg;
 916                         digest_ctx.tctx = tctx;
 917                         digest_ctx.hash_mode = 0; /* Don't use explicit IV in this API. */
 918                         switch (tctx->init.alg){
 919                         case cryptocop_alg_md5:
 920                                 digest_ctx.blocklength = MD5_BLOCK_LENGTH;
 921                                 digest_ctx.unit_no = src_md5;
 922                                 digest_ctx.hash_conf = 1; /* 1 => MD-5 */
 923                                 break;
 924                         case cryptocop_alg_sha1:
 925                                 digest_ctx.blocklength = SHA1_BLOCK_LENGTH;
 926                                 digest_ctx.unit_no = src_sha1;
 927                                 digest_ctx.hash_conf = 0; /* 0 => SHA-1 */
 928                                 break;
 929                         default:
 930                                 panic("cryptocop_setup_dma_list: impossible digest algorithm\n");
 931                         }
 932                         break;
 933                 case cryptocop_alg_csum:
 934                         /* digest */
 935                         if (csum_ctx.tcfg != NULL){
 936                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple checksums in operation.\n"));
 937                                 failed = -EINVAL;
 938                                 goto error_cleanup;
 939                         }
 940                         (*int_op)->csum_mode = tctx->init.csum_mode;
 941                         csum_ctx.tcfg = tcfg;
 942                         csum_ctx.tctx = tctx;
 943                         break;
 944                 default:
 945                         /* no algorithm. */
 946                         DEBUG_API(printk("cryptocop_setup_dma_list: invalid algorithm %d specified in tfrm %d.\n", tctx->init.alg, tcfg->tid));
 947                         failed = -EINVAL;
 948                         goto error_cleanup;
 949                 }
 950                 tcfg = tcfg->next;
 951         }
 952         /* Download key if a cipher is used. */
 953         if (cipher_ctx.tcfg && (cipher_ctx.tctx->init.alg != cryptocop_alg_mem2mem)){
 954                 struct cryptocop_dma_desc  *key_desc = NULL;
 955
 956                 failed = setup_key_dl_desc(&cipher_ctx, &key_desc, alloc_flag);
 957                 if (failed) {
 958                         DEBUG_API(printk("cryptocop_setup_dma_list: setup key dl\n"));
 959                         goto error_cleanup;
 960                 }
 961                 current_out_cdesc->next = key_desc;
 962                 current_out_cdesc = key_desc;
 963                 indata_ix += (unsigned int)(key_desc->dma_descr->after - key_desc->dma_descr->buf);
 964
 965                 /* Download explicit IV if a cipher is used and CBC mode and explicit IV selected. */
 966                 if ((cipher_ctx.tctx->init.cipher_mode == cryptocop_cipher_mode_cbc) && (cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV)) {
 967                         struct cryptocop_dma_desc  *iv_desc = NULL;
 968
 969                         DEBUG(printk("cryptocop_setup_dma_list: setup cipher CBC IV descriptor.\n"));
 970
 971                         failed = setup_cipher_iv_desc(&cipher_ctx, &iv_desc, alloc_flag);
 972                         if (failed) {
 973                                 DEBUG_API(printk("cryptocop_setup_dma_list: CBC IV descriptor.\n"));
 974                                 goto error_cleanup;
 975                         }
 976                         current_out_cdesc->next = iv_desc;
 977                         current_out_cdesc = iv_desc;
 978                         indata_ix += (unsigned int)(iv_desc->dma_descr->after - iv_desc->dma_descr->buf);
 979                 }
 980         }
 981
 982         /* Process descriptors. */
 983         odsc = operation->tfrm_op.desc;
 984         while (odsc) {
 985                 struct cryptocop_desc_cfg   *dcfg = odsc->cfg;
 986                 struct strcop_meta_out      meta_out = {0};
 987                 size_t                      desc_len = odsc->length;
 988                 int                         active_count, eop_needed_count;
 989
 990                 output_tc = NULL;
 991
 992                 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor\n"));
 993
 994                 while (dcfg) {
 995                         struct cryptocop_tfrm_ctx  *tc = NULL;
 996
 997                         DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor configuration.\n"));
 998                         /* Get the local context for the transform and mark it as the output unit if it produces output. */
 999                         if (digest_ctx.tcfg && (digest_ctx.tcfg->tid == dcfg->tid)){
1000                                 tc = &digest_ctx;
1001                         } else if (cipher_ctx.tcfg && (cipher_ctx.tcfg->tid == dcfg->tid)){
1002                                 tc = &cipher_ctx;
1003                         } else if (csum_ctx.tcfg && (csum_ctx.tcfg->tid == dcfg->tid)){
1004                                 tc = &csum_ctx;
1005                         }
1006                         if (!tc) {
1007                                 DEBUG_API(printk("cryptocop_setup_dma_list: invalid transform %d specified in descriptor.\n", dcfg->tid));
1008                                 failed = -EINVAL;
1009                                 goto error_cleanup;
1010                         }
1011                         if (tc->done) {
1012                                 DEBUG_API(printk("cryptocop_setup_dma_list: completed transform %d reused.\n", dcfg->tid));
1013                                 failed = -EINVAL;
1014                                 goto error_cleanup;
1015                         }
1016                         if (!tc->active) {
1017                                 tc->start_ix = indata_ix;
1018                                 tc->active = 1;
1019                         }
1020
1021                         tc->previous_src = tc->current_src;
1022                         tc->prev_src = tc->curr_src;
1023                         /* Map source unit id to DMA source config. */
1024                         switch (dcfg->src){
1025                         case cryptocop_source_dma:
1026                                 tc->current_src = src_dma;
1027                                 break;
1028                         case cryptocop_source_des:
1029                                 tc->current_src = src_des;
1030                                 break;
1031                         case cryptocop_source_3des:
1032                                 tc->current_src = src_des;
1033                                 break;
1034                         case cryptocop_source_aes:
1035                                 tc->current_src = src_aes;
1036                                 break;
1037                         case cryptocop_source_md5:
1038                         case cryptocop_source_sha1:
1039                         case cryptocop_source_csum:
1040                         case cryptocop_source_none:
1041                         default:
1042                                 /* We do not allow using accumulating style units (SHA-1, MD5, checksum) as sources to other units.
1043                                  */
1044                                 DEBUG_API(printk("cryptocop_setup_dma_list: bad unit source configured %d.\n", dcfg->src));
1045                                 failed = -EINVAL;
1046                                 goto error_cleanup;
1047                         }
1048                         if (tc->current_src != src_dma) {
1049                                 /* Find the unit we are sourcing from. */
1050                                 if (digest_ctx.unit_no == tc->current_src){
1051                                         tc->curr_src = &digest_ctx;
1052                                 } else if (cipher_ctx.unit_no == tc->current_src){
1053                                         tc->curr_src = &cipher_ctx;
1054                                 } else if (csum_ctx.unit_no == tc->current_src){
1055                                         tc->curr_src = &csum_ctx;
1056                                 }
1057                                 if ((tc->curr_src == tc) && (tc->unit_no != src_dma)){
1058                                         DEBUG_API(printk("cryptocop_setup_dma_list: unit %d configured to source from itself.\n", tc->unit_no));
1059                                         failed = -EINVAL;
1060                                         goto error_cleanup;
1061                                 }
1062                         } else {
1063                                 tc->curr_src = NULL;
1064                         }
1065
1066                         /* Detect source switch. */
1067                         DEBUG(printk("cryptocop_setup_dma_list: tc->active=%d tc->unit_no=%d tc->current_src=%d tc->previous_src=%d, tc->curr_src=0x%p, tc->prev_srv=0x%p\n", tc->active, tc->unit_no, tc->current_src, tc->previous_src, tc->curr_src, tc->prev_src));
1068                         if (tc->active && (tc->current_src != tc->previous_src)) {
1069                                 /* Only allow source switch when both the old source unit and the new one have
1070                                  * no pending data to process (i.e. the consumed length must be a multiple of the
1071                                  * transform blocklength). */
1072                                 /* Note: if the src == NULL we are actually sourcing from DMA out. */
1073                                 if (((tc->prev_src != NULL) && (tc->prev_src->consumed % tc->prev_src->blocklength)) ||
1074                                     ((tc->curr_src != NULL) && (tc->curr_src->consumed % tc->curr_src->blocklength)))
1075                                 {
1076                                         DEBUG_API(printk("cryptocop_setup_dma_list: can only disconnect from or connect to a unit on a multiple of the blocklength, old: cons=%d, prod=%d, block=%d, new: cons=%d prod=%d, block=%d.\n", tc->prev_src ? tc->prev_src->consumed : INT_MIN, tc->prev_src ? tc->prev_src->produced : INT_MIN, tc->prev_src ? tc->prev_src->blocklength : INT_MIN, tc->curr_src ? tc->curr_src->consumed : INT_MIN, tc->curr_src ? tc->curr_src->produced : INT_MIN, tc->curr_src ? tc->curr_src->blocklength : INT_MIN));
1077                                         failed = -EINVAL;
1078                                         goto error_cleanup;
1079                                 }
1080                         }
1081                         /* Detect unit deactivation. */
1082                         if (dcfg->last) {
1083                                 /* Length check of this is handled below. */
1084                                 tc->done = 1;
1085                         }
1086                         dcfg = dcfg->next;
1087                 } /* while (dcfg) */
1088                 DEBUG(printk("cryptocop_setup_dma_list: parsing operation descriptor configuration complete.\n"));
1089
1090                 if (cipher_ctx.active && (cipher_ctx.curr_src != NULL) && !cipher_ctx.curr_src->active){
1091                         DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", cipher_ctx.curr_src->unit_no));
1092                         failed = -EINVAL;
1093                         goto error_cleanup;
1094                 }
1095                 if (digest_ctx.active && (digest_ctx.curr_src != NULL) && !digest_ctx.curr_src->active){
1096                         DEBUG_API(printk("cryptocop_setup_dma_list: digest source from inactive unit %d\n", digest_ctx.curr_src->unit_no));
1097                         failed = -EINVAL;
1098                         goto error_cleanup;
1099                 }
1100                 if (csum_ctx.active && (csum_ctx.curr_src != NULL) && !csum_ctx.curr_src->active){
1101                         DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", csum_ctx.curr_src->unit_no));
1102                         failed = -EINVAL;
1103                         goto error_cleanup;
1104                 }
1105
1106                 /* Update consumed and produced lengths.
1107
1108                    The consumed length accounting here is actually cheating.  If a unit source from DMA (or any
1109                    other unit that process data in blocks of one octet) it is correct, but if it source from a
1110                    block processing unit, i.e. a cipher, it will be temporarily incorrect at some times.  However
1111                    since it is only allowed--by the HW--to change source to or from a block processing unit at times where that
1112                    unit has processed an exact multiple of its block length the end result will be correct.
1113                    Beware that if the source change restriction change this code will need to be (much) reworked.
1114                 */
1115                 DEBUG(printk("cryptocop_setup_dma_list: desc->length=%d, desc_len=%d.\n", odsc->length, desc_len));
1116
1117                 if (csum_ctx.active) {
1118                         csum_ctx.consumed += desc_len;
1119                         if (csum_ctx.done) {
1120                                 csum_ctx.produced = 2;
1121                         }
1122                         DEBUG(printk("cryptocop_setup_dma_list: csum_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", csum_ctx.consumed, csum_ctx.produced, csum_ctx.blocklength));
1123                 }
1124                 if (digest_ctx.active) {
1125                         digest_ctx.consumed += desc_len;
1126                         if (digest_ctx.done) {
1127                                 if (digest_ctx.unit_no == src_md5) {
1128                                         digest_ctx.produced = MD5_STATE_LENGTH;
1129                                 } else {
1130                                         digest_ctx.produced = SHA1_STATE_LENGTH;
1131                                 }
1132                         }
1133                         DEBUG(printk("cryptocop_setup_dma_list: digest_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", digest_ctx.consumed, digest_ctx.produced, digest_ctx.blocklength));
1134                 }
1135                 if (cipher_ctx.active) {
1136                         /* Ciphers are allowed only to source from DMA out.  That is filtered above. */
1137                         assert(cipher_ctx.current_src == src_dma);
1138                         cipher_ctx.consumed += desc_len;
1139                         cipher_ctx.produced = cipher_ctx.blocklength * (cipher_ctx.consumed / cipher_ctx.blocklength);
1140                         if (cipher_ctx.cbcmode && !(cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV) && cipher_ctx.produced){
1141                                 cipher_ctx.produced -= cipher_ctx.blocklength; /* Compensate for CBC iv. */
1142                         }
1143                         DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", cipher_ctx.consumed, cipher_ctx.produced, cipher_ctx.blocklength));
1144                 }
1145
1146                 /* Setup the DMA out descriptors. */
1147                 /* Configure the metadata. */
1148                 active_count = 0;
1149                 eop_needed_count = 0;
1150                 if (cipher_ctx.active) {
1151                         ++active_count;
1152                         if (cipher_ctx.unit_no == src_dma){
1153                                 /* mem2mem */
1154                                 meta_out.ciphsel = src_none;
1155                         } else {
1156                                 meta_out.ciphsel = cipher_ctx.current_src;
1157                         }
1158                         meta_out.ciphconf = cipher_ctx.ciph_conf;
1159                         meta_out.cbcmode = cipher_ctx.cbcmode;
1160                         meta_out.decrypt = cipher_ctx.decrypt;
1161                         DEBUG(printk("set ciphsel=%d ciphconf=%d cbcmode=%d decrypt=%d\n", meta_out.ciphsel, meta_out.ciphconf, meta_out.cbcmode, meta_out.decrypt));
1162                         if (cipher_ctx.done) ++eop_needed_count;
1163                 } else {
1164                         meta_out.ciphsel = src_none;
1165                 }
1166
1167                 if (digest_ctx.active) {
1168                         ++active_count;
1169                         meta_out.hashsel = digest_ctx.current_src;
1170                         meta_out.hashconf = digest_ctx.hash_conf;
1171                         meta_out.hashmode = 0; /* Explicit mode is not used here. */
1172                         DEBUG(printk("set hashsel=%d hashconf=%d hashmode=%d\n", meta_out.hashsel, meta_out.hashconf, meta_out.hashmode));
1173                         if (digest_ctx.done) {
1174                                 assert(digest_ctx.pad_descs == NULL);
1175                                 failed = create_pad_descriptor(&digest_ctx, &digest_ctx.pad_descs, alloc_flag);
1176                                 if (failed) {
1177                                         DEBUG_API(printk("cryptocop_setup_dma_list: failed digest pad creation.\n"));
1178                                         goto error_cleanup;
1179                                 }
1180                         }
1181                 } else {
1182                         meta_out.hashsel = src_none;
1183                 }
1184
1185                 if (csum_ctx.active) {
1186                         ++active_count;
1187                         meta_out.csumsel = csum_ctx.current_src;
1188                         if (csum_ctx.done) {
1189                                 assert(csum_ctx.pad_descs == NULL);
1190                                 failed = create_pad_descriptor(&csum_ctx, &csum_ctx.pad_descs, alloc_flag);
1191                                 if (failed) {
1192                                         DEBUG_API(printk("cryptocop_setup_dma_list: failed csum pad creation.\n"));
1193                                         goto error_cleanup;
1194                                 }
1195                         }
1196                 } else {
1197                         meta_out.csumsel = src_none;
1198                 }
1199                 DEBUG(printk("cryptocop_setup_dma_list: %d eop needed, %d active units\n", eop_needed_count, active_count));
1200                 /* Setup DMA out descriptors for the indata. */
1201                 failed = create_output_descriptors(operation, &iniov_ix, &iniov_offset, desc_len, &current_out_cdesc, &meta_out, alloc_flag);
1202                 if (failed) {
1203                         DEBUG_API(printk("cryptocop_setup_dma_list: create_output_descriptors %d\n", failed));
1204                         goto error_cleanup;
1205                 }
1206                 /* Setup out EOP.  If there are active units that are not done here they cannot get an EOP
1207                  * so we ust setup a zero length descriptor to DMA to signal EOP only to done units.
1208                  * If there is a pad descriptor EOP for the padded unit will be EOPed by it.
1209                  */
1210                 assert(active_count >= eop_needed_count);
1211                 assert((eop_needed_count == 0) || (eop_needed_count == 1));
1212                 if (eop_needed_count) {
1213                         /* This means that the bulk operation (cipeher/m2m) is terminated. */
1214                         if (active_count > 1) {
1215                                 /* Use zero length EOP descriptor. */
1216                                 struct cryptocop_dma_desc *ed = alloc_cdesc(alloc_flag);
1217                                 struct strcop_meta_out    ed_mo = {0};
1218                                 if (!ed) {
1219                                         DEBUG_API(printk("cryptocop_setup_dma_list: alloc EOP descriptor for cipher\n"));
1220                                         failed = -ENOMEM;
1221                                         goto error_cleanup;
1222                                 }
1223
1224                                 assert(cipher_ctx.active && cipher_ctx.done);
1225
1226                                 if (cipher_ctx.unit_no == src_dma){
1227                                         /* mem2mem */
1228                                         ed_mo.ciphsel = src_none;
1229                                 } else {
1230                                         ed_mo.ciphsel = cipher_ctx.current_src;
1231                                 }
1232                                 ed_mo.ciphconf = cipher_ctx.ciph_conf;
1233                                 ed_mo.cbcmode = cipher_ctx.cbcmode;
1234                                 ed_mo.decrypt = cipher_ctx.decrypt;
1235
1236                                 ed->free_buf = NULL;
1237                                 ed->dma_descr->wait = 1;
1238                                 ed->dma_descr->out_eop = 1;
1239
1240                                 ed->dma_descr->buf = (char*)virt_to_phys(&ed); /* Use any valid physical address for zero length descriptor. */
1241                                 ed->dma_descr->after = ed->dma_descr->buf;
1242                                 ed->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, ed_mo);
1243                                 current_out_cdesc->next = ed;
1244                                 current_out_cdesc = ed;
1245                         } else {
1246                                 /* Set EOP in the current out descriptor since the only active module is
1247                                  * the one needing the EOP. */
1248
1249                                 current_out_cdesc->dma_descr->out_eop = 1;
1250                         }
1251                 }
1252
1253                 if (cipher_ctx.done && cipher_ctx.active) cipher_ctx.active = 0;
1254                 if (digest_ctx.done && digest_ctx.active) digest_ctx.active = 0;
1255                 if (csum_ctx.done && csum_ctx.active) csum_ctx.active = 0;
1256                 indata_ix += odsc->length;
1257                 odsc = odsc->next;
1258         } /* while (odsc) */ /* Process descriptors. */
1259         DEBUG(printk("cryptocop_setup_dma_list: done parsing operation descriptors\n"));
1260         if (cipher_ctx.tcfg && (cipher_ctx.active || !cipher_ctx.done)){
1261                 DEBUG_API(printk("cryptocop_setup_dma_list: cipher operation not terminated.\n"));
1262                 failed = -EINVAL;
1263                 goto error_cleanup;
1264         }
1265         if (digest_ctx.tcfg && (digest_ctx.active || !digest_ctx.done)){
1266                 DEBUG_API(printk("cryptocop_setup_dma_list: digest operation not terminated.\n"));
1267                 failed = -EINVAL;
1268                 goto error_cleanup;
1269         }
1270         if (csum_ctx.tcfg && (csum_ctx.active || !csum_ctx.done)){
1271                 DEBUG_API(printk("cryptocop_setup_dma_list: csum operation not terminated.\n"));
1272                 failed = -EINVAL;
1273                 goto error_cleanup;
1274         }
1275
1276         failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &cipher_ctx, alloc_flag);
1277         if (failed){
1278                 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1279                 goto error_cleanup;
1280         }
1281         failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &digest_ctx, alloc_flag);
1282         if (failed){
1283                 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1284                 goto error_cleanup;
1285         }
1286         failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &csum_ctx, alloc_flag);
1287         if (failed){
1288                 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1289                 goto error_cleanup;
1290         }
1291
1292         DEBUG(printk("cryptocop_setup_dma_list: int_op=0x%p, *int_op=0x%p\n", int_op, *int_op));
1293         (*int_op)->cdesc_out = out_cdesc_head.next;
1294         (*int_op)->cdesc_in = in_cdesc_head.next;
1295         DEBUG(printk("cryptocop_setup_dma_list: out_cdesc_head=0x%p in_cdesc_head=0x%p\n", (*int_op)->cdesc_out, (*int_op)->cdesc_in));
1296
1297         setup_descr_chain(out_cdesc_head.next);
1298         setup_descr_chain(in_cdesc_head.next);
1299
1300         /* Last but not least: mark the last DMA in descriptor for a INTR and EOL and the the
1301          * last DMA out descriptor for EOL.
1302          */
1303         current_in_cdesc->dma_descr->intr = 1;
1304         current_in_cdesc->dma_descr->eol = 1;
1305         current_out_cdesc->dma_descr->eol = 1;
1306
1307         /* Setup DMA contexts. */
1308         (*int_op)->ctx_out.next = NULL;
1309         (*int_op)->ctx_out.eol = 1;
1310         (*int_op)->ctx_out.intr = 0;
1311         (*int_op)->ctx_out.store_mode = 0;
1312         (*int_op)->ctx_out.en = 0;
1313         (*int_op)->ctx_out.dis = 0;
1314         (*int_op)->ctx_out.md0 = 0;
1315         (*int_op)->ctx_out.md1 = 0;
1316         (*int_op)->ctx_out.md2 = 0;
1317         (*int_op)->ctx_out.md3 = 0;
1318         (*int_op)->ctx_out.md4 = 0;
1319         (*int_op)->ctx_out.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_out->dma_descr);
1320         (*int_op)->ctx_out.saved_data_buf = (*int_op)->cdesc_out->dma_descr->buf; /* Already physical address. */
1321
1322         (*int_op)->ctx_in.next = NULL;
1323         (*int_op)->ctx_in.eol = 1;
1324         (*int_op)->ctx_in.intr = 0;
1325         (*int_op)->ctx_in.store_mode = 0;
1326         (*int_op)->ctx_in.en = 0;
1327         (*int_op)->ctx_in.dis = 0;
1328         (*int_op)->ctx_in.md0 = 0;
1329         (*int_op)->ctx_in.md1 = 0;
1330         (*int_op)->ctx_in.md2 = 0;
1331         (*int_op)->ctx_in.md3 = 0;
1332         (*int_op)->ctx_in.md4 = 0;
1333
1334         (*int_op)->ctx_in.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_in->dma_descr);
1335         (*int_op)->ctx_in.saved_data_buf = (*int_op)->cdesc_in->dma_descr->buf; /* Already physical address. */
1336
1337         DEBUG(printk("cryptocop_setup_dma_list: done\n"));
1338         return 0;
1339
1340 error_cleanup:
1341         {
1342                 /* Free all allocated resources. */
1343                 struct cryptocop_dma_desc *tmp_cdesc;
1344                 while (digest_ctx.pad_descs){
1345                         tmp_cdesc = digest_ctx.pad_descs->next;
1346                         free_cdesc(digest_ctx.pad_descs);
1347                         digest_ctx.pad_descs = tmp_cdesc;
1348                 }
1349                 while (csum_ctx.pad_descs){
1350                         tmp_cdesc = csum_ctx.pad_descs->next;
1351                         free_cdesc(csum_ctx.pad_descs);
1352                         csum_ctx.pad_descs = tmp_cdesc;
1353                 }
1354                 assert(cipher_ctx.pad_descs == NULL); /* The ciphers are never padded. */
1355
1356                 if (*int_op != NULL) delete_internal_operation(*int_op);
1357         }
1358         DEBUG_API(printk("cryptocop_setup_dma_list: done with error %d\n", failed));
1359         return failed;
1360 }
1361
1362
1363 static void delete_internal_operation(struct cryptocop_int_operation *iop)
1364 {
1365         void                      *ptr = iop->alloc_ptr;
1366         struct cryptocop_dma_desc *cd = iop->cdesc_out;
1367         struct cryptocop_dma_desc *next;
1368
1369         DEBUG(printk("delete_internal_operation: iop=0x%p, alloc_ptr=0x%p\n", iop, ptr));
1370
1371         while (cd) {
1372                 next = cd->next;
1373                 free_cdesc(cd);
1374                 cd = next;
1375         }
1376         cd = iop->cdesc_in;
1377         while (cd) {
1378                 next = cd->next;
1379                 free_cdesc(cd);
1380                 cd = next;
1381         }
1382         kfree(ptr);
1383 }
1384
1385 #define MD5_MIN_PAD_LENGTH (9)
1386 #define MD5_PAD_LENGTH_FIELD_LENGTH (8)
1387
1388 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1389 {
1390         size_t                  padlen = MD5_BLOCK_LENGTH - (hashed_length % MD5_BLOCK_LENGTH);
1391         unsigned char           *p;
1392         int                     i;
1393         unsigned long long int  bit_length = hashed_length << 3;
1394
1395         if (padlen < MD5_MIN_PAD_LENGTH) padlen += MD5_BLOCK_LENGTH;
1396
1397         p = kmalloc(padlen, alloc_flag);
1398         if (!p) return -ENOMEM;
1399
1400         *p = 0x80;
1401         memset(p+1, 0, padlen - 1);
1402
1403         DEBUG(printk("create_md5_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1404
1405         i = padlen - MD5_PAD_LENGTH_FIELD_LENGTH;
1406         while (bit_length != 0){
1407                 p[i++] = bit_length % 0x100;
1408                 bit_length >>= 8;
1409         }
1410
1411         *pad = (char*)p;
1412         *pad_length = padlen;
1413
1414         return 0;
1415 }
1416
1417 #define SHA1_MIN_PAD_LENGTH (9)
1418 #define SHA1_PAD_LENGTH_FIELD_LENGTH (8)
1419
1420 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1421 {
1422         size_t                  padlen = SHA1_BLOCK_LENGTH - (hashed_length % SHA1_BLOCK_LENGTH);
1423         unsigned char           *p;
1424         int                     i;
1425         unsigned long long int  bit_length = hashed_length << 3;
1426
1427         if (padlen < SHA1_MIN_PAD_LENGTH) padlen += SHA1_BLOCK_LENGTH;
1428
1429         p = kmalloc(padlen, alloc_flag);
1430         if (!p) return -ENOMEM;
1431
1432         *p = 0x80;
1433         memset(p+1, 0, padlen - 1);
1434
1435         DEBUG(printk("create_sha1_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1436
1437         i = padlen - 1;
1438         while (bit_length != 0){
1439                 p[i--] = bit_length % 0x100;
1440                 bit_length >>= 8;
1441         }
1442
1443         *pad = (char*)p;
1444         *pad_length = padlen;
1445
1446         return 0;
1447 }
1448
1449
1450 static int transform_ok(struct cryptocop_transform_init *tinit)
1451 {
1452         switch (tinit->alg){
1453         case cryptocop_alg_csum:
1454                 switch (tinit->csum_mode){
1455                 case cryptocop_csum_le:
1456                 case cryptocop_csum_be:
1457                         break;
1458                 default:
1459                         DEBUG_API(printk("transform_ok: Bad mode set for csum transform\n"));
1460                         return -EINVAL;
1461                 }
1462         case cryptocop_alg_mem2mem:
1463         case cryptocop_alg_md5:
1464         case cryptocop_alg_sha1:
1465                 if (tinit->keylen != 0) {
1466                         DEBUG_API(printk("transform_ok: non-zero keylength, %d, for a digest/csum algorithm\n", tinit->keylen));
1467                         return -EINVAL; /* This check is a bit strict. */
1468                 }
1469                 break;
1470         case cryptocop_alg_des:
1471                 if (tinit->keylen != 64) {
1472                         DEBUG_API(printk("transform_ok: keylen %d invalid for DES\n", tinit->keylen));
1473                         return -EINVAL;
1474                 }
1475                 break;
1476         case cryptocop_alg_3des:
1477                 if (tinit->keylen != 192) {
1478                         DEBUG_API(printk("transform_ok: keylen %d invalid for 3DES\n", tinit->keylen));
1479                         return -EINVAL;
1480                 }
1481                 break;
1482         case cryptocop_alg_aes:
1483                 if (tinit->keylen != 128 && tinit->keylen != 192 && tinit->keylen != 256) {
1484                         DEBUG_API(printk("transform_ok: keylen %d invalid for AES\n", tinit->keylen));
1485                         return -EINVAL;
1486                 }
1487                 break;
1488         case cryptocop_no_alg:
1489         default:
1490                 DEBUG_API(printk("transform_ok: no such algorithm %d\n", tinit->alg));
1491                 return -EINVAL;
1492         }
1493
1494         switch (tinit->alg){
1495         case cryptocop_alg_des:
1496         case cryptocop_alg_3des:
1497         case cryptocop_alg_aes:
1498                 if (tinit->cipher_mode != cryptocop_cipher_mode_ecb && tinit->cipher_mode != cryptocop_cipher_mode_cbc) return -EINVAL;
1499         default:
1500                  break;
1501         }
1502         return 0;
1503 }
1504
1505
1506 int cryptocop_new_session(cryptocop_session_id *sid, struct cryptocop_transform_init *tinit, int alloc_flag)
1507 {
1508         struct cryptocop_session         *sess;
1509         struct cryptocop_transform_init  *tfrm_in = tinit;
1510         struct cryptocop_transform_init  *tmp_in;
1511         int                              no_tfrms = 0;
1512         int                              i;
1513         unsigned long int                flags;
1514
1515         init_stream_coprocessor(); /* For safety if we are called early */
1516
1517         while (tfrm_in){
1518                 int err;
1519                 ++no_tfrms;
1520                 if ((err = transform_ok(tfrm_in))) {
1521                         DEBUG_API(printk("cryptocop_new_session, bad transform\n"));
1522                         return err;
1523                 }
1524                 tfrm_in = tfrm_in->next;
1525         }
1526         if (0 == no_tfrms) {
1527                 DEBUG_API(printk("cryptocop_new_session, no transforms specified\n"));
1528                 return -EINVAL;
1529         }
1530
1531         sess = kmalloc(sizeof(struct cryptocop_session), alloc_flag);
1532         if (!sess){
1533                 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_session\n"));
1534                 return -ENOMEM;
1535         }
1536
1537         sess->tfrm_ctx = kmalloc(no_tfrms * sizeof(struct cryptocop_transform_ctx), alloc_flag);
1538         if (!sess->tfrm_ctx) {
1539                 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_transform_ctx\n"));
1540                 kfree(sess);
1541                 return -ENOMEM;
1542         }
1543
1544         tfrm_in = tinit;
1545         for (i = 0; i < no_tfrms; i++){
1546                 tmp_in = tfrm_in->next;
1547                 while (tmp_in){
1548                         if (tmp_in->tid == tfrm_in->tid) {
1549                                 DEBUG_API(printk("cryptocop_new_session, duplicate transform ids\n"));
1550                                 kfree(sess->tfrm_ctx);
1551                                 kfree(sess);
1552                                 return -EINVAL;
1553                         }
1554                         tmp_in = tmp_in->next;
1555                 }
1556                 memcpy(&sess->tfrm_ctx[i].init, tfrm_in, sizeof(struct cryptocop_transform_init));
1557                 sess->tfrm_ctx[i].dec_key_set = 0;
1558                 sess->tfrm_ctx[i].next = &sess->tfrm_ctx[i] + 1;
1559
1560                 tfrm_in = tfrm_in->next;
1561         }
1562         sess->tfrm_ctx[i-1].next = NULL;
1563
1564         spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1565         sess->sid = next_sid;
1566         next_sid++;
1567         /* TODO If we are really paranoid we should do duplicate check to handle sid wraparound.
1568          *      OTOH 2^64 is a really large number of session. */
1569         if (next_sid == 0) next_sid = 1;
1570
1571         /* Prepend to session list. */
1572         sess->next = cryptocop_sessions;
1573         cryptocop_sessions = sess;
1574         spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1575         *sid = sess->sid;
1576         return 0;
1577 }
1578
1579
1580 int cryptocop_free_session(cryptocop_session_id sid)
1581 {
1582         struct cryptocop_transform_ctx    *tc;
1583         struct cryptocop_session          *sess = NULL;
1584         struct cryptocop_session          *psess = NULL;
1585         unsigned long int                 flags;
1586         int                               i;
1587         LIST_HEAD(remove_list);
1588         struct list_head                  *node, *tmp;
1589         struct cryptocop_prio_job         *pj;
1590
1591         DEBUG(printk("cryptocop_free_session: sid=%lld\n", sid));
1592
1593         spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1594         sess = cryptocop_sessions;
1595         while (sess && sess->sid != sid){
1596                 psess = sess;
1597                 sess = sess->next;
1598         }
1599         if (sess){
1600                 if (psess){
1601                         psess->next = sess->next;
1602                 } else {
1603                         cryptocop_sessions = sess->next;
1604                 }
1605         }
1606         spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1607
1608         if (!sess) return -EINVAL;
1609
1610         /* Remove queued jobs. */
1611         spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1612
1613         for (i = 0; i < cryptocop_prio_no_prios; i++){
1614                 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
1615                         list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
1616                                 pj = list_entry(node, struct cryptocop_prio_job, node);
1617                                 if (pj->oper->sid == sid) {
1618                                         list_move_tail(node, &remove_list);
1619                                 }
1620                         }
1621                 }
1622         }
1623         spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1624
1625         list_for_each_safe(node, tmp, &remove_list) {
1626                 list_del(node);
1627                 pj = list_entry(node, struct cryptocop_prio_job, node);
1628                 pj->oper->operation_status = -EAGAIN;  /* EAGAIN is not ideal for job/session terminated but it's the best choice I know of. */
1629                 DEBUG(printk("cryptocop_free_session: pj=0x%p, pj->oper=0x%p, pj->iop=0x%p\n", pj, pj->oper, pj->iop));
1630                 pj->oper->cb(pj->oper, pj->oper->cb_data);
1631                 delete_internal_operation(pj->iop);
1632                 kfree(pj);
1633         }
1634
1635         tc = sess->tfrm_ctx;
1636         /* Erase keying data. */
1637         while (tc){
1638                 DEBUG(printk("cryptocop_free_session: memset keys, tfrm id=%d\n", tc->init.tid));
1639                 memset(tc->init.key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1640                 memset(tc->dec_key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1641                 tc = tc->next;
1642         }
1643         kfree(sess->tfrm_ctx);
1644         kfree(sess);
1645
1646         return 0;
1647 }
1648
1649 static struct cryptocop_session *get_session(cryptocop_session_id sid)
1650 {
1651         struct cryptocop_session    *sess;
1652         unsigned long int           flags;
1653
1654         spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1655         sess = cryptocop_sessions;
1656         while (sess && (sess->sid != sid)){
1657                 sess = sess->next;
1658         }
1659         spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1660
1661         return sess;
1662 }
1663
1664 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid)
1665 {
1666         struct cryptocop_transform_ctx *tc = sess->tfrm_ctx;
1667
1668         DEBUG(printk("get_transform_ctx, sess=0x%p, tid=%d\n", sess, tid));
1669         assert(sess != NULL);
1670         while (tc && tc->init.tid != tid){
1671                 DEBUG(printk("tc=0x%p, tc->next=0x%p\n", tc, tc->next));
1672                 tc = tc->next;
1673         }
1674         DEBUG(printk("get_transform_ctx, returning tc=0x%p\n", tc));
1675         return tc;
1676 }
1677
1678
1679
1680 /* The AES s-transform matrix (s-box). */
1681 static const u8 aes_sbox[256] = {
1682         99,  124, 119, 123, 242, 107, 111, 197, 48,  1,   103, 43,  254, 215, 171, 118,
1683         202, 130, 201, 125, 250, 89,  71,  240, 173, 212, 162, 175, 156, 164, 114, 192,
1684         183, 253, 147, 38,  54,  63,  247, 204, 52,  165, 229, 241, 113, 216, 49,  21,
1685         4,   199, 35,  195, 24,  150, 5,   154, 7,   18,  128, 226, 235, 39,  178, 117,
1686         9,   131, 44,  26,  27,  110, 90,  160, 82,  59,  214, 179, 41,  227, 47,  132,
1687         83,  209, 0,   237, 32,  252, 177, 91,  106, 203, 190, 57,  74,  76,  88,  207,
1688         208, 239, 170, 251, 67,  77,  51,  133, 69,  249, 2,   127, 80,  60,  159, 168,
1689         81,  163, 64,  143, 146, 157, 56,  245, 188, 182, 218, 33,  16,  255, 243, 210,
1690         205, 12,  19,  236, 95,  151, 68,  23,  196, 167, 126, 61,  100, 93,  25,  115,
1691         96,  129, 79,  220, 34,  42,  144, 136, 70,  238, 184, 20,  222, 94,  11,  219,
1692         224, 50,  58,  10,  73,  6,   36,  92,  194, 211, 172, 98,  145, 149, 228, 121,
1693         231, 200, 55,  109, 141, 213, 78,  169, 108, 86,  244, 234, 101, 122, 174, 8,
1694         186, 120, 37,  46,  28,  166, 180, 198, 232, 221, 116, 31,  75,  189, 139, 138,
1695         112, 62,  181, 102, 72,  3,   246, 14,  97,  53,  87,  185, 134, 193, 29,  158,
1696         225, 248, 152, 17,  105, 217, 142, 148, 155, 30,  135, 233, 206, 85,  40,  223,
1697         140, 161, 137, 13,  191, 230, 66,  104, 65,  153, 45,  15,  176, 84,  187, 22
1698 };
1699
1700 /* AES has a 32 bit word round constants for each round in the
1701  * key schedule.  round_constant[i] is really Rcon[i+1] in FIPS187.
1702  */
1703 static u32 round_constant[11] = {
1704         0x01000000, 0x02000000, 0x04000000, 0x08000000,
1705         0x10000000, 0x20000000, 0x40000000, 0x80000000,
1706         0x1B000000, 0x36000000, 0x6C000000
1707 };
1708
1709 /* Apply the s-box to each of the four occtets in w. */
1710 static u32 aes_ks_subword(const u32 w)
1711 {
1712         u8 bytes[4];
1713
1714         *(u32*)(&bytes[0]) = w;
1715         bytes[0] = aes_sbox[bytes[0]];
1716         bytes[1] = aes_sbox[bytes[1]];
1717         bytes[2] = aes_sbox[bytes[2]];
1718         bytes[3] = aes_sbox[bytes[3]];
1719         return *(u32*)(&bytes[0]);
1720 }
1721
1722 /* The encrypt (forward) Rijndael key schedule algorithm pseudo code:
1723  * (Note that AES words are 32 bit long)
1724  *
1725  * KeyExpansion(byte key[4*Nk], word w[Nb*(Nr+1)], Nk){
1726  * word temp
1727  * i = 0
1728  * while (i < Nk) {
1729  *   w[i] = word(key[4*i, 4*i + 1, 4*i + 2, 4*i + 3])
1730  *   i = i + 1
1731  * }
1732  * i = Nk
1733  *
1734  * while (i < (Nb * (Nr + 1))) {
1735  *   temp = w[i - 1]
1736  *   if ((i mod Nk) == 0) {
1737  *     temp = SubWord(RotWord(temp)) xor Rcon[i/Nk]
1738  *   }
1739  *   else if ((Nk > 6) && ((i mod Nk) == 4)) {
1740  *     temp = SubWord(temp)
1741  *   }
1742  *   w[i] = w[i - Nk] xor temp
1743  * }
1744  * RotWord(t) does a 8 bit cyclic shift left on a 32 bit word.
1745  * SubWord(t) applies the AES s-box individually to each octet
1746  * in a 32 bit word.
1747  *
1748  * For AES Nk can have the values 4, 6, and 8 (corresponding to
1749  * values for Nr of 10, 12, and 14).  Nb is always 4.
1750  *
1751  * To construct w[i], w[i - 1] and w[i - Nk] must be
1752  * available.  Consequently we must keep a state of the last Nk words
1753  * to be able to create the last round keys.
1754  */
1755 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned  char *key, unsigned int keylength)
1756 {
1757         u32 temp;
1758         u32 w_ring[8]; /* nk is max 8, use elements 0..(nk - 1) as a ringbuffer */
1759         u8  w_last_ix;
1760         int i;
1761         u8  nr, nk;
1762
1763         switch (keylength){
1764         case 128:
1765                 nk = 4;
1766                 nr = 10;
1767                 break;
1768         case 192:
1769                 nk = 6;
1770                 nr = 12;
1771                 break;
1772         case 256:
1773                 nk = 8;
1774                 nr = 14;
1775                 break;
1776         default:
1777                 panic("stream co-processor: bad aes key length in get_aes_decrypt_key\n");
1778         };
1779
1780         /* Need to do host byte order correction here since key is byte oriented and the
1781          * kx algorithm is word (u32) oriented. */
1782         for (i = 0; i < nk; i+=1) {
1783                 w_ring[i] = be32_to_cpu(*(u32*)&key[4*i]);
1784         }
1785
1786         i = (int)nk;
1787         w_last_ix = i - 1;
1788         while (i < (4 * (nr + 2))) {
1789                 temp = w_ring[w_last_ix];
1790                 if (!(i % nk)) {
1791                         /* RotWord(temp) */
1792                         temp = (temp << 8) | (temp >> 24);
1793                         temp = aes_ks_subword(temp);
1794                         temp ^= round_constant[i/nk - 1];
1795                 } else if ((nk > 6) && ((i % nk) == 4)) {
1796                         temp = aes_ks_subword(temp);
1797                 }
1798                 w_last_ix = (w_last_ix + 1) % nk; /* This is the same as (i-Nk) mod Nk */
1799                 temp ^= w_ring[w_last_ix];
1800                 w_ring[w_last_ix] = temp;
1801
1802                 /* We need the round keys for round Nr+1 and Nr+2 (round key
1803                  * Nr+2 is the round key beyond the last one used when
1804                  * encrypting).  Rounds are numbered starting from 0, Nr=10
1805                  * implies 11 rounds are used in encryption/decryption.
1806                  */
1807                 if (i >= (4 * nr)) {
1808                         /* Need to do host byte order correction here, the key
1809                          * is byte oriented. */
1810                         *(u32*)dec_key = cpu_to_be32(temp);
1811                         dec_key += 4;
1812                 }
1813                 ++i;
1814         }
1815 }
1816
1817
1818 /**** Job/operation management. ****/
1819
1820 int cryptocop_job_queue_insert_csum(struct cryptocop_operation *operation)
1821 {
1822         return cryptocop_job_queue_insert(cryptocop_prio_kernel_csum, operation);
1823 }
1824
1825 int cryptocop_job_queue_insert_crypto(struct cryptocop_operation *operation)
1826 {
1827         return cryptocop_job_queue_insert(cryptocop_prio_kernel, operation);
1828 }
1829
1830 int cryptocop_job_queue_insert_user_job(struct cryptocop_operation *operation)
1831 {
1832         return cryptocop_job_queue_insert(cryptocop_prio_user, operation);
1833 }
1834
1835 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation)
1836 {
1837         int                           ret;
1838         struct cryptocop_prio_job     *pj = NULL;
1839         unsigned long int             flags;
1840
1841         DEBUG(printk("cryptocop_job_queue_insert(%d, 0x%p)\n", prio, operation));
1842
1843         if (!operation || !operation->cb){
1844                 DEBUG_API(printk("cryptocop_job_queue_insert oper=0x%p, NULL operation or callback\n", operation));
1845                 return -EINVAL;
1846         }
1847
1848         if ((ret = cryptocop_job_setup(&pj, operation)) != 0){
1849                 DEBUG_API(printk("cryptocop_job_queue_insert: job setup failed\n"));
1850                 return ret;
1851         }
1852         assert(pj != NULL);
1853
1854         spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1855         list_add_tail(&pj->node, &cryptocop_job_queues[prio].jobs);
1856         spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1857
1858         /* Make sure a job is running */
1859         cryptocop_start_job();
1860         return 0;
1861 }
1862
1863 static void cryptocop_do_tasklet(unsigned long unused);
1864 DECLARE_TASKLET (cryptocop_tasklet, cryptocop_do_tasklet, 0);
1865
1866 static void cryptocop_do_tasklet(unsigned long unused)
1867 {
1868         struct list_head             *node;
1869         struct cryptocop_prio_job    *pj = NULL;
1870         unsigned long                flags;
1871
1872         DEBUG(printk("cryptocop_do_tasklet: entering\n"));
1873
1874         do {
1875                 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
1876                 if (!list_empty(&cryptocop_completed_jobs)){
1877                         node = cryptocop_completed_jobs.next;
1878                         list_del(node);
1879                         pj = list_entry(node, struct cryptocop_prio_job, node);
1880                 } else {
1881                         pj = NULL;
1882                 }
1883                 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
1884                 if (pj) {
1885                         assert(pj->oper != NULL);
1886
1887                         /* Notify consumer of operation completeness. */
1888                         DEBUG(printk("cryptocop_do_tasklet: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
1889
1890                         pj->oper->operation_status = 0; /* Job is completed. */
1891                         pj->oper->cb(pj->oper, pj->oper->cb_data);
1892                         delete_internal_operation(pj->iop);
1893                         kfree(pj);
1894                 }
1895         } while (pj != NULL);
1896
1897         DEBUG(printk("cryptocop_do_tasklet: exiting\n"));
1898 }
1899
1900 static irqreturn_t
1901 dma_done_interrupt(int irq, void *dev_id)
1902 {
1903         struct cryptocop_prio_job *done_job;
1904         reg_dma_rw_ack_intr ack_intr = {
1905                 .data = 1,
1906         };
1907
1908         REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1909
1910         DEBUG(printk("cryptocop DMA done\n"));
1911
1912         spin_lock(&running_job_lock);
1913         if (cryptocop_running_job == NULL){
1914                 printk("stream co-processor got interrupt when not busy\n");
1915                 spin_unlock(&running_job_lock);
1916                 return IRQ_HANDLED;
1917         }
1918         done_job = cryptocop_running_job;
1919         cryptocop_running_job = NULL;
1920         spin_unlock(&running_job_lock);
1921
1922         /* Start processing a job. */
1923         if (!spin_trylock(&cryptocop_process_lock)){
1924                 DEBUG(printk("cryptocop irq handler, not starting a job\n"));
1925         } else {
1926                 cryptocop_start_job();
1927                 spin_unlock(&cryptocop_process_lock);
1928         }
1929
1930         done_job->oper->operation_status = 0; /* Job is completed. */
1931         if (done_job->oper->fast_callback){
1932                 /* This operation wants callback from interrupt. */
1933                 done_job->oper->cb(done_job->oper, done_job->oper->cb_data);
1934                 delete_internal_operation(done_job->iop);
1935                 kfree(done_job);
1936         } else {
1937                 spin_lock(&cryptocop_completed_jobs_lock);
1938                 list_add_tail(&(done_job->node), &cryptocop_completed_jobs);
1939                 spin_unlock(&cryptocop_completed_jobs_lock);
1940                 tasklet_schedule(&cryptocop_tasklet);
1941         }
1942
1943         DEBUG(printk("cryptocop leave irq handler\n"));
1944         return IRQ_HANDLED;
1945 }
1946
1947
1948 /* Setup interrupts and DMA channels. */
1949 static int init_cryptocop(void)
1950 {
1951         unsigned long          flags;
1952         reg_dma_rw_cfg         dma_cfg = {.en = 1};
1953         reg_dma_rw_intr_mask   intr_mask_in = {.data = regk_dma_yes}; /* Only want descriptor interrupts from the DMA in channel. */
1954         reg_dma_rw_ack_intr    ack_intr = {.data = 1,.in_eop = 1 };
1955         reg_strcop_rw_cfg      strcop_cfg = {
1956                 .ipend = regk_strcop_little,
1957                 .td1 = regk_strcop_e,
1958                 .td2 = regk_strcop_d,
1959                 .td3 = regk_strcop_e,
1960                 .ignore_sync = 0,
1961                 .en = 1
1962         };
1963
1964         if (request_irq(DMA_IRQ, dma_done_interrupt, 0,
1965                         "stream co-processor DMA", NULL))
1966                 panic("request_irq stream co-processor irq dma9");
1967
1968         (void)crisv32_request_dma(OUT_DMA, "strcop", DMA_PANIC_ON_ERROR,
1969                 0, dma_strp);
1970         (void)crisv32_request_dma(IN_DMA, "strcop", DMA_PANIC_ON_ERROR,
1971                 0, dma_strp);
1972
1973         local_irq_save(flags);
1974
1975         /* Reset and enable the cryptocop. */
1976         strcop_cfg.en = 0;
1977         REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1978         strcop_cfg.en = 1;
1979         REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1980
1981         /* Enable DMAs. */
1982         REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
1983         REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
1984
1985         /* Set up wordsize = 4 for DMAs. */
1986         DMA_WR_CMD(OUT_DMA_INST, regk_dma_set_w_size4);
1987         DMA_WR_CMD(IN_DMA_INST, regk_dma_set_w_size4);
1988
1989         /* Enable interrupts. */
1990         REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
1991
1992         /* Clear intr ack. */
1993         REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1994
1995         local_irq_restore(flags);
1996
1997         return 0;
1998 }
1999
2000 /* Free used cryptocop hw resources (interrupt and DMA channels). */
2001 static void release_cryptocop(void)
2002 {
2003         unsigned long          flags;
2004         reg_dma_rw_cfg         dma_cfg = {.en = 0};
2005         reg_dma_rw_intr_mask   intr_mask_in = {0};
2006         reg_dma_rw_ack_intr    ack_intr = {.data = 1,.in_eop = 1 };
2007
2008         local_irq_save(flags);
2009
2010         /* Clear intr ack. */
2011         REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
2012
2013         /* Disable DMAs. */
2014         REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
2015         REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
2016
2017         /* Disable interrupts. */
2018         REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
2019
2020         local_irq_restore(flags);
2021
2022         free_irq(DMA_IRQ, NULL);
2023
2024         (void)crisv32_free_dma(OUT_DMA);
2025         (void)crisv32_free_dma(IN_DMA);
2026 }
2027
2028
2029 /* Init job queue. */
2030 static int cryptocop_job_queue_init(void)
2031 {
2032         int i;
2033
2034         INIT_LIST_HEAD(&cryptocop_completed_jobs);
2035
2036         for (i = 0; i < cryptocop_prio_no_prios; i++){
2037                 cryptocop_job_queues[i].prio = (cryptocop_queue_priority)i;
2038                 INIT_LIST_HEAD(&cryptocop_job_queues[i].jobs);
2039         }
2040         return 0;
2041 }
2042
2043
2044 static void cryptocop_job_queue_close(void)
2045 {
2046         struct list_head               *node, *tmp;
2047         struct cryptocop_prio_job      *pj = NULL;
2048         unsigned long int              process_flags, flags;
2049         int                            i;
2050
2051         /* FIXME: This is as yet untested code. */
2052
2053         /* Stop strcop from getting an operation to process while we are closing the
2054            module. */
2055         spin_lock_irqsave(&cryptocop_process_lock, process_flags);
2056
2057         /* Empty the job queue. */
2058         for (i = 0; i < cryptocop_prio_no_prios; i++){
2059                 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
2060                         list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
2061                                 pj = list_entry(node, struct cryptocop_prio_job, node);
2062                                 list_del(node);
2063
2064                                 /* Call callback to notify consumer of job removal. */
2065                                 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2066                                 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2067                                 pj->oper->cb(pj->oper, pj->oper->cb_data);
2068
2069                                 delete_internal_operation(pj->iop);
2070                                 kfree(pj);
2071                         }
2072                 }
2073         }
2074         spin_unlock_irqrestore(&cryptocop_process_lock, process_flags);
2075
2076         /* Remove the running job, if any. */
2077         spin_lock_irqsave(&running_job_lock, flags);
2078         if (cryptocop_running_job){
2079                 reg_strcop_rw_cfg rw_cfg;
2080                 reg_dma_rw_cfg    dma_out_cfg, dma_in_cfg;
2081
2082                 /* Stop DMA. */
2083                 dma_out_cfg = REG_RD(dma, OUT_DMA_INST, rw_cfg);
2084                 dma_out_cfg.en = regk_dma_no;
2085                 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_out_cfg);
2086
2087                 dma_in_cfg = REG_RD(dma, IN_DMA_INST, rw_cfg);
2088                 dma_in_cfg.en = regk_dma_no;
2089                 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_in_cfg);
2090
2091                 /* Disble the cryptocop. */
2092                 rw_cfg = REG_RD(strcop, regi_strcop, rw_cfg);
2093                 rw_cfg.en = 0;
2094                 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2095
2096                 pj = cryptocop_running_job;
2097                 cryptocop_running_job = NULL;
2098
2099                 /* Call callback to notify consumer of job removal. */
2100                 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2101                 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2102                 pj->oper->cb(pj->oper, pj->oper->cb_data);
2103
2104                 delete_internal_operation(pj->iop);
2105                 kfree(pj);
2106         }
2107         spin_unlock_irqrestore(&running_job_lock, flags);
2108
2109         /* Remove completed jobs, if any. */
2110         spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
2111
2112         list_for_each_safe(node, tmp, &cryptocop_completed_jobs) {
2113                 pj = list_entry(node, struct cryptocop_prio_job, node);
2114                 list_del(node);
2115                 /* Call callback to notify consumer of job removal. */
2116                 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2117                 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2118                 pj->oper->cb(pj->oper, pj->oper->cb_data);
2119
2120                 delete_internal_operation(pj->iop);
2121                 kfree(pj);
2122         }
2123         spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
2124 }
2125
2126
2127 static void cryptocop_start_job(void)
2128 {
2129         int                          i;
2130         struct cryptocop_prio_job    *pj;
2131         unsigned long int            flags;
2132         unsigned long int            running_job_flags;
2133         reg_strcop_rw_cfg            rw_cfg = {.en = 1, .ignore_sync = 0};
2134
2135         DEBUG(printk("cryptocop_start_job: entering\n"));
2136
2137         spin_lock_irqsave(&running_job_lock, running_job_flags);
2138         if (cryptocop_running_job != NULL){
2139                 /* Already running. */
2140                 DEBUG(printk("cryptocop_start_job: already running, exit\n"));
2141                 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2142                 return;
2143         }
2144         spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
2145
2146         /* Check the queues in priority order. */
2147         for (i = cryptocop_prio_kernel_csum; (i < cryptocop_prio_no_prios) && list_empty(&cryptocop_job_queues[i].jobs); i++);
2148         if (i == cryptocop_prio_no_prios) {
2149                 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2150                 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2151                 DEBUG(printk("cryptocop_start_job: no jobs to run\n"));
2152                 return; /* No jobs to run */
2153         }
2154         DEBUG(printk("starting job for prio %d\n", i));
2155
2156         /* TODO: Do not starve lower priority jobs.  Let in a lower
2157          * prio job for every N-th processed higher prio job or some
2158          * other scheduling policy.  This could reasonably be
2159          * tweakable since the optimal balance would depend on the
2160          * type of load on the system. */
2161
2162         /* Pull the DMA lists from the job and start the DMA client. */
2163         pj = list_entry(cryptocop_job_queues[i].jobs.next, struct cryptocop_prio_job, node);
2164         list_del(&pj->node);
2165         spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2166         cryptocop_running_job = pj;
2167
2168         /* Set config register (3DES and CSUM modes). */
2169         switch (pj->iop->tdes_mode){
2170         case cryptocop_3des_eee:
2171                 rw_cfg.td1 = regk_strcop_e;
2172                 rw_cfg.td2 = regk_strcop_e;
2173                 rw_cfg.td3 = regk_strcop_e;
2174                 break;
2175         case cryptocop_3des_eed:
2176                 rw_cfg.td1 = regk_strcop_e;
2177                 rw_cfg.td2 = regk_strcop_e;
2178                 rw_cfg.td3 = regk_strcop_d;
2179                 break;
2180         case cryptocop_3des_ede:
2181                 rw_cfg.td1 = regk_strcop_e;
2182                 rw_cfg.td2 = regk_strcop_d;
2183                 rw_cfg.td3 = regk_strcop_e;
2184                 break;
2185         case cryptocop_3des_edd:
2186                 rw_cfg.td1 = regk_strcop_e;
2187                 rw_cfg.td2 = regk_strcop_d;
2188                 rw_cfg.td3 = regk_strcop_d;
2189                 break;
2190         case cryptocop_3des_dee:
2191                 rw_cfg.td1 = regk_strcop_d;
2192                 rw_cfg.td2 = regk_strcop_e;
2193                 rw_cfg.td3 = regk_strcop_e;
2194                 break;
2195         case cryptocop_3des_ded:
2196                 rw_cfg.td1 = regk_strcop_d;
2197                 rw_cfg.td2 = regk_strcop_e;
2198                 rw_cfg.td3 = regk_strcop_d;
2199                 break;
2200         case cryptocop_3des_dde:
2201                 rw_cfg.td1 = regk_strcop_d;
2202                 rw_cfg.td2 = regk_strcop_d;
2203                 rw_cfg.td3 = regk_strcop_e;
2204                 break;
2205         case cryptocop_3des_ddd:
2206                 rw_cfg.td1 = regk_strcop_d;
2207                 rw_cfg.td2 = regk_strcop_d;
2208                 rw_cfg.td3 = regk_strcop_d;
2209                 break;
2210         default:
2211                 DEBUG(printk("cryptocop_setup_dma_list: bad 3DES mode\n"));
2212         }
2213         switch (pj->iop->csum_mode){
2214         case cryptocop_csum_le:
2215                 rw_cfg.ipend = regk_strcop_little;
2216                 break;
2217         case cryptocop_csum_be:
2218                 rw_cfg.ipend = regk_strcop_big;
2219                 break;
2220         default:
2221                 DEBUG(printk("cryptocop_setup_dma_list: bad checksum mode\n"));
2222         }
2223         REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2224
2225         DEBUG(printk("cryptocop_start_job: starting DMA, new cryptocop_running_job=0x%p\n"
2226                      "ctx_in: 0x%p, phys: 0x%p\n"
2227                      "ctx_out: 0x%p, phys: 0x%p\n",
2228                      pj,
2229                      &pj->iop->ctx_in, (char*)virt_to_phys(&pj->iop->ctx_in),
2230                      &pj->iop->ctx_out, (char*)virt_to_phys(&pj->iop->ctx_out)));
2231
2232         /* Start input DMA. */
2233         flush_dma_context(&pj->iop->ctx_in);
2234         DMA_START_CONTEXT(IN_DMA_INST, virt_to_phys(&pj->iop->ctx_in));
2235
2236         /* Start output DMA. */
2237         DMA_START_CONTEXT(OUT_DMA_INST, virt_to_phys(&pj->iop->ctx_out));
2238
2239         spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2240         DEBUG(printk("cryptocop_start_job: exiting\n"));
2241 }
2242
2243
2244 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation)
2245 {
2246         int  err;
2247         int  alloc_flag = operation->in_interrupt ? GFP_ATOMIC : GFP_KERNEL;
2248         void *iop_alloc_ptr = NULL;
2249
2250         *pj = kmalloc(sizeof (struct cryptocop_prio_job), alloc_flag);
2251         if (!*pj) return -ENOMEM;
2252
2253         DEBUG(printk("cryptocop_job_setup: operation=0x%p\n", operation));
2254
2255         (*pj)->oper = operation;
2256         DEBUG(printk("cryptocop_job_setup, cb=0x%p cb_data=0x%p\n",  (*pj)->oper->cb, (*pj)->oper->cb_data));
2257
2258         if (operation->use_dmalists) {
2259                 DEBUG(print_user_dma_lists(&operation->list_op));
2260                 if (!operation->list_op.inlist || !operation->list_op.outlist || !operation->list_op.out_data_buf || !operation->list_op.in_data_buf){
2261                         DEBUG_API(printk("cryptocop_job_setup: bad indata (use_dmalists)\n"));
2262                         kfree(*pj);
2263                         return -EINVAL;
2264                 }
2265                 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
2266                 if (!iop_alloc_ptr) {
2267                         DEBUG_API(printk("cryptocop_job_setup: kmalloc cryptocop_int_operation\n"));
2268                         kfree(*pj);
2269                         return -ENOMEM;
2270                 }
2271                 (*pj)->iop = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
2272                 DEBUG(memset((*pj)->iop, 0xff, sizeof(struct cryptocop_int_operation)));
2273                 (*pj)->iop->alloc_ptr = iop_alloc_ptr;
2274                 (*pj)->iop->sid = operation->sid;
2275                 (*pj)->iop->cdesc_out = NULL;
2276                 (*pj)->iop->cdesc_in = NULL;
2277                 (*pj)->iop->tdes_mode = operation->list_op.tdes_mode;
2278                 (*pj)->iop->csum_mode = operation->list_op.csum_mode;
2279                 (*pj)->iop->ddesc_out = operation->list_op.outlist;
2280                 (*pj)->iop->ddesc_in = operation->list_op.inlist;
2281
2282                 /* Setup DMA contexts. */
2283                 (*pj)->iop->ctx_out.next = NULL;
2284                 (*pj)->iop->ctx_out.eol = 1;
2285                 (*pj)->iop->ctx_out.saved_data = operation->list_op.outlist;
2286                 (*pj)->iop->ctx_out.saved_data_buf = operation->list_op.out_data_buf;
2287
2288                 (*pj)->iop->ctx_in.next = NULL;
2289                 (*pj)->iop->ctx_in.eol = 1;
2290                 (*pj)->iop->ctx_in.saved_data = operation->list_op.inlist;
2291                 (*pj)->iop->ctx_in.saved_data_buf = operation->list_op.in_data_buf;
2292         } else {
2293                 if ((err = cryptocop_setup_dma_list(operation, &(*pj)->iop, alloc_flag))) {
2294                         DEBUG_API(printk("cryptocop_job_setup: cryptocop_setup_dma_list failed %d\n", err));
2295                         kfree(*pj);
2296                         return err;
2297                 }
2298         }
2299         DEBUG(print_dma_descriptors((*pj)->iop));
2300
2301         DEBUG(printk("cryptocop_job_setup, DMA list setup successful\n"));
2302
2303         return 0;
2304 }
2305
2306 static int cryptocop_open(struct inode *inode, struct file *filp)
2307 {
2308         int p = iminor(inode);
2309
2310         if (p != CRYPTOCOP_MINOR) return -EINVAL;
2311
2312         filp->private_data = NULL;
2313         return 0;
2314 }
2315
2316
2317 static int cryptocop_release(struct inode *inode, struct file *filp)
2318 {
2319         struct cryptocop_private *dev = filp->private_data;
2320         struct cryptocop_private *dev_next;
2321
2322         while (dev){
2323                 dev_next = dev->next;
2324                 if (dev->sid != CRYPTOCOP_SESSION_ID_NONE) {
2325                         (void)cryptocop_free_session(dev->sid);
2326                 }
2327                 kfree(dev);
2328                 dev = dev_next;
2329         }
2330
2331         return 0;
2332 }
2333
2334
2335 static int cryptocop_ioctl_close_session(struct inode *inode, struct file *filp,
2336                                          unsigned int cmd, unsigned long arg)
2337 {
2338         struct cryptocop_private  *dev = filp->private_data;
2339         struct cryptocop_private  *prev_dev = NULL;
2340         struct strcop_session_op  *sess_op = (struct strcop_session_op *)arg;
2341         struct strcop_session_op  sop;
2342         int                       err;
2343
2344         DEBUG(printk("cryptocop_ioctl_close_session\n"));
2345
2346         if (!access_ok(VERIFY_READ, sess_op, sizeof(struct strcop_session_op)))
2347                 return -EFAULT;
2348         err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2349         if (err) return -EFAULT;
2350
2351         while (dev && (dev->sid != sop.ses_id)) {
2352                 prev_dev = dev;
2353                 dev = dev->next;
2354         }
2355         if (dev){
2356                 if (prev_dev){
2357                         prev_dev->next = dev->next;
2358                 } else {
2359                         filp->private_data = dev->next;
2360                 }
2361                 err = cryptocop_free_session(dev->sid);
2362                 if (err) return -EFAULT;
2363         } else {
2364                 DEBUG_API(printk("cryptocop_ioctl_close_session: session %lld not found\n", sop.ses_id));
2365                 return -EINVAL;
2366         }
2367         return 0;
2368 }
2369
2370
2371 static void ioctl_process_job_callback(struct cryptocop_operation *op, void*cb_data)
2372 {
2373         struct ioctl_job_cb_ctx *jc = (struct ioctl_job_cb_ctx *)cb_data;
2374
2375         DEBUG(printk("ioctl_process_job_callback: op=0x%p, cb_data=0x%p\n", op, cb_data));
2376
2377         jc->processed = 1;
2378         wake_up(&cryptocop_ioc_process_wq);
2379 }
2380
2381
2382 #define CRYPTOCOP_IOCTL_CIPHER_TID  (1)
2383 #define CRYPTOCOP_IOCTL_DIGEST_TID  (2)
2384 #define CRYPTOCOP_IOCTL_CSUM_TID    (3)
2385
2386 static size_t first_cfg_change_ix(struct strcop_crypto_op *crp_op)
2387 {
2388         size_t ch_ix = 0;
2389
2390         if (crp_op->do_cipher) ch_ix = crp_op->cipher_start;
2391         if (crp_op->do_digest && (crp_op->digest_start < ch_ix)) ch_ix = crp_op->digest_start;
2392         if (crp_op->do_csum && (crp_op->csum_start < ch_ix)) ch_ix = crp_op->csum_start;
2393
2394         DEBUG(printk("first_cfg_change_ix: ix=%d\n", ch_ix));
2395         return ch_ix;
2396 }
2397
2398
2399 static size_t next_cfg_change_ix(struct strcop_crypto_op *crp_op, size_t ix)
2400 {
2401         size_t ch_ix = INT_MAX;
2402         size_t tmp_ix = 0;
2403
2404         if (crp_op->do_cipher && ((crp_op->cipher_start + crp_op->cipher_len) > ix)){
2405                 if (crp_op->cipher_start > ix) {
2406                         ch_ix = crp_op->cipher_start;
2407                 } else {
2408                         ch_ix = crp_op->cipher_start + crp_op->cipher_len;
2409                 }
2410         }
2411         if (crp_op->do_digest && ((crp_op->digest_start + crp_op->digest_len) > ix)){
2412                 if (crp_op->digest_start > ix) {
2413                         tmp_ix = crp_op->digest_start;
2414                 } else {
2415                         tmp_ix = crp_op->digest_start + crp_op->digest_len;
2416                 }
2417                 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2418         }
2419         if (crp_op->do_csum && ((crp_op->csum_start + crp_op->csum_len) > ix)){
2420                 if (crp_op->csum_start > ix) {
2421                         tmp_ix = crp_op->csum_start;
2422                 } else {
2423                         tmp_ix = crp_op->csum_start + crp_op->csum_len;
2424                 }
2425                 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2426         }
2427         if (ch_ix == INT_MAX) ch_ix = ix;
2428         DEBUG(printk("next_cfg_change_ix prev ix=%d, next ix=%d\n", ix, ch_ix));
2429         return ch_ix;
2430 }
2431
2432
2433 /* Map map_length bytes from the pages starting on *pageix and *pageoffset to iovecs starting on *iovix.
2434  * Return -1 for ok, 0 for fail. */
2435 static int map_pages_to_iovec(struct iovec *iov, int iovlen, int *iovix, struct page **pages, int nopages, int *pageix, int *pageoffset, int map_length )
2436 {
2437         int tmplen;
2438
2439         assert(iov != NULL);
2440         assert(iovix != NULL);
2441         assert(pages != NULL);
2442         assert(pageix != NULL);
2443         assert(pageoffset != NULL);
2444
2445         DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2446
2447         while (map_length > 0){
2448                 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2449                 if (*iovix >= iovlen){
2450                         DEBUG_API(printk("map_page_to_iovec: *iovix=%d >= iovlen=%d\n", *iovix, iovlen));
2451                         return 0;
2452                 }
2453                 if (*pageix >= nopages){
2454                         DEBUG_API(printk("map_page_to_iovec: *pageix=%d >= nopages=%d\n", *pageix, nopages));
2455                         return 0;
2456                 }
2457                 iov[*iovix].iov_base = (unsigned char*)page_address(pages[*pageix]) + *pageoffset;
2458                 tmplen = PAGE_SIZE - *pageoffset;
2459                 if (tmplen < map_length){
2460                         (*pageoffset) = 0;
2461                         (*pageix)++;
2462                 } else {
2463                         tmplen = map_length;
2464                         (*pageoffset) += map_length;
2465                 }
2466                 DEBUG(printk("mapping %d bytes from page %d (or %d) to iovec %d\n", tmplen, *pageix, *pageix-1, *iovix));
2467                 iov[*iovix].iov_len = tmplen;
2468                 map_length -= tmplen;
2469                 (*iovix)++;
2470         }
2471         DEBUG(printk("map_page_to_iovec, exit, *iovix=%d\n", *iovix));
2472         return -1;
2473 }
2474
2475
2476
2477 static int cryptocop_ioctl_process(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2478 {
2479         int                             i;
2480         struct cryptocop_private        *dev = filp->private_data;
2481         struct strcop_crypto_op         *crp_oper = (struct strcop_crypto_op *)arg;
2482         struct strcop_crypto_op         oper = {0};
2483         int                             err = 0;
2484         struct cryptocop_operation      *cop = NULL;
2485
2486         struct ioctl_job_cb_ctx         *jc = NULL;
2487
2488         struct page                     **inpages = NULL;
2489         struct page                     **outpages = NULL;
2490         int                             noinpages = 0;
2491         int                             nooutpages = 0;
2492
2493         struct cryptocop_desc           descs[5]; /* Max 5 descriptors are needed, there are three transforms that
2494                                                    * can get connected/disconnected on different places in the indata. */
2495         struct cryptocop_desc_cfg       dcfgs[5*3];
2496         int                             desc_ix = 0;
2497         int                             dcfg_ix = 0;
2498         struct cryptocop_tfrm_cfg       ciph_tcfg = {0};
2499         struct cryptocop_tfrm_cfg       digest_tcfg = {0};
2500         struct cryptocop_tfrm_cfg       csum_tcfg = {0};
2501
2502         unsigned char                   *digest_result = NULL;
2503         int                             digest_length = 0;
2504         int                             cblocklen = 0;
2505         unsigned char                   csum_result[CSUM_BLOCK_LENGTH];
2506         struct cryptocop_session        *sess;
2507
2508         int    iovlen = 0;
2509         int    iovix = 0;
2510         int    pageix = 0;
2511         int    pageoffset = 0;
2512
2513         size_t prev_ix = 0;
2514         size_t next_ix;
2515
2516         int    cipher_active, digest_active, csum_active;
2517         int    end_digest, end_csum;
2518         int    digest_done = 0;
2519         int    cipher_done = 0;
2520         int    csum_done = 0;
2521
2522         DEBUG(printk("cryptocop_ioctl_process\n"));
2523
2524         if (!access_ok(VERIFY_WRITE, crp_oper, sizeof(struct strcop_crypto_op))){
2525                 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok crp_oper!\n"));
2526                 return -EFAULT;
2527         }
2528         if (copy_from_user(&oper, crp_oper, sizeof(struct strcop_crypto_op))) {
2529                 DEBUG_API(printk("cryptocop_ioctl_process: copy_from_user\n"));
2530                 return -EFAULT;
2531         }
2532         DEBUG(print_strcop_crypto_op(&oper));
2533
2534         while (dev && dev->sid != oper.ses_id) dev = dev->next;
2535         if (!dev){
2536                 DEBUG_API(printk("cryptocop_ioctl_process: session %lld not found\n", oper.ses_id));
2537                 return -EINVAL;
2538         }
2539
2540         /* Check buffers. */
2541         if (((oper.indata + oper.inlen) < oper.indata) || ((oper.cipher_outdata + oper.cipher_outlen) < oper.cipher_outdata)){
2542                 DEBUG_API(printk("cryptocop_ioctl_process: user buffers wrapped around, bad user!\n"));
2543                 return -EINVAL;
2544         }
2545
2546         if (!access_ok(VERIFY_WRITE, oper.cipher_outdata, oper.cipher_outlen)){
2547                 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok out data!\n"));
2548                 return -EFAULT;
2549         }
2550         if (!access_ok(VERIFY_READ, oper.indata, oper.inlen)){
2551                 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok in data!\n"));
2552                 return -EFAULT;
2553         }
2554
2555         cop = kmalloc(sizeof(struct cryptocop_operation), GFP_KERNEL);
2556         if (!cop) {
2557                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2558                 return -ENOMEM;
2559         }
2560         jc = kmalloc(sizeof(struct ioctl_job_cb_ctx), GFP_KERNEL);
2561         if (!jc) {
2562                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2563                 err = -ENOMEM;
2564                 goto error_cleanup;
2565         }
2566         jc->processed = 0;
2567
2568         cop->cb_data = jc;
2569         cop->cb = ioctl_process_job_callback;
2570         cop->operation_status = 0;
2571         cop->use_dmalists = 0;
2572         cop->in_interrupt = 0;
2573         cop->fast_callback = 0;
2574         cop->tfrm_op.tfrm_cfg = NULL;
2575         cop->tfrm_op.desc = NULL;
2576         cop->tfrm_op.indata = NULL;
2577         cop->tfrm_op.incount = 0;
2578         cop->tfrm_op.inlen = 0;
2579         cop->tfrm_op.outdata = NULL;
2580         cop->tfrm_op.outcount = 0;
2581         cop->tfrm_op.outlen = 0;
2582
2583         sess = get_session(oper.ses_id);
2584         if (!sess){
2585                 DEBUG_API(printk("cryptocop_ioctl_process: bad session id.\n"));
2586                 kfree(cop);
2587                 kfree(jc);
2588                 return -EINVAL;
2589         }
2590
2591         if (oper.do_cipher) {
2592                 unsigned int                    cipher_outlen = 0;
2593                 struct cryptocop_transform_ctx  *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_CIPHER_TID);
2594                 if (!tc) {
2595                         DEBUG_API(printk("cryptocop_ioctl_process: no cipher transform in session.\n"));
2596                         err = -EINVAL;
2597                         goto error_cleanup;
2598                 }
2599                 ciph_tcfg.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2600                 ciph_tcfg.inject_ix = 0;
2601                 ciph_tcfg.flags = 0;
2602                 if ((oper.cipher_start < 0) || (oper.cipher_len <= 0) || (oper.cipher_start > oper.inlen) || ((oper.cipher_start + oper.cipher_len) > oper.inlen)){
2603                         DEBUG_API(printk("cryptocop_ioctl_process: bad cipher length\n"));
2604                         kfree(cop);
2605                         kfree(jc);
2606                         return -EINVAL;
2607                 }
2608                 cblocklen = tc->init.alg == cryptocop_alg_aes ? AES_BLOCK_LENGTH : DES_BLOCK_LENGTH;
2609                 if (oper.cipher_len % cblocklen) {
2610                         kfree(cop);
2611                         kfree(jc);
2612                         DEBUG_API(printk("cryptocop_ioctl_process: cipher inlength not multiple of block length.\n"));
2613                         return -EINVAL;
2614                 }
2615                 cipher_outlen = oper.cipher_len;
2616                 if (tc->init.cipher_mode == cryptocop_cipher_mode_cbc){
2617                         if (oper.cipher_explicit) {
2618                                 ciph_tcfg.flags |= CRYPTOCOP_EXPLICIT_IV;
2619                                 memcpy(ciph_tcfg.iv, oper.cipher_iv, cblocklen);
2620                         } else {
2621                                 cipher_outlen = oper.cipher_len - cblocklen;
2622                         }
2623                 } else {
2624                         if (oper.cipher_explicit){
2625                                 kfree(cop);
2626                                 kfree(jc);
2627                                 DEBUG_API(printk("cryptocop_ioctl_process: explicit_iv when not CBC mode\n"));
2628                                 return -EINVAL;
2629                         }
2630                 }
2631                 if (oper.cipher_outlen != cipher_outlen) {
2632                         kfree(cop);
2633                         kfree(jc);
2634                         DEBUG_API(printk("cryptocop_ioctl_process: cipher_outlen incorrect, should be %d not %d.\n", cipher_outlen, oper.cipher_outlen));
2635                         return -EINVAL;
2636                 }
2637
2638                 if (oper.decrypt){
2639                         ciph_tcfg.flags |= CRYPTOCOP_DECRYPT;
2640                 } else {
2641                         ciph_tcfg.flags |= CRYPTOCOP_ENCRYPT;
2642                 }
2643                 ciph_tcfg.next = cop->tfrm_op.tfrm_cfg;
2644                 cop->tfrm_op.tfrm_cfg = &ciph_tcfg;
2645         }
2646         if (oper.do_digest){
2647                 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_DIGEST_TID);
2648                 if (!tc) {
2649                         DEBUG_API(printk("cryptocop_ioctl_process: no digest transform in session.\n"));
2650                         err = -EINVAL;
2651                         goto error_cleanup;
2652                 }
2653                 digest_length = tc->init.alg == cryptocop_alg_md5 ? 16 : 20;
2654                 digest_result = kmalloc(digest_length, GFP_KERNEL);
2655                 if (!digest_result) {
2656                         DEBUG_API(printk("cryptocop_ioctl_process: kmalloc digest_result\n"));
2657                         err = -EINVAL;
2658                         goto error_cleanup;
2659                 }
2660                 DEBUG(memset(digest_result, 0xff, digest_length));
2661
2662                 digest_tcfg.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2663                 digest_tcfg.inject_ix = 0;
2664                 ciph_tcfg.inject_ix += digest_length;
2665                 if ((oper.digest_start < 0) || (oper.digest_len <= 0) || (oper.digest_start > oper.inlen) || ((oper.digest_start + oper.digest_len) > oper.inlen)){
2666                         DEBUG_API(printk("cryptocop_ioctl_process: bad digest length\n"));
2667                         err = -EINVAL;
2668                         goto error_cleanup;
2669                 }
2670
2671                 digest_tcfg.next = cop->tfrm_op.tfrm_cfg;
2672                 cop->tfrm_op.tfrm_cfg = &digest_tcfg;
2673         }
2674         if (oper.do_csum){
2675                 csum_tcfg.tid = CRYPTOCOP_IOCTL_CSUM_TID;
2676                 csum_tcfg.inject_ix = digest_length;
2677                 ciph_tcfg.inject_ix += 2;
2678
2679                 if ((oper.csum_start < 0) || (oper.csum_len <= 0) || (oper.csum_start > oper.inlen) || ((oper.csum_start + oper.csum_len) > oper.inlen)){
2680                         DEBUG_API(printk("cryptocop_ioctl_process: bad csum length\n"));
2681                         kfree(cop);
2682                         kfree(jc);
2683                         return -EINVAL;
2684                 }
2685
2686                 csum_tcfg.next = cop->tfrm_op.tfrm_cfg;
2687                 cop->tfrm_op.tfrm_cfg = &csum_tcfg;
2688         }
2689
2690         prev_ix = first_cfg_change_ix(&oper);
2691         if (prev_ix > oper.inlen) {
2692                 DEBUG_API(printk("cryptocop_ioctl_process: length mismatch\n"));
2693                 nooutpages = noinpages = 0;
2694                 err = -EINVAL;
2695                 goto error_cleanup;
2696         }
2697         DEBUG(printk("cryptocop_ioctl_process: inlen=%d, cipher_outlen=%d\n", oper.inlen, oper.cipher_outlen));
2698
2699         /* Map user pages for in and out data of the operation. */
2700         noinpages = (((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK) + oper.inlen - 1 - prev_ix + ~PAGE_MASK) >> PAGE_SHIFT;
2701         DEBUG(printk("cryptocop_ioctl_process: noinpages=%d\n", noinpages));
2702         inpages = kmalloc(noinpages * sizeof(struct page*), GFP_KERNEL);
2703         if (!inpages){
2704                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc inpages\n"));
2705                 nooutpages = noinpages = 0;
2706                 err = -ENOMEM;
2707                 goto error_cleanup;
2708         }
2709         if (oper.do_cipher){
2710                 nooutpages = (((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) + oper.cipher_outlen - 1 + ~PAGE_MASK) >> PAGE_SHIFT;
2711                 DEBUG(printk("cryptocop_ioctl_process: nooutpages=%d\n", nooutpages));
2712                 outpages = kmalloc(nooutpages * sizeof(struct page*), GFP_KERNEL);
2713                 if (!outpages){
2714                         DEBUG_API(printk("cryptocop_ioctl_process: kmalloc outpages\n"));
2715                         nooutpages = noinpages = 0;
2716                         err = -ENOMEM;
2717                         goto error_cleanup;
2718                 }
2719         }
2720
2721         /* Acquire the mm page semaphore. */
2722         down_read(&current->mm->mmap_sem);
2723
2724         err = get_user_pages(current,
2725                              current->mm,
2726                              (unsigned long int)(oper.indata + prev_ix),
2727                              noinpages,
2728                              0,  /* read access only for in data */
2729                              0, /* no force */
2730                              inpages,
2731                              NULL);
2732
2733         if (err < 0) {
2734                 up_read(&current->mm->mmap_sem);
2735                 nooutpages = noinpages = 0;
2736                 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages indata\n"));
2737                 goto error_cleanup;
2738         }
2739         noinpages = err;
2740         if (oper.do_cipher){
2741                 err = get_user_pages(current,
2742                                      current->mm,
2743                                      (unsigned long int)oper.cipher_outdata,
2744                                      nooutpages,
2745                                      1, /* write access for out data */
2746                                      0, /* no force */
2747                                      outpages,
2748                                      NULL);
2749                 up_read(&current->mm->mmap_sem);
2750                 if (err < 0) {
2751                         nooutpages = 0;
2752                         DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages outdata\n"));
2753                         goto error_cleanup;
2754                 }
2755                 nooutpages = err;
2756         } else {
2757                 up_read(&current->mm->mmap_sem);
2758         }
2759
2760         /* Add 6 to nooutpages to make room for possibly inserted buffers for storing digest and
2761          * csum output and splits when units are (dis-)connected. */
2762         cop->tfrm_op.indata = kmalloc((noinpages) * sizeof(struct iovec), GFP_KERNEL);
2763         cop->tfrm_op.outdata = kmalloc((6 + nooutpages) * sizeof(struct iovec), GFP_KERNEL);
2764         if (!cop->tfrm_op.indata || !cop->tfrm_op.outdata) {
2765                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc iovecs\n"));
2766                 err = -ENOMEM;
2767                 goto error_cleanup;
2768         }
2769
2770         cop->tfrm_op.inlen = oper.inlen - prev_ix;
2771         cop->tfrm_op.outlen = 0;
2772         if (oper.do_cipher) cop->tfrm_op.outlen += oper.cipher_outlen;
2773         if (oper.do_digest) cop->tfrm_op.outlen += digest_length;
2774         if (oper.do_csum) cop->tfrm_op.outlen += 2;
2775
2776         /* Setup the in iovecs. */
2777         cop->tfrm_op.incount = noinpages;
2778         if (noinpages > 1){
2779                 size_t tmplen = cop->tfrm_op.inlen;
2780
2781                 cop->tfrm_op.indata[0].iov_len = PAGE_SIZE - ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2782                 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2783                 tmplen -= cop->tfrm_op.indata[0].iov_len;
2784                 for (i = 1; i<noinpages; i++){
2785                         cop->tfrm_op.indata[i].iov_len = tmplen < PAGE_SIZE ? tmplen : PAGE_SIZE;
2786                         cop->tfrm_op.indata[i].iov_base = (unsigned char*)page_address(inpages[i]);
2787                         tmplen -= PAGE_SIZE;
2788                 }
2789         } else {
2790                 cop->tfrm_op.indata[0].iov_len = oper.inlen - prev_ix;
2791                 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2792         }
2793
2794         iovlen = nooutpages + 6;
2795         pageoffset = oper.do_cipher ? ((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) : 0;
2796
2797         next_ix = next_cfg_change_ix(&oper, prev_ix);
2798         if (prev_ix == next_ix){
2799                 DEBUG_API(printk("cryptocop_ioctl_process: length configuration broken.\n"));
2800                 err = -EINVAL;  /* This should be impossible barring bugs. */
2801                 goto error_cleanup;
2802         }
2803         while (prev_ix != next_ix){
2804                 end_digest = end_csum = cipher_active = digest_active = csum_active = 0;
2805                 descs[desc_ix].cfg = NULL;
2806                 descs[desc_ix].length = next_ix - prev_ix;
2807
2808                 if (oper.do_cipher && (oper.cipher_start < next_ix) && (prev_ix < (oper.cipher_start + oper.cipher_len))) {
2809                         dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2810                         dcfgs[dcfg_ix].src = cryptocop_source_dma;
2811                         cipher_active = 1;
2812
2813                         if (next_ix == (oper.cipher_start + oper.cipher_len)){
2814                                 cipher_done = 1;
2815                                 dcfgs[dcfg_ix].last = 1;
2816                         } else {
2817                                 dcfgs[dcfg_ix].last = 0;
2818                         }
2819                         dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2820                         descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2821                         ++dcfg_ix;
2822                 }
2823                 if (oper.do_digest && (oper.digest_start < next_ix) && (prev_ix < (oper.digest_start + oper.digest_len))) {
2824                         digest_active = 1;
2825                         dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2826                         dcfgs[dcfg_ix].src = cryptocop_source_dma;
2827                         if (next_ix == (oper.digest_start + oper.digest_len)){
2828                                 assert(!digest_done);
2829                                 digest_done = 1;
2830                                 dcfgs[dcfg_ix].last = 1;
2831                         } else {
2832                                 dcfgs[dcfg_ix].last = 0;
2833                         }
2834                         dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2835                         descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2836                         ++dcfg_ix;
2837                 }
2838                 if (oper.do_csum && (oper.csum_start < next_ix) && (prev_ix < (oper.csum_start + oper.csum_len))){
2839                         csum_active = 1;
2840                         dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CSUM_TID;
2841                         dcfgs[dcfg_ix].src = cryptocop_source_dma;
2842                         if (next_ix == (oper.csum_start + oper.csum_len)){
2843                                 csum_done = 1;
2844                                 dcfgs[dcfg_ix].last = 1;
2845                         } else {
2846                                 dcfgs[dcfg_ix].last = 0;
2847                         }
2848                         dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2849                         descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2850                         ++dcfg_ix;
2851                 }
2852                 if (!descs[desc_ix].cfg){
2853                         DEBUG_API(printk("cryptocop_ioctl_process: data segment %d (%d to %d) had no active transforms\n", desc_ix, prev_ix, next_ix));
2854                         err = -EINVAL;
2855                         goto error_cleanup;
2856                 }
2857                 descs[desc_ix].next = &(descs[desc_ix]) + 1;
2858                 ++desc_ix;
2859                 prev_ix = next_ix;
2860                 next_ix = next_cfg_change_ix(&oper, prev_ix);
2861         }
2862         if (desc_ix > 0){
2863                 descs[desc_ix-1].next = NULL;
2864         } else {
2865                 descs[0].next = NULL;
2866         }
2867         if (oper.do_digest) {
2868                 DEBUG(printk("cryptocop_ioctl_process: mapping %d byte digest output to iovec %d\n", digest_length, iovix));
2869                 /* Add outdata iovec, length == <length of type of digest> */
2870                 cop->tfrm_op.outdata[iovix].iov_base = digest_result;
2871                 cop->tfrm_op.outdata[iovix].iov_len = digest_length;
2872                 ++iovix;
2873         }
2874         if (oper.do_csum) {
2875                 /* Add outdata iovec, length == 2, the length of csum. */
2876                 DEBUG(printk("cryptocop_ioctl_process: mapping 2 byte csum output to iovec %d\n", iovix));
2877                 /* Add outdata iovec, length == <length of type of digest> */
2878                 cop->tfrm_op.outdata[iovix].iov_base = csum_result;
2879                 cop->tfrm_op.outdata[iovix].iov_len = 2;
2880                 ++iovix;
2881         }
2882         if (oper.do_cipher) {
2883                 if (!map_pages_to_iovec(cop->tfrm_op.outdata, iovlen, &iovix, outpages, nooutpages, &pageix, &pageoffset, oper.cipher_outlen)){
2884                         DEBUG_API(printk("cryptocop_ioctl_process: failed to map pages to iovec.\n"));
2885                         err = -ENOSYS; /* This should be impossible barring bugs. */
2886                         goto error_cleanup;
2887                 }
2888         }
2889         DEBUG(printk("cryptocop_ioctl_process: setting cop->tfrm_op.outcount %d\n", iovix));
2890         cop->tfrm_op.outcount = iovix;
2891         assert(iovix <= (nooutpages + 6));
2892
2893         cop->sid = oper.ses_id;
2894         cop->tfrm_op.desc = &descs[0];
2895
2896         DEBUG(printk("cryptocop_ioctl_process: inserting job, cb_data=0x%p\n", cop->cb_data));
2897
2898         if ((err = cryptocop_job_queue_insert_user_job(cop)) != 0) {
2899                 DEBUG_API(printk("cryptocop_ioctl_process: insert job %d\n", err));
2900                 err = -EINVAL;
2901                 goto error_cleanup;
2902         }
2903
2904         DEBUG(printk("cryptocop_ioctl_process: begin wait for result\n"));
2905
2906         wait_event(cryptocop_ioc_process_wq, (jc->processed != 0));
2907         DEBUG(printk("cryptocop_ioctl_process: end wait for result\n"));
2908         if (!jc->processed){
2909                 printk(KERN_WARNING "cryptocop_ioctl_process: job not processed at completion\n");
2910                 err = -EIO;
2911                 goto error_cleanup;
2912         }
2913
2914         /* Job process done.  Cipher output should already be correct in job so no post processing of outdata. */
2915         DEBUG(printk("cryptocop_ioctl_process: operation_status = %d\n", cop->operation_status));
2916         if (cop->operation_status == 0){
2917                 if (oper.do_digest){
2918                         DEBUG(printk("cryptocop_ioctl_process: copy %d bytes digest to user\n", digest_length));
2919                         err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, digest), digest_result, digest_length);
2920                         if (0 != err){
2921                                 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, digest length %d, err %d\n", digest_length, err));
2922                                 err = -EFAULT;
2923                                 goto error_cleanup;
2924                         }
2925                 }
2926                 if (oper.do_csum){
2927                         DEBUG(printk("cryptocop_ioctl_process: copy 2 bytes checksum to user\n"));
2928                         err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, csum), csum_result, 2);
2929                         if (0 != err){
2930                                 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, csum, err %d\n", err));
2931                                 err = -EFAULT;
2932                                 goto error_cleanup;
2933                         }
2934                 }
2935                 err = 0;
2936         } else {
2937                 DEBUG(printk("cryptocop_ioctl_process: returning err = operation_status = %d\n", cop->operation_status));
2938                 err = cop->operation_status;
2939         }
2940
2941  error_cleanup:
2942         /* Release page caches. */
2943         for (i = 0; i < noinpages; i++){
2944                 put_page(inpages[i]);
2945         }
2946         for (i = 0; i < nooutpages; i++){
2947                 int spdl_err;
2948                 /* Mark output pages dirty. */
2949                 spdl_err = set_page_dirty_lock(outpages[i]);
2950                 DEBUG(if (spdl_err < 0)printk("cryptocop_ioctl_process: set_page_dirty_lock returned %d\n", spdl_err));
2951         }
2952         for (i = 0; i < nooutpages; i++){
2953                 put_page(outpages[i]);
2954         }
2955
2956         kfree(digest_result);
2957         kfree(inpages);
2958         kfree(outpages);
2959         if (cop){
2960                 kfree(cop->tfrm_op.indata);
2961                 kfree(cop->tfrm_op.outdata);
2962                 kfree(cop);
2963         }
2964         kfree(jc);
2965
2966         DEBUG(print_lock_status());
2967
2968         return err;
2969 }
2970
2971
2972 static int cryptocop_ioctl_create_session(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2973 {
2974         cryptocop_session_id             sid;
2975         int                              err;
2976         struct cryptocop_private         *dev;
2977         struct strcop_session_op         *sess_op = (struct strcop_session_op *)arg;
2978         struct strcop_session_op         sop;
2979         struct cryptocop_transform_init  *tis = NULL;
2980         struct cryptocop_transform_init  ti_cipher = {0};
2981         struct cryptocop_transform_init  ti_digest = {0};
2982         struct cryptocop_transform_init  ti_csum = {0};
2983
2984         if (!access_ok(VERIFY_WRITE, sess_op, sizeof(struct strcop_session_op)))
2985                 return -EFAULT;
2986         err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2987         if (err) return -EFAULT;
2988         if (sop.cipher != cryptocop_cipher_none) {
2989                 if (!access_ok(VERIFY_READ, sop.key, sop.keylen)) return -EFAULT;
2990         }
2991         DEBUG(printk("cryptocop_ioctl_create_session, sess_op:\n"));
2992
2993         DEBUG(printk("\tcipher:%d\n"
2994                      "\tcipher_mode:%d\n"
2995                      "\tdigest:%d\n"
2996                      "\tcsum:%d\n",
2997                      (int)sop.cipher,
2998                      (int)sop.cmode,
2999                      (int)sop.digest,
3000                      (int)sop.csum));
3001
3002         if (sop.cipher != cryptocop_cipher_none){
3003                 /* Init the cipher. */
3004                 switch (sop.cipher){
3005                 case cryptocop_cipher_des:
3006                         ti_cipher.alg = cryptocop_alg_des;
3007                         break;
3008                 case cryptocop_cipher_3des:
3009                         ti_cipher.alg = cryptocop_alg_3des;
3010                         break;
3011                 case cryptocop_cipher_aes:
3012                         ti_cipher.alg = cryptocop_alg_aes;
3013                         break;
3014                 default:
3015                         DEBUG_API(printk("create session, bad cipher algorithm %d\n", sop.cipher));
3016                         return -EINVAL;
3017                 };
3018                 DEBUG(printk("setting cipher transform %d\n", ti_cipher.alg));
3019                 copy_from_user(ti_cipher.key, sop.key, sop.keylen/8);
3020                 ti_cipher.keylen = sop.keylen;
3021                 switch (sop.cmode){
3022                 case cryptocop_cipher_mode_cbc:
3023                 case cryptocop_cipher_mode_ecb:
3024                         ti_cipher.cipher_mode = sop.cmode;
3025                         break;
3026                 default:
3027                         DEBUG_API(printk("create session, bad cipher mode %d\n", sop.cmode));
3028                         return -EINVAL;
3029                 }
3030                 DEBUG(printk("cryptocop_ioctl_create_session: setting CBC mode %d\n", ti_cipher.cipher_mode));
3031                 switch (sop.des3_mode){
3032                 case cryptocop_3des_eee:
3033                 case cryptocop_3des_eed:
3034                 case cryptocop_3des_ede:
3035                 case cryptocop_3des_edd:
3036                 case cryptocop_3des_dee:
3037                 case cryptocop_3des_ded:
3038                 case cryptocop_3des_dde:
3039                 case cryptocop_3des_ddd:
3040                         ti_cipher.tdes_mode = sop.des3_mode;
3041                         break;
3042                 default:
3043                         DEBUG_API(printk("create session, bad 3DES mode %d\n", sop.des3_mode));
3044                         return -EINVAL;
3045                 }
3046                 ti_cipher.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
3047                 ti_cipher.next = tis;
3048                 tis = &ti_cipher;
3049         } /* if (sop.cipher != cryptocop_cipher_none) */
3050         if (sop.digest != cryptocop_digest_none){
3051                 DEBUG(printk("setting digest transform\n"));
3052                 switch (sop.digest){
3053                 case cryptocop_digest_md5:
3054                         ti_digest.alg = cryptocop_alg_md5;
3055                         break;
3056                 case cryptocop_digest_sha1:
3057                         ti_digest.alg = cryptocop_alg_sha1;
3058                         break;
3059                 default:
3060                         DEBUG_API(printk("create session, bad digest algorithm %d\n", sop.digest));
3061                         return -EINVAL;
3062                 }
3063                 ti_digest.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
3064                 ti_digest.next = tis;
3065                 tis = &ti_digest;
3066         } /* if (sop.digest != cryptocop_digest_none) */
3067         if (sop.csum != cryptocop_csum_none){
3068                 DEBUG(printk("setting csum transform\n"));
3069                 switch (sop.csum){
3070                 case cryptocop_csum_le:
3071                 case cryptocop_csum_be:
3072                         ti_csum.csum_mode = sop.csum;
3073                         break;
3074                 default:
3075                         DEBUG_API(printk("create session, bad checksum algorithm %d\n", sop.csum));
3076                         return -EINVAL;
3077                 }
3078                 ti_csum.alg = cryptocop_alg_csum;
3079                 ti_csum.tid = CRYPTOCOP_IOCTL_CSUM_TID;
3080                 ti_csum.next = tis;
3081                 tis = &ti_csum;
3082         } /* (sop.csum != cryptocop_csum_none) */
3083         dev = kmalloc(sizeof(struct cryptocop_private), GFP_KERNEL);
3084         if (!dev){
3085                 DEBUG_API(printk("create session, alloc dev\n"));
3086                 return -ENOMEM;
3087         }
3088
3089         err = cryptocop_new_session(&sid, tis, GFP_KERNEL);
3090         DEBUG({ if (err) printk("create session, cryptocop_new_session %d\n", err);});
3091
3092         if (err) {
3093                 kfree(dev);
3094                 return err;
3095         }
3096         sess_op->ses_id = sid;
3097         dev->sid = sid;
3098         dev->next = filp->private_data;
3099         filp->private_data = dev;
3100
3101         return 0;
3102 }
3103
3104 static long cryptocop_ioctl_unlocked(struct inode *inode,
3105         struct file *filp, unsigned int cmd, unsigned long arg)
3106 {
3107         int err = 0;
3108         if (_IOC_TYPE(cmd) != ETRAXCRYPTOCOP_IOCTYPE) {
3109                 DEBUG_API(printk("cryptocop_ioctl: wrong type\n"));
3110                 return -ENOTTY;
3111         }
3112         if (_IOC_NR(cmd) > CRYPTOCOP_IO_MAXNR){
3113                 return -ENOTTY;
3114         }
3115         /* Access check of the argument.  Some commands, e.g. create session and process op,
3116            needs additional checks.  Those are handled in the command handling functions. */
3117         if (_IOC_DIR(cmd) & _IOC_READ)
3118                 err = !access_ok(VERIFY_WRITE, (void *)arg, _IOC_SIZE(cmd));
3119         else if (_IOC_DIR(cmd) & _IOC_WRITE)
3120                 err = !access_ok(VERIFY_READ, (void *)arg, _IOC_SIZE(cmd));
3121         if (err) return -EFAULT;
3122
3123         switch (cmd) {
3124         case CRYPTOCOP_IO_CREATE_SESSION:
3125                 return cryptocop_ioctl_create_session(inode, filp, cmd, arg);
3126         case CRYPTOCOP_IO_CLOSE_SESSION:
3127                 return cryptocop_ioctl_close_session(inode, filp, cmd, arg);
3128         case CRYPTOCOP_IO_PROCESS_OP:
3129                 return cryptocop_ioctl_process(inode, filp, cmd, arg);
3130         default:
3131                 DEBUG_API(printk("cryptocop_ioctl: unknown command\n"));
3132                 return -ENOTTY;
3133         }
3134         return 0;
3135 }
3136
3137 static long
3138 cryptocop_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
3139 {
3140        struct inode *inode = file->f_path.dentry->d_inode;
3141        long ret;
3142
3143        mutex_lock(&cryptocop_mutex);
3144        ret = cryptocop_ioctl_unlocked(inode, filp, cmd, arg);
3145        mutex_unlock(&cryptocop_mutex);
3146
3147        return ret;
3148 }
3149
3150
3151 #ifdef LDEBUG
3152 static void print_dma_descriptors(struct cryptocop_int_operation *iop)
3153 {
3154         struct cryptocop_dma_desc *cdesc_out = iop->cdesc_out;
3155         struct cryptocop_dma_desc *cdesc_in = iop->cdesc_in;
3156         int                       i;
3157
3158         printk("print_dma_descriptors start\n");
3159
3160         printk("iop:\n");
3161         printk("\tsid: 0x%lld\n", iop->sid);
3162
3163         printk("\tcdesc_out: 0x%p\n", iop->cdesc_out);
3164         printk("\tcdesc_in: 0x%p\n", iop->cdesc_in);
3165         printk("\tddesc_out: 0x%p\n", iop->ddesc_out);
3166         printk("\tddesc_in: 0x%p\n", iop->ddesc_in);
3167
3168         printk("\niop->ctx_out: 0x%p phys: 0x%p\n", &iop->ctx_out, (char*)virt_to_phys(&iop->ctx_out));
3169         printk("\tnext: 0x%p\n"
3170                "\tsaved_data: 0x%p\n"
3171                "\tsaved_data_buf: 0x%p\n",
3172                iop->ctx_out.next,
3173                iop->ctx_out.saved_data,
3174                iop->ctx_out.saved_data_buf);
3175
3176         printk("\niop->ctx_in: 0x%p phys: 0x%p\n", &iop->ctx_in, (char*)virt_to_phys(&iop->ctx_in));
3177         printk("\tnext: 0x%p\n"
3178                "\tsaved_data: 0x%p\n"
3179                "\tsaved_data_buf: 0x%p\n",
3180                iop->ctx_in.next,
3181                iop->ctx_in.saved_data,
3182                iop->ctx_in.saved_data_buf);
3183
3184         i = 0;
3185         while (cdesc_out) {
3186                 dma_descr_data *td;
3187                 printk("cdesc_out %d, desc=0x%p\n", i, cdesc_out->dma_descr);
3188                 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_out->dma_descr));
3189                 td = cdesc_out->dma_descr;
3190                 printk("\n\tbuf: 0x%p\n"
3191                        "\tafter: 0x%p\n"
3192                        "\tmd: 0x%04x\n"
3193                        "\tnext: 0x%p\n",
3194                        td->buf,
3195                        td->after,
3196                        td->md,
3197                        td->next);
3198                 printk("flags:\n"
3199                        "\twait:\t%d\n"
3200                        "\teol:\t%d\n"
3201                        "\touteop:\t%d\n"
3202                        "\tineop:\t%d\n"
3203                        "\tintr:\t%d\n",
3204                        td->wait,
3205                        td->eol,
3206                        td->out_eop,
3207                        td->in_eop,
3208                        td->intr);
3209                 cdesc_out = cdesc_out->next;
3210                 i++;
3211         }
3212         i = 0;
3213         while (cdesc_in) {
3214                 dma_descr_data *td;
3215                 printk("cdesc_in %d, desc=0x%p\n", i, cdesc_in->dma_descr);
3216                 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_in->dma_descr));
3217                 td = cdesc_in->dma_descr;
3218                 printk("\n\tbuf: 0x%p\n"
3219                        "\tafter: 0x%p\n"
3220                        "\tmd: 0x%04x\n"
3221                        "\tnext: 0x%p\n",
3222                        td->buf,
3223                        td->after,
3224                        td->md,
3225                        td->next);
3226                 printk("flags:\n"
3227                        "\twait:\t%d\n"
3228                        "\teol:\t%d\n"
3229                        "\touteop:\t%d\n"
3230                        "\tineop:\t%d\n"
3231                        "\tintr:\t%d\n",
3232                        td->wait,
3233                        td->eol,
3234                        td->out_eop,
3235                        td->in_eop,
3236                        td->intr);
3237                 cdesc_in = cdesc_in->next;
3238                 i++;
3239         }
3240
3241         printk("print_dma_descriptors end\n");
3242 }
3243
3244
3245 static void print_strcop_crypto_op(struct strcop_crypto_op *cop)
3246 {
3247         printk("print_strcop_crypto_op, 0x%p\n", cop);
3248
3249         /* Indata. */
3250         printk("indata=0x%p\n"
3251                "inlen=%d\n"
3252                "do_cipher=%d\n"
3253                "decrypt=%d\n"
3254                "cipher_explicit=%d\n"
3255                "cipher_start=%d\n"
3256                "cipher_len=%d\n"
3257                "outdata=0x%p\n"
3258                "outlen=%d\n",
3259                cop->indata,
3260                cop->inlen,
3261                cop->do_cipher,
3262                cop->decrypt,
3263                cop->cipher_explicit,
3264                cop->cipher_start,
3265                cop->cipher_len,
3266                cop->cipher_outdata,
3267                cop->cipher_outlen);
3268
3269         printk("do_digest=%d\n"
3270                "digest_start=%d\n"
3271                "digest_len=%d\n",
3272                cop->do_digest,
3273                cop->digest_start,
3274                cop->digest_len);
3275
3276         printk("do_csum=%d\n"
3277                "csum_start=%d\n"
3278                "csum_len=%d\n",
3279                cop->do_csum,
3280                cop->csum_start,
3281                cop->csum_len);
3282 }
3283
3284 static void print_cryptocop_operation(struct cryptocop_operation *cop)
3285 {
3286         struct cryptocop_desc      *d;
3287         struct cryptocop_tfrm_cfg  *tc;
3288         struct cryptocop_desc_cfg  *dc;
3289         int                        i;
3290
3291         printk("print_cryptocop_operation, cop=0x%p\n\n", cop);
3292         printk("sid: %lld\n", cop->sid);
3293         printk("operation_status=%d\n"
3294                "use_dmalists=%d\n"
3295                "in_interrupt=%d\n"
3296                "fast_callback=%d\n",
3297                cop->operation_status,
3298                cop->use_dmalists,
3299                cop->in_interrupt,
3300                cop->fast_callback);
3301
3302         if (cop->use_dmalists){
3303                 print_user_dma_lists(&cop->list_op);
3304         } else {
3305                 printk("cop->tfrm_op\n"
3306                        "tfrm_cfg=0x%p\n"
3307                        "desc=0x%p\n"
3308                        "indata=0x%p\n"
3309                        "incount=%d\n"
3310                        "inlen=%d\n"
3311                        "outdata=0x%p\n"
3312                        "outcount=%d\n"
3313                        "outlen=%d\n\n",
3314                        cop->tfrm_op.tfrm_cfg,
3315                        cop->tfrm_op.desc,
3316                        cop->tfrm_op.indata,
3317                        cop->tfrm_op.incount,
3318                        cop->tfrm_op.inlen,
3319                        cop->tfrm_op.outdata,
3320                        cop->tfrm_op.outcount,
3321                        cop->tfrm_op.outlen);
3322
3323                 tc = cop->tfrm_op.tfrm_cfg;
3324                 while (tc){
3325                         printk("tfrm_cfg, 0x%p\n"
3326                                "tid=%d\n"
3327                                "flags=%d\n"
3328                                "inject_ix=%d\n"
3329                                "next=0x%p\n",
3330                                tc,
3331                                tc->tid,
3332                                tc->flags,
3333                                tc->inject_ix,
3334                                tc->next);
3335                         tc = tc->next;
3336                 }
3337                 d = cop->tfrm_op.desc;
3338                 while (d){
3339                         printk("\n======================desc, 0x%p\n"
3340                                "length=%d\n"
3341                                "cfg=0x%p\n"
3342                                "next=0x%p\n",
3343                                d,
3344                                d->length,
3345                                d->cfg,
3346                                d->next);
3347                         dc = d->cfg;
3348                         while (dc){
3349                                 printk("=========desc_cfg, 0x%p\n"
3350                                        "tid=%d\n"
3351                                        "src=%d\n"
3352                                        "last=%d\n"
3353                                        "next=0x%p\n",
3354                                        dc,
3355                                        dc->tid,
3356                                        dc->src,
3357                                        dc->last,
3358                                        dc->next);
3359                                 dc = dc->next;
3360                         }
3361                         d = d->next;
3362                 }
3363                 printk("\n====iniov\n");
3364                 for (i = 0; i < cop->tfrm_op.incount; i++){
3365                         printk("indata[%d]\n"
3366                                "base=0x%p\n"
3367                                "len=%d\n",
3368                                i,
3369                                cop->tfrm_op.indata[i].iov_base,
3370                                cop->tfrm_op.indata[i].iov_len);
3371                 }
3372                 printk("\n====outiov\n");
3373                 for (i = 0; i < cop->tfrm_op.outcount; i++){
3374                         printk("outdata[%d]\n"
3375                                "base=0x%p\n"
3376                                "len=%d\n",
3377                                i,
3378                                cop->tfrm_op.outdata[i].iov_base,
3379                                cop->tfrm_op.outdata[i].iov_len);
3380                 }
3381         }
3382         printk("------------end print_cryptocop_operation\n");
3383 }
3384
3385
3386 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op)
3387 {
3388         dma_descr_data *dd;
3389         int i;
3390
3391         printk("print_user_dma_lists, dma_op=0x%p\n", dma_op);
3392
3393         printk("out_data_buf = 0x%p, phys_to_virt(out_data_buf) = 0x%p\n", dma_op->out_data_buf, phys_to_virt((unsigned long int)dma_op->out_data_buf));
3394         printk("in_data_buf = 0x%p, phys_to_virt(in_data_buf) = 0x%p\n", dma_op->in_data_buf, phys_to_virt((unsigned long int)dma_op->in_data_buf));
3395
3396         printk("##############outlist\n");
3397         dd = phys_to_virt((unsigned long int)dma_op->outlist);
3398         i = 0;
3399         while (dd != NULL) {
3400                 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3401                 printk("\n\tbuf: 0x%p\n"
3402                        "\tafter: 0x%p\n"
3403                        "\tmd: 0x%04x\n"
3404                        "\tnext: 0x%p\n",
3405                        dd->buf,
3406                        dd->after,
3407                        dd->md,
3408                        dd->next);
3409                 printk("flags:\n"
3410                        "\twait:\t%d\n"
3411                        "\teol:\t%d\n"
3412                        "\touteop:\t%d\n"
3413                        "\tineop:\t%d\n"
3414                        "\tintr:\t%d\n",
3415                        dd->wait,
3416                        dd->eol,
3417                        dd->out_eop,
3418                        dd->in_eop,
3419                        dd->intr);
3420                 if (dd->eol)
3421                         dd = NULL;
3422                 else
3423                         dd = phys_to_virt((unsigned long int)dd->next);
3424                 ++i;
3425         }
3426
3427         printk("##############inlist\n");
3428         dd = phys_to_virt((unsigned long int)dma_op->inlist);
3429         i = 0;
3430         while (dd != NULL) {
3431                 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3432                 printk("\n\tbuf: 0x%p\n"
3433                        "\tafter: 0x%p\n"
3434                        "\tmd: 0x%04x\n"
3435                        "\tnext: 0x%p\n",
3436                        dd->buf,
3437                        dd->after,
3438                        dd->md,
3439                        dd->next);
3440                 printk("flags:\n"
3441                        "\twait:\t%d\n"
3442                        "\teol:\t%d\n"
3443                        "\touteop:\t%d\n"
3444                        "\tineop:\t%d\n"
3445                        "\tintr:\t%d\n",
3446                        dd->wait,
3447                        dd->eol,
3448                        dd->out_eop,
3449                        dd->in_eop,
3450                        dd->intr);
3451                 if (dd->eol)
3452                         dd = NULL;
3453                 else
3454                         dd = phys_to_virt((unsigned long int)dd->next);
3455                 ++i;
3456         }
3457 }
3458
3459
3460 static void print_lock_status(void)
3461 {
3462         printk("**********************print_lock_status\n");
3463         printk("cryptocop_completed_jobs_lock %d\n", spin_is_locked(&cryptocop_completed_jobs_lock));
3464         printk("cryptocop_job_queue_lock %d\n", spin_is_locked(&cryptocop_job_queue_lock));
3465         printk("descr_pool_lock %d\n", spin_is_locked(&descr_pool_lock));
3466         printk("cryptocop_sessions_lock %d\n", spin_is_locked(cryptocop_sessions_lock));
3467         printk("running_job_lock %d\n", spin_is_locked(running_job_lock));
3468         printk("cryptocop_process_lock %d\n", spin_is_locked(cryptocop_process_lock));
3469 }
3470 #endif /* LDEBUG */
3471
3472
3473 static const char cryptocop_name[] = "ETRAX FS stream co-processor";
3474
3475 static int init_stream_coprocessor(void)
3476 {
3477         int err;
3478         int i;
3479         static int initialized = 0;
3480
3481         if (initialized)
3482                 return 0;
3483
3484         initialized = 1;
3485
3486         printk("ETRAX FS stream co-processor driver v0.01, (c) 2003 Axis Communications AB\n");
3487
3488         err = register_chrdev(CRYPTOCOP_MAJOR, cryptocop_name, &cryptocop_fops);
3489         if (err < 0) {
3490                 printk(KERN_ERR "stream co-processor: could not get major number.\n");
3491                 return err;
3492         }
3493
3494         err = init_cryptocop();
3495         if (err) {
3496                 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3497                 return err;
3498         }
3499         err = cryptocop_job_queue_init();
3500         if (err) {
3501                 release_cryptocop();
3502                 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3503                 return err;
3504         }
3505         /* Init the descriptor pool. */
3506         for (i = 0; i < CRYPTOCOP_DESCRIPTOR_POOL_SIZE - 1; i++) {
3507                 descr_pool[i].from_pool = 1;
3508                 descr_pool[i].next = &descr_pool[i + 1];
3509         }
3510         descr_pool[i].from_pool = 1;
3511         descr_pool[i].next = NULL;
3512         descr_pool_free_list = &descr_pool[0];
3513         descr_pool_no_free = CRYPTOCOP_DESCRIPTOR_POOL_SIZE;
3514
3515         spin_lock_init(&cryptocop_completed_jobs_lock);
3516         spin_lock_init(&cryptocop_job_queue_lock);
3517         spin_lock_init(&descr_pool_lock);
3518         spin_lock_init(&cryptocop_sessions_lock);
3519         spin_lock_init(&running_job_lock);
3520         spin_lock_init(&cryptocop_process_lock);
3521
3522         cryptocop_sessions = NULL;
3523         next_sid = 1;
3524
3525         cryptocop_running_job = NULL;
3526
3527         printk("stream co-processor: init done.\n");
3528         return 0;
3529 }
3530
3531 static void __exit exit_stream_coprocessor(void)
3532 {
3533         release_cryptocop();
3534         cryptocop_job_queue_close();
3535 }
3536
3537 module_init(init_stream_coprocessor);
3538 module_exit(exit_stream_coprocessor);
3539