drivers/s390/crypto/ap_bus.c

   1 /*
   2  * linux/drivers/s390/crypto/ap_bus.c
   3  *
   4  * Copyright (C) 2006 IBM Corporation
   5  * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
   6  *            Martin Schwidefsky <schwidefsky@de.ibm.com>
   7  *            Ralph Wuerthner <rwuerthn@de.ibm.com>
   8  *            Felix Beck <felix.beck@de.ibm.com>
   9  *
  10  * Adjunct processor bus.
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License as published by
  14  * the Free Software Foundation; either version 2, or (at your option)
  15  * any later version.
  16  *
  17  * This program is distributed in the hope that it will be useful,
  18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  20  * GNU General Public License for more details.
  21  *
  22  * You should have received a copy of the GNU General Public License
  23  * along with this program; if not, write to the Free Software
  24  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  25  */
  26
  27 #define KMSG_COMPONENT "ap"
  28 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  29
  30 #include <linux/module.h>
  31 #include <linux/init.h>
  32 #include <linux/delay.h>
  33 #include <linux/err.h>
  34 #include <linux/interrupt.h>
  35 #include <linux/workqueue.h>
  36 #include <linux/notifier.h>
  37 #include <linux/kthread.h>
  38 #include <linux/mutex.h>
  39 #include <asm/reset.h>
  40 #include <asm/airq.h>
  41 #include <asm/atomic.h>
  42 #include <asm/system.h>
  43 #include <asm/isc.h>
  44 #include <linux/hrtimer.h>
  45 #include <linux/ktime.h>
  46
  47 #include "ap_bus.h"
  48
  49 /* Some prototypes. */
  50 static void ap_scan_bus(struct work_struct *);
  51 static void ap_poll_all(unsigned long);
  52 static enum hrtimer_restart ap_poll_timeout(struct hrtimer *);
  53 static int ap_poll_thread_start(void);
  54 static void ap_poll_thread_stop(void);
  55 static void ap_request_timeout(unsigned long);
  56 static inline void ap_schedule_poll_timer(void);
  57 static int __ap_poll_device(struct ap_device *ap_dev, unsigned long *flags);
  58 static int ap_device_remove(struct device *dev);
  59 static int ap_device_probe(struct device *dev);
  60 static void ap_interrupt_handler(void *unused1, void *unused2);
  61 static void ap_reset(struct ap_device *ap_dev);
  62 static void ap_config_timeout(unsigned long ptr);
  63
  64 /*
  65  * Module description.
  66  */
  67 MODULE_AUTHOR("IBM Corporation");
  68 MODULE_DESCRIPTION("Adjunct Processor Bus driver, "
  69                    "Copyright 2006 IBM Corporation");
  70 MODULE_LICENSE("GPL");
  71
  72 /*
  73  * Module parameter
  74  */
  75 int ap_domain_index = -1;       /* Adjunct Processor Domain Index */
  76 module_param_named(domain, ap_domain_index, int, 0000);
  77 MODULE_PARM_DESC(domain, "domain index for ap devices");
  78 EXPORT_SYMBOL(ap_domain_index);
  79
  80 static int ap_thread_flag = 0;
  81 module_param_named(poll_thread, ap_thread_flag, int, 0000);
  82 MODULE_PARM_DESC(poll_thread, "Turn on/off poll thread, default is 0 (off).");
  83
  84 static struct device *ap_root_device = NULL;
  85 static DEFINE_SPINLOCK(ap_device_list_lock);
  86 static LIST_HEAD(ap_device_list);
  87
  88 /*
  89  * Workqueue & timer for bus rescan.
  90  */
  91 static struct workqueue_struct *ap_work_queue;
  92 static struct timer_list ap_config_timer;
  93 static int ap_config_time = AP_CONFIG_TIME;
  94 static DECLARE_WORK(ap_config_work, ap_scan_bus);
  95
  96 /*
  97  * Tasklet & timer for AP request polling and interrupts
  98  */
  99 static DECLARE_TASKLET(ap_tasklet, ap_poll_all, 0);
 100 static atomic_t ap_poll_requests = ATOMIC_INIT(0);
 101 static DECLARE_WAIT_QUEUE_HEAD(ap_poll_wait);
 102 static struct task_struct *ap_poll_kthread = NULL;
 103 static DEFINE_MUTEX(ap_poll_thread_mutex);
 104 static void *ap_interrupt_indicator;
 105 static struct hrtimer ap_poll_timer;
 106 /* In LPAR poll with 4kHz frequency. Poll every 250000 nanoseconds.
 107  * If z/VM change to 1500000 nanoseconds to adjust to z/VM polling.*/
 108 static unsigned long long poll_timeout = 250000;
 109
 110 /* Suspend flag */
 111 static int ap_suspend_flag;
 112 static struct bus_type ap_bus_type;
 113
 114 /**
 115  * ap_using_interrupts() - Returns non-zero if interrupt support is
 116  * available.
 117  */
 118 static inline int ap_using_interrupts(void)
 119 {
 120         return ap_interrupt_indicator != NULL;
 121 }
 122
 123 /**
 124  * ap_intructions_available() - Test if AP instructions are available.
 125  *
 126  * Returns 0 if the AP instructions are installed.
 127  */
 128 static inline int ap_instructions_available(void)
 129 {
 130         register unsigned long reg0 asm ("0") = AP_MKQID(0,0);
 131         register unsigned long reg1 asm ("1") = -ENODEV;
 132         register unsigned long reg2 asm ("2") = 0UL;
 133
 134         asm volatile(
 135                 "   .long 0xb2af0000\n"         /* PQAP(TAPQ) */
 136                 "0: la    %1,0\n"
 137                 "1:\n"
 138                 EX_TABLE(0b, 1b)
 139                 : "+d" (reg0), "+d" (reg1), "+d" (reg2) : : "cc" );
 140         return reg1;
 141 }
 142
 143 /**
 144  * ap_interrupts_available(): Test if AP interrupts are available.
 145  *
 146  * Returns 1 if AP interrupts are available.
 147  */
 148 static int ap_interrupts_available(void)
 149 {
 150         unsigned long long facility_bits[2];
 151
 152         if (stfle(facility_bits, 2) <= 1)
 153                 return 0;
 154         if (!(facility_bits[0] & (1ULL << 61)) ||
 155             !(facility_bits[1] & (1ULL << 62)))
 156                 return 0;
 157         return 1;
 158 }
 159
 160 /**
 161  * ap_test_queue(): Test adjunct processor queue.
 162  * @qid: The AP queue number
 163  * @queue_depth: Pointer to queue depth value
 164  * @device_type: Pointer to device type value
 165  *
 166  * Returns AP queue status structure.
 167  */
 168 static inline struct ap_queue_status
 169 ap_test_queue(ap_qid_t qid, int *queue_depth, int *device_type)
 170 {
 171         register unsigned long reg0 asm ("0") = qid;
 172         register struct ap_queue_status reg1 asm ("1");
 173         register unsigned long reg2 asm ("2") = 0UL;
 174
 175         asm volatile(".long 0xb2af0000"         /* PQAP(TAPQ) */
 176                      : "+d" (reg0), "=d" (reg1), "+d" (reg2) : : "cc");
 177         *device_type = (int) (reg2 >> 24);
 178         *queue_depth = (int) (reg2 & 0xff);
 179         return reg1;
 180 }
 181
 182 /**
 183  * ap_reset_queue(): Reset adjunct processor queue.
 184  * @qid: The AP queue number
 185  *
 186  * Returns AP queue status structure.
 187  */
 188 static inline struct ap_queue_status ap_reset_queue(ap_qid_t qid)
 189 {
 190         register unsigned long reg0 asm ("0") = qid | 0x01000000UL;
 191         register struct ap_queue_status reg1 asm ("1");
 192         register unsigned long reg2 asm ("2") = 0UL;
 193
 194         asm volatile(
 195                 ".long 0xb2af0000"              /* PQAP(RAPQ) */
 196                 : "+d" (reg0), "=d" (reg1), "+d" (reg2) : : "cc");
 197         return reg1;
 198 }
 199
 200 #ifdef CONFIG_64BIT
 201 /**
 202  * ap_queue_interruption_control(): Enable interruption for a specific AP.
 203  * @qid: The AP queue number
 204  * @ind: The notification indicator byte
 205  *
 206  * Returns AP queue status.
 207  */
 208 static inline struct ap_queue_status
 209 ap_queue_interruption_control(ap_qid_t qid, void *ind)
 210 {
 211         register unsigned long reg0 asm ("0") = qid | 0x03000000UL;
 212         register unsigned long reg1_in asm ("1") = 0x0000800000000000UL | AP_ISC;
 213         register struct ap_queue_status reg1_out asm ("1");
 214         register void *reg2 asm ("2") = ind;
 215         asm volatile(
 216                 ".long 0xb2af0000"              /* PQAP(RAPQ) */
 217                 : "+d" (reg0), "+d" (reg1_in), "=d" (reg1_out), "+d" (reg2)
 218                 :
 219                 : "cc" );
 220         return reg1_out;
 221 }
 222 #endif
 223
 224 /**
 225  * ap_queue_enable_interruption(): Enable interruption on an AP.
 226  * @qid: The AP queue number
 227  * @ind: the notification indicator byte
 228  *
 229  * Enables interruption on AP queue via ap_queue_interruption_control(). Based
 230  * on the return value it waits a while and tests the AP queue if interrupts
 231  * have been switched on using ap_test_queue().
 232  */
 233 static int ap_queue_enable_interruption(ap_qid_t qid, void *ind)
 234 {
 235 #ifdef CONFIG_64BIT
 236         struct ap_queue_status status;
 237         int t_depth, t_device_type, rc, i;
 238
 239         rc = -EBUSY;
 240         status = ap_queue_interruption_control(qid, ind);
 241
 242         for (i = 0; i < AP_MAX_RESET; i++) {
 243                 switch (status.response_code) {
 244                 case AP_RESPONSE_NORMAL:
 245                         if (status.int_enabled)
 246                                 return 0;
 247                         break;
 248                 case AP_RESPONSE_RESET_IN_PROGRESS:
 249                 case AP_RESPONSE_BUSY:
 250                         break;
 251                 case AP_RESPONSE_Q_NOT_AVAIL:
 252                 case AP_RESPONSE_DECONFIGURED:
 253                 case AP_RESPONSE_CHECKSTOPPED:
 254                 case AP_RESPONSE_INVALID_ADDRESS:
 255                         return -ENODEV;
 256                 case AP_RESPONSE_OTHERWISE_CHANGED:
 257                         if (status.int_enabled)
 258                                 return 0;
 259                         break;
 260                 default:
 261                         break;
 262                 }
 263                 if (i < AP_MAX_RESET - 1) {
 264                         udelay(5);
 265                         status = ap_test_queue(qid, &t_depth, &t_device_type);
 266                 }
 267         }
 268         return rc;
 269 #else
 270         return -EINVAL;
 271 #endif
 272 }
 273
 274 /**
 275  * __ap_send(): Send message to adjunct processor queue.
 276  * @qid: The AP queue number
 277  * @psmid: The program supplied message identifier
 278  * @msg: The message text
 279  * @length: The message length
 280  *
 281  * Returns AP queue status structure.
 282  * Condition code 1 on NQAP can't happen because the L bit is 1.
 283  * Condition code 2 on NQAP also means the send is incomplete,
 284  * because a segment boundary was reached. The NQAP is repeated.
 285  */
 286 static inline struct ap_queue_status
 287 __ap_send(ap_qid_t qid, unsigned long long psmid, void *msg, size_t length)
 288 {
 289         typedef struct { char _[length]; } msgblock;
 290         register unsigned long reg0 asm ("0") = qid | 0x40000000UL;
 291         register struct ap_queue_status reg1 asm ("1");
 292         register unsigned long reg2 asm ("2") = (unsigned long) msg;
 293         register unsigned long reg3 asm ("3") = (unsigned long) length;
 294         register unsigned long reg4 asm ("4") = (unsigned int) (psmid >> 32);
 295         register unsigned long reg5 asm ("5") = (unsigned int) psmid;
 296
 297         asm volatile (
 298                 "0: .long 0xb2ad0042\n"         /* DQAP */
 299                 "   brc   2,0b"
 300                 : "+d" (reg0), "=d" (reg1), "+d" (reg2), "+d" (reg3)
 301                 : "d" (reg4), "d" (reg5), "m" (*(msgblock *) msg)
 302                 : "cc" );
 303         return reg1;
 304 }
 305
 306 int ap_send(ap_qid_t qid, unsigned long long psmid, void *msg, size_t length)
 307 {
 308         struct ap_queue_status status;
 309
 310         status = __ap_send(qid, psmid, msg, length);
 311         switch (status.response_code) {
 312         case AP_RESPONSE_NORMAL:
 313                 return 0;
 314         case AP_RESPONSE_Q_FULL:
 315         case AP_RESPONSE_RESET_IN_PROGRESS:
 316                 return -EBUSY;
 317         default:        /* Device is gone. */
 318                 return -ENODEV;
 319         }
 320 }
 321 EXPORT_SYMBOL(ap_send);
 322
 323 /**
 324  * __ap_recv(): Receive message from adjunct processor queue.
 325  * @qid: The AP queue number
 326  * @psmid: Pointer to program supplied message identifier
 327  * @msg: The message text
 328  * @length: The message length
 329  *
 330  * Returns AP queue status structure.
 331  * Condition code 1 on DQAP means the receive has taken place
 332  * but only partially.  The response is incomplete, hence the
 333  * DQAP is repeated.
 334  * Condition code 2 on DQAP also means the receive is incomplete,
 335  * this time because a segment boundary was reached. Again, the
 336  * DQAP is repeated.
 337  * Note that gpr2 is used by the DQAP instruction to keep track of
 338  * any 'residual' length, in case the instruction gets interrupted.
 339  * Hence it gets zeroed before the instruction.
 340  */
 341 static inline struct ap_queue_status
 342 __ap_recv(ap_qid_t qid, unsigned long long *psmid, void *msg, size_t length)
 343 {
 344         typedef struct { char _[length]; } msgblock;
 345         register unsigned long reg0 asm("0") = qid | 0x80000000UL;
 346         register struct ap_queue_status reg1 asm ("1");
 347         register unsigned long reg2 asm("2") = 0UL;
 348         register unsigned long reg4 asm("4") = (unsigned long) msg;
 349         register unsigned long reg5 asm("5") = (unsigned long) length;
 350         register unsigned long reg6 asm("6") = 0UL;
 351         register unsigned long reg7 asm("7") = 0UL;
 352
 353
 354         asm volatile(
 355                 "0: .long 0xb2ae0064\n"
 356                 "   brc   6,0b\n"
 357                 : "+d" (reg0), "=d" (reg1), "+d" (reg2),
 358                 "+d" (reg4), "+d" (reg5), "+d" (reg6), "+d" (reg7),
 359                 "=m" (*(msgblock *) msg) : : "cc" );
 360         *psmid = (((unsigned long long) reg6) << 32) + reg7;
 361         return reg1;
 362 }
 363
 364 int ap_recv(ap_qid_t qid, unsigned long long *psmid, void *msg, size_t length)
 365 {
 366         struct ap_queue_status status;
 367
 368         status = __ap_recv(qid, psmid, msg, length);
 369         switch (status.response_code) {
 370         case AP_RESPONSE_NORMAL:
 371                 return 0;
 372         case AP_RESPONSE_NO_PENDING_REPLY:
 373                 if (status.queue_empty)
 374                         return -ENOENT;
 375                 return -EBUSY;
 376         case AP_RESPONSE_RESET_IN_PROGRESS:
 377                 return -EBUSY;
 378         default:
 379                 return -ENODEV;
 380         }
 381 }
 382 EXPORT_SYMBOL(ap_recv);
 383
 384 /**
 385  * ap_query_queue(): Check if an AP queue is available.
 386  * @qid: The AP queue number
 387  * @queue_depth: Pointer to queue depth value
 388  * @device_type: Pointer to device type value
 389  *
 390  * The test is repeated for AP_MAX_RESET times.
 391  */
 392 static int ap_query_queue(ap_qid_t qid, int *queue_depth, int *device_type)
 393 {
 394         struct ap_queue_status status;
 395         int t_depth, t_device_type, rc, i;
 396
 397         rc = -EBUSY;
 398         for (i = 0; i < AP_MAX_RESET; i++) {
 399                 status = ap_test_queue(qid, &t_depth, &t_device_type);
 400                 switch (status.response_code) {
 401                 case AP_RESPONSE_NORMAL:
 402                         *queue_depth = t_depth + 1;
 403                         *device_type = t_device_type;
 404                         rc = 0;
 405                         break;
 406                 case AP_RESPONSE_Q_NOT_AVAIL:
 407                         rc = -ENODEV;
 408                         break;
 409                 case AP_RESPONSE_RESET_IN_PROGRESS:
 410                         break;
 411                 case AP_RESPONSE_DECONFIGURED:
 412                         rc = -ENODEV;
 413                         break;
 414                 case AP_RESPONSE_CHECKSTOPPED:
 415                         rc = -ENODEV;
 416                         break;
 417                 case AP_RESPONSE_INVALID_ADDRESS:
 418                         rc = -ENODEV;
 419                         break;
 420                 case AP_RESPONSE_OTHERWISE_CHANGED:
 421                         break;
 422                 case AP_RESPONSE_BUSY:
 423                         break;
 424                 default:
 425                         BUG();
 426                 }
 427                 if (rc != -EBUSY)
 428                         break;
 429                 if (i < AP_MAX_RESET - 1)
 430                         udelay(5);
 431         }
 432         return rc;
 433 }
 434
 435 /**
 436  * ap_init_queue(): Reset an AP queue.
 437  * @qid: The AP queue number
 438  *
 439  * Reset an AP queue and wait for it to become available again.
 440  */
 441 static int ap_init_queue(ap_qid_t qid)
 442 {
 443         struct ap_queue_status status;
 444         int rc, dummy, i;
 445
 446         rc = -ENODEV;
 447         status = ap_reset_queue(qid);
 448         for (i = 0; i < AP_MAX_RESET; i++) {
 449                 switch (status.response_code) {
 450                 case AP_RESPONSE_NORMAL:
 451                         if (status.queue_empty)
 452                                 rc = 0;
 453                         break;
 454                 case AP_RESPONSE_Q_NOT_AVAIL:
 455                 case AP_RESPONSE_DECONFIGURED:
 456                 case AP_RESPONSE_CHECKSTOPPED:
 457                         i = AP_MAX_RESET;       /* return with -ENODEV */
 458                         break;
 459                 case AP_RESPONSE_RESET_IN_PROGRESS:
 460                         rc = -EBUSY;
 461                 case AP_RESPONSE_BUSY:
 462                 default:
 463                         break;
 464                 }
 465                 if (rc != -ENODEV && rc != -EBUSY)
 466                         break;
 467                 if (i < AP_MAX_RESET - 1) {
 468                         udelay(5);
 469                         status = ap_test_queue(qid, &dummy, &dummy);
 470                 }
 471         }
 472         if (rc == 0 && ap_using_interrupts()) {
 473                 rc = ap_queue_enable_interruption(qid, ap_interrupt_indicator);
 474                 /* If interruption mode is supported by the machine,
 475                 * but an AP can not be enabled for interruption then
 476                 * the AP will be discarded.    */
 477                 if (rc)
 478                         pr_err("Registering adapter interrupts for "
 479                                "AP %d failed\n", AP_QID_DEVICE(qid));
 480         }
 481         return rc;
 482 }
 483
 484 /**
 485  * ap_increase_queue_count(): Arm request timeout.
 486  * @ap_dev: Pointer to an AP device.
 487  *
 488  * Arm request timeout if an AP device was idle and a new request is submitted.
 489  */
 490 static void ap_increase_queue_count(struct ap_device *ap_dev)
 491 {
 492         int timeout = ap_dev->drv->request_timeout;
 493
 494         ap_dev->queue_count++;
 495         if (ap_dev->queue_count == 1) {
 496                 mod_timer(&ap_dev->timeout, jiffies + timeout);
 497                 ap_dev->reset = AP_RESET_ARMED;
 498         }
 499 }
 500
 501 /**
 502  * ap_decrease_queue_count(): Decrease queue count.
 503  * @ap_dev: Pointer to an AP device.
 504  *
 505  * If AP device is still alive, re-schedule request timeout if there are still
 506  * pending requests.
 507  */
 508 static void ap_decrease_queue_count(struct ap_device *ap_dev)
 509 {
 510         int timeout = ap_dev->drv->request_timeout;
 511
 512         ap_dev->queue_count--;
 513         if (ap_dev->queue_count > 0)
 514                 mod_timer(&ap_dev->timeout, jiffies + timeout);
 515         else
 516                 /*
 517                  * The timeout timer should to be disabled now - since
 518                  * del_timer_sync() is very expensive, we just tell via the
 519                  * reset flag to ignore the pending timeout timer.
 520                  */
 521                 ap_dev->reset = AP_RESET_IGNORE;
 522 }
 523
 524 /*
 525  * AP device related attributes.
 526  */
 527 static ssize_t ap_hwtype_show(struct device *dev,
 528                               struct device_attribute *attr, char *buf)
 529 {
 530         struct ap_device *ap_dev = to_ap_dev(dev);
 531         return snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->device_type);
 532 }
 533
 534 static DEVICE_ATTR(hwtype, 0444, ap_hwtype_show, NULL);
 535 static ssize_t ap_depth_show(struct device *dev, struct device_attribute *attr,
 536                              char *buf)
 537 {
 538         struct ap_device *ap_dev = to_ap_dev(dev);
 539         return snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->queue_depth);
 540 }
 541
 542 static DEVICE_ATTR(depth, 0444, ap_depth_show, NULL);
 543 static ssize_t ap_request_count_show(struct device *dev,
 544                                      struct device_attribute *attr,
 545                                      char *buf)
 546 {
 547         struct ap_device *ap_dev = to_ap_dev(dev);
 548         int rc;
 549
 550         spin_lock_bh(&ap_dev->lock);
 551         rc = snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->total_request_count);
 552         spin_unlock_bh(&ap_dev->lock);
 553         return rc;
 554 }
 555
 556 static DEVICE_ATTR(request_count, 0444, ap_request_count_show, NULL);
 557
 558 static ssize_t ap_modalias_show(struct device *dev,
 559                                 struct device_attribute *attr, char *buf)
 560 {
 561         return sprintf(buf, "ap:t%02X", to_ap_dev(dev)->device_type);
 562 }
 563
 564 static DEVICE_ATTR(modalias, 0444, ap_modalias_show, NULL);
 565
 566 static struct attribute *ap_dev_attrs[] = {
 567         &dev_attr_hwtype.attr,
 568         &dev_attr_depth.attr,
 569         &dev_attr_request_count.attr,
 570         &dev_attr_modalias.attr,
 571         NULL
 572 };
 573 static struct attribute_group ap_dev_attr_group = {
 574         .attrs = ap_dev_attrs
 575 };
 576
 577 /**
 578  * ap_bus_match()
 579  * @dev: Pointer to device
 580  * @drv: Pointer to device_driver
 581  *
 582  * AP bus driver registration/unregistration.
 583  */
 584 static int ap_bus_match(struct device *dev, struct device_driver *drv)
 585 {
 586         struct ap_device *ap_dev = to_ap_dev(dev);
 587         struct ap_driver *ap_drv = to_ap_drv(drv);
 588         struct ap_device_id *id;
 589
 590         /*
 591          * Compare device type of the device with the list of
 592          * supported types of the device_driver.
 593          */
 594         for (id = ap_drv->ids; id->match_flags; id++) {
 595                 if ((id->match_flags & AP_DEVICE_ID_MATCH_DEVICE_TYPE) &&
 596                     (id->dev_type != ap_dev->device_type))
 597                         continue;
 598                 return 1;
 599         }
 600         return 0;
 601 }
 602
 603 /**
 604  * ap_uevent(): Uevent function for AP devices.
 605  * @dev: Pointer to device
 606  * @env: Pointer to kobj_uevent_env
 607  *
 608  * It sets up a single environment variable DEV_TYPE which contains the
 609  * hardware device type.
 610  */
 611 static int ap_uevent (struct device *dev, struct kobj_uevent_env *env)
 612 {
 613         struct ap_device *ap_dev = to_ap_dev(dev);
 614         int retval = 0;
 615
 616         if (!ap_dev)
 617                 return -ENODEV;
 618
 619         /* Set up DEV_TYPE environment variable. */
 620         retval = add_uevent_var(env, "DEV_TYPE=%04X", ap_dev->device_type);
 621         if (retval)
 622                 return retval;
 623
 624         /* Add MODALIAS= */
 625         retval = add_uevent_var(env, "MODALIAS=ap:t%02X", ap_dev->device_type);
 626
 627         return retval;
 628 }
 629
 630 static int ap_bus_suspend(struct device *dev, pm_message_t state)
 631 {
 632         struct ap_device *ap_dev = to_ap_dev(dev);
 633         unsigned long flags;
 634
 635         if (!ap_suspend_flag) {
 636                 ap_suspend_flag = 1;
 637
 638                 /* Disable scanning for devices, thus we do not want to scan
 639                  * for them after removing.
 640                  */
 641                 del_timer_sync(&ap_config_timer);
 642                 if (ap_work_queue != NULL) {
 643                         destroy_workqueue(ap_work_queue);
 644                         ap_work_queue = NULL;
 645                 }
 646                 tasklet_disable(&ap_tasklet);
 647         }
 648         /* Poll on the device until all requests are finished. */
 649         do {
 650                 flags = 0;
 651                 __ap_poll_device(ap_dev, &flags);
 652         } while ((flags & 1) || (flags & 2));
 653
 654         ap_device_remove(dev);
 655         return 0;
 656 }
 657
 658 static int ap_bus_resume(struct device *dev)
 659 {
 660         int rc = 0;
 661         struct ap_device *ap_dev = to_ap_dev(dev);
 662
 663         if (ap_suspend_flag) {
 664                 ap_suspend_flag = 0;
 665                 if (!ap_interrupts_available())
 666                         ap_interrupt_indicator = NULL;
 667                 ap_device_probe(dev);
 668                 ap_reset(ap_dev);
 669                 setup_timer(&ap_dev->timeout, ap_request_timeout,
 670                             (unsigned long) ap_dev);
 671                 ap_scan_bus(NULL);
 672                 init_timer(&ap_config_timer);
 673                 ap_config_timer.function = ap_config_timeout;
 674                 ap_config_timer.data = 0;
 675                 ap_config_timer.expires = jiffies + ap_config_time * HZ;
 676                 add_timer(&ap_config_timer);
 677                 ap_work_queue = create_singlethread_workqueue("kapwork");
 678                 if (!ap_work_queue)
 679                         return -ENOMEM;
 680                 tasklet_enable(&ap_tasklet);
 681                 if (!ap_using_interrupts())
 682                         ap_schedule_poll_timer();
 683                 else
 684                         tasklet_schedule(&ap_tasklet);
 685                 if (ap_thread_flag)
 686                         rc = ap_poll_thread_start();
 687         } else {
 688                 ap_device_probe(dev);
 689                 ap_reset(ap_dev);
 690                 setup_timer(&ap_dev->timeout, ap_request_timeout,
 691                             (unsigned long) ap_dev);
 692         }
 693
 694         return rc;
 695 }
 696
 697 static struct bus_type ap_bus_type = {
 698         .name = "ap",
 699         .match = &ap_bus_match,
 700         .uevent = &ap_uevent,
 701         .suspend = ap_bus_suspend,
 702         .resume = ap_bus_resume
 703 };
 704
 705 static int ap_device_probe(struct device *dev)
 706 {
 707         struct ap_device *ap_dev = to_ap_dev(dev);
 708         struct ap_driver *ap_drv = to_ap_drv(dev->driver);
 709         int rc;
 710
 711         ap_dev->drv = ap_drv;
 712         rc = ap_drv->probe ? ap_drv->probe(ap_dev) : -ENODEV;
 713         if (!rc) {
 714                 spin_lock_bh(&ap_device_list_lock);
 715                 list_add(&ap_dev->list, &ap_device_list);
 716                 spin_unlock_bh(&ap_device_list_lock);
 717         }
 718         return rc;
 719 }
 720
 721 /**
 722  * __ap_flush_queue(): Flush requests.
 723  * @ap_dev: Pointer to the AP device
 724  *
 725  * Flush all requests from the request/pending queue of an AP device.
 726  */
 727 static void __ap_flush_queue(struct ap_device *ap_dev)
 728 {
 729         struct ap_message *ap_msg, *next;
 730
 731         list_for_each_entry_safe(ap_msg, next, &ap_dev->pendingq, list) {
 732                 list_del_init(&ap_msg->list);
 733                 ap_dev->pendingq_count--;
 734                 ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
 735         }
 736         list_for_each_entry_safe(ap_msg, next, &ap_dev->requestq, list) {
 737                 list_del_init(&ap_msg->list);
 738                 ap_dev->requestq_count--;
 739                 ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
 740         }
 741 }
 742
 743 void ap_flush_queue(struct ap_device *ap_dev)
 744 {
 745         spin_lock_bh(&ap_dev->lock);
 746         __ap_flush_queue(ap_dev);
 747         spin_unlock_bh(&ap_dev->lock);
 748 }
 749 EXPORT_SYMBOL(ap_flush_queue);
 750
 751 static int ap_device_remove(struct device *dev)
 752 {
 753         struct ap_device *ap_dev = to_ap_dev(dev);
 754         struct ap_driver *ap_drv = ap_dev->drv;
 755
 756         ap_flush_queue(ap_dev);
 757         del_timer_sync(&ap_dev->timeout);
 758         spin_lock_bh(&ap_device_list_lock);
 759         list_del_init(&ap_dev->list);
 760         spin_unlock_bh(&ap_device_list_lock);
 761         if (ap_drv->remove)
 762                 ap_drv->remove(ap_dev);
 763         spin_lock_bh(&ap_dev->lock);
 764         atomic_sub(ap_dev->queue_count, &ap_poll_requests);
 765         spin_unlock_bh(&ap_dev->lock);
 766         return 0;
 767 }
 768
 769 int ap_driver_register(struct ap_driver *ap_drv, struct module *owner,
 770                        char *name)
 771 {
 772         struct device_driver *drv = &ap_drv->driver;
 773
 774         drv->bus = &ap_bus_type;
 775         drv->probe = ap_device_probe;
 776         drv->remove = ap_device_remove;
 777         drv->owner = owner;
 778         drv->name = name;
 779         return driver_register(drv);
 780 }
 781 EXPORT_SYMBOL(ap_driver_register);
 782
 783 void ap_driver_unregister(struct ap_driver *ap_drv)
 784 {
 785         driver_unregister(&ap_drv->driver);
 786 }
 787 EXPORT_SYMBOL(ap_driver_unregister);
 788
 789 /*
 790  * AP bus attributes.
 791  */
 792 static ssize_t ap_domain_show(struct bus_type *bus, char *buf)
 793 {
 794         return snprintf(buf, PAGE_SIZE, "%d\n", ap_domain_index);
 795 }
 796
 797 static BUS_ATTR(ap_domain, 0444, ap_domain_show, NULL);
 798
 799 static ssize_t ap_config_time_show(struct bus_type *bus, char *buf)
 800 {
 801         return snprintf(buf, PAGE_SIZE, "%d\n", ap_config_time);
 802 }
 803
 804 static ssize_t ap_interrupts_show(struct bus_type *bus, char *buf)
 805 {
 806         return snprintf(buf, PAGE_SIZE, "%d\n",
 807                         ap_using_interrupts() ? 1 : 0);
 808 }
 809
 810 static BUS_ATTR(ap_interrupts, 0444, ap_interrupts_show, NULL);
 811
 812 static ssize_t ap_config_time_store(struct bus_type *bus,
 813                                     const char *buf, size_t count)
 814 {
 815         int time;
 816
 817         if (sscanf(buf, "%d\n", &time) != 1 || time < 5 || time > 120)
 818                 return -EINVAL;
 819         ap_config_time = time;
 820         if (!timer_pending(&ap_config_timer) ||
 821             !mod_timer(&ap_config_timer, jiffies + ap_config_time * HZ)) {
 822                 ap_config_timer.expires = jiffies + ap_config_time * HZ;
 823                 add_timer(&ap_config_timer);
 824         }
 825         return count;
 826 }
 827
 828 static BUS_ATTR(config_time, 0644, ap_config_time_show, ap_config_time_store);
 829
 830 static ssize_t ap_poll_thread_show(struct bus_type *bus, char *buf)
 831 {
 832         return snprintf(buf, PAGE_SIZE, "%d\n", ap_poll_kthread ? 1 : 0);
 833 }
 834
 835 static ssize_t ap_poll_thread_store(struct bus_type *bus,
 836                                     const char *buf, size_t count)
 837 {
 838         int flag, rc;
 839
 840         if (sscanf(buf, "%d\n", &flag) != 1)
 841                 return -EINVAL;
 842         if (flag) {
 843                 rc = ap_poll_thread_start();
 844                 if (rc)
 845                         return rc;
 846         }
 847         else
 848                 ap_poll_thread_stop();
 849         return count;
 850 }
 851
 852 static BUS_ATTR(poll_thread, 0644, ap_poll_thread_show, ap_poll_thread_store);
 853
 854 static ssize_t poll_timeout_show(struct bus_type *bus, char *buf)
 855 {
 856         return snprintf(buf, PAGE_SIZE, "%llu\n", poll_timeout);
 857 }
 858
 859 static ssize_t poll_timeout_store(struct bus_type *bus, const char *buf,
 860                                   size_t count)
 861 {
 862         unsigned long long time;
 863         ktime_t hr_time;
 864
 865         /* 120 seconds = maximum poll interval */
 866         if (sscanf(buf, "%llu\n", &time) != 1 || time < 1 ||
 867             time > 120000000000ULL)
 868                 return -EINVAL;
 869         poll_timeout = time;
 870         hr_time = ktime_set(0, poll_timeout);
 871
 872         if (!hrtimer_is_queued(&ap_poll_timer) ||
 873             !hrtimer_forward(&ap_poll_timer, hrtimer_get_expires(&ap_poll_timer), hr_time)) {
 874                 hrtimer_set_expires(&ap_poll_timer, hr_time);
 875                 hrtimer_start_expires(&ap_poll_timer, HRTIMER_MODE_ABS);
 876         }
 877         return count;
 878 }
 879
 880 static BUS_ATTR(poll_timeout, 0644, poll_timeout_show, poll_timeout_store);
 881
 882 static struct bus_attribute *const ap_bus_attrs[] = {
 883         &bus_attr_ap_domain,
 884         &bus_attr_config_time,
 885         &bus_attr_poll_thread,
 886         &bus_attr_ap_interrupts,
 887         &bus_attr_poll_timeout,
 888         NULL,
 889 };
 890
 891 /**
 892  * ap_select_domain(): Select an AP domain.
 893  *
 894  * Pick one of the 16 AP domains.
 895  */
 896 static int ap_select_domain(void)
 897 {
 898         int queue_depth, device_type, count, max_count, best_domain;
 899         int rc, i, j;
 900
 901         /*
 902          * We want to use a single domain. Either the one specified with
 903          * the "domain=" parameter or the domain with the maximum number
 904          * of devices.
 905          */
 906         if (ap_domain_index >= 0 && ap_domain_index < AP_DOMAINS)
 907                 /* Domain has already been selected. */
 908                 return 0;
 909         best_domain = -1;
 910         max_count = 0;
 911         for (i = 0; i < AP_DOMAINS; i++) {
 912                 count = 0;
 913                 for (j = 0; j < AP_DEVICES; j++) {
 914                         ap_qid_t qid = AP_MKQID(j, i);
 915                         rc = ap_query_queue(qid, &queue_depth, &device_type);
 916                         if (rc)
 917                                 continue;
 918                         count++;
 919                 }
 920                 if (count > max_count) {
 921                         max_count = count;
 922                         best_domain = i;
 923                 }
 924         }
 925         if (best_domain >= 0){
 926                 ap_domain_index = best_domain;
 927                 return 0;
 928         }
 929         return -ENODEV;
 930 }
 931
 932 /**
 933  * ap_probe_device_type(): Find the device type of an AP.
 934  * @ap_dev: pointer to the AP device.
 935  *
 936  * Find the device type if query queue returned a device type of 0.
 937  */
 938 static int ap_probe_device_type(struct ap_device *ap_dev)
 939 {
 940         static unsigned char msg[] = {
 941                 0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,
 942                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 943                 0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,
 944                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 945                 0x01,0x00,0x43,0x43,0x41,0x2d,0x41,0x50,
 946                 0x50,0x4c,0x20,0x20,0x20,0x01,0x01,0x01,
 947                 0x00,0x00,0x00,0x00,0x50,0x4b,0x00,0x00,
 948                 0x00,0x00,0x01,0x1c,0x00,0x00,0x00,0x00,
 949                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 950                 0x00,0x00,0x05,0xb8,0x00,0x00,0x00,0x00,
 951                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 952                 0x70,0x00,0x41,0x00,0x00,0x00,0x00,0x00,
 953                 0x00,0x00,0x54,0x32,0x01,0x00,0xa0,0x00,
 954                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 955                 0x00,0x00,0x00,0x00,0xb8,0x05,0x00,0x00,
 956                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 957                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 958                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 959                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 960                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 961                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 962                 0x00,0x00,0x0a,0x00,0x00,0x00,0x00,0x00,
 963                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 964                 0x00,0x00,0x00,0x00,0x00,0x00,0x08,0x00,
 965                 0x49,0x43,0x53,0x46,0x20,0x20,0x20,0x20,
 966                 0x50,0x4b,0x0a,0x00,0x50,0x4b,0x43,0x53,
 967                 0x2d,0x31,0x2e,0x32,0x37,0x00,0x11,0x22,
 968                 0x33,0x44,0x55,0x66,0x77,0x88,0x99,0x00,
 969                 0x11,0x22,0x33,0x44,0x55,0x66,0x77,0x88,
 970                 0x99,0x00,0x11,0x22,0x33,0x44,0x55,0x66,
 971                 0x77,0x88,0x99,0x00,0x11,0x22,0x33,0x44,
 972                 0x55,0x66,0x77,0x88,0x99,0x00,0x11,0x22,
 973                 0x33,0x44,0x55,0x66,0x77,0x88,0x99,0x00,
 974                 0x11,0x22,0x33,0x5d,0x00,0x5b,0x00,0x77,
 975                 0x88,0x1e,0x00,0x00,0x57,0x00,0x00,0x00,
 976                 0x00,0x04,0x00,0x00,0x4f,0x00,0x00,0x00,
 977                 0x03,0x02,0x00,0x00,0x40,0x01,0x00,0x01,
 978                 0xce,0x02,0x68,0x2d,0x5f,0xa9,0xde,0x0c,
 979                 0xf6,0xd2,0x7b,0x58,0x4b,0xf9,0x28,0x68,
 980                 0x3d,0xb4,0xf4,0xef,0x78,0xd5,0xbe,0x66,
 981                 0x63,0x42,0xef,0xf8,0xfd,0xa4,0xf8,0xb0,
 982                 0x8e,0x29,0xc2,0xc9,0x2e,0xd8,0x45,0xb8,
 983                 0x53,0x8c,0x6f,0x4e,0x72,0x8f,0x6c,0x04,
 984                 0x9c,0x88,0xfc,0x1e,0xc5,0x83,0x55,0x57,
 985                 0xf7,0xdd,0xfd,0x4f,0x11,0x36,0x95,0x5d,
 986         };
 987         struct ap_queue_status status;
 988         unsigned long long psmid;
 989         char *reply;
 990         int rc, i;
 991
 992         reply = (void *) get_zeroed_page(GFP_KERNEL);
 993         if (!reply) {
 994                 rc = -ENOMEM;
 995                 goto out;
 996         }
 997
 998         status = __ap_send(ap_dev->qid, 0x0102030405060708ULL,
 999                            msg, sizeof(msg));
1000         if (status.response_code != AP_RESPONSE_NORMAL) {
1001                 rc = -ENODEV;
1002                 goto out_free;
1003         }
1004
1005         /* Wait for the test message to complete. */
1006         for (i = 0; i < 6; i++) {
1007                 mdelay(300);
1008                 status = __ap_recv(ap_dev->qid, &psmid, reply, 4096);
1009                 if (status.response_code == AP_RESPONSE_NORMAL &&
1010                     psmid == 0x0102030405060708ULL)
1011                         break;
1012         }
1013         if (i < 6) {
1014                 /* Got an answer. */
1015                 if (reply[0] == 0x00 && reply[1] == 0x86)
1016                         ap_dev->device_type = AP_DEVICE_TYPE_PCICC;
1017                 else
1018                         ap_dev->device_type = AP_DEVICE_TYPE_PCICA;
1019                 rc = 0;
1020         } else
1021                 rc = -ENODEV;
1022
1023 out_free:
1024         free_page((unsigned long) reply);
1025 out:
1026         return rc;
1027 }
1028
1029 static void ap_interrupt_handler(void *unused1, void *unused2)
1030 {
1031         tasklet_schedule(&ap_tasklet);
1032 }
1033
1034 /**
1035  * __ap_scan_bus(): Scan the AP bus.
1036  * @dev: Pointer to device
1037  * @data: Pointer to data
1038  *
1039  * Scan the AP bus for new devices.
1040  */
1041 static int __ap_scan_bus(struct device *dev, void *data)
1042 {
1043         return to_ap_dev(dev)->qid == (ap_qid_t)(unsigned long) data;
1044 }
1045
1046 static void ap_device_release(struct device *dev)
1047 {
1048         struct ap_device *ap_dev = to_ap_dev(dev);
1049
1050         kfree(ap_dev);
1051 }
1052
1053 static void ap_scan_bus(struct work_struct *unused)
1054 {
1055         struct ap_device *ap_dev;
1056         struct device *dev;
1057         ap_qid_t qid;
1058         int queue_depth, device_type;
1059         int rc, i;
1060
1061         if (ap_select_domain() != 0)
1062                 return;
1063         for (i = 0; i < AP_DEVICES; i++) {
1064                 qid = AP_MKQID(i, ap_domain_index);
1065                 dev = bus_find_device(&ap_bus_type, NULL,
1066                                       (void *)(unsigned long)qid,
1067                                       __ap_scan_bus);
1068                 rc = ap_query_queue(qid, &queue_depth, &device_type);
1069                 if (dev) {
1070                         if (rc == -EBUSY) {
1071                                 set_current_state(TASK_UNINTERRUPTIBLE);
1072                                 schedule_timeout(AP_RESET_TIMEOUT);
1073                                 rc = ap_query_queue(qid, &queue_depth,
1074                                                     &device_type);
1075                         }
1076                         ap_dev = to_ap_dev(dev);
1077                         spin_lock_bh(&ap_dev->lock);
1078                         if (rc || ap_dev->unregistered) {
1079                                 spin_unlock_bh(&ap_dev->lock);
1080                                 device_unregister(dev);
1081                                 put_device(dev);
1082                                 continue;
1083                         }
1084                         spin_unlock_bh(&ap_dev->lock);
1085                         put_device(dev);
1086                         continue;
1087                 }
1088                 if (rc)
1089                         continue;
1090                 rc = ap_init_queue(qid);
1091                 if (rc)
1092                         continue;
1093                 ap_dev = kzalloc(sizeof(*ap_dev), GFP_KERNEL);
1094                 if (!ap_dev)
1095                         break;
1096                 ap_dev->qid = qid;
1097                 ap_dev->queue_depth = queue_depth;
1098                 ap_dev->unregistered = 1;
1099                 spin_lock_init(&ap_dev->lock);
1100                 INIT_LIST_HEAD(&ap_dev->pendingq);
1101                 INIT_LIST_HEAD(&ap_dev->requestq);
1102                 INIT_LIST_HEAD(&ap_dev->list);
1103                 setup_timer(&ap_dev->timeout, ap_request_timeout,
1104                             (unsigned long) ap_dev);
1105                 if (device_type == 0)
1106                         ap_probe_device_type(ap_dev);
1107                 else
1108                         ap_dev->device_type = device_type;
1109
1110                 ap_dev->device.bus = &ap_bus_type;
1111                 ap_dev->device.parent = ap_root_device;
1112                 dev_set_name(&ap_dev->device, "card%02x",
1113                              AP_QID_DEVICE(ap_dev->qid));
1114                 ap_dev->device.release = ap_device_release;
1115                 rc = device_register(&ap_dev->device);
1116                 if (rc) {
1117                         kfree(ap_dev);
1118                         continue;
1119                 }
1120                 /* Add device attributes. */
1121                 rc = sysfs_create_group(&ap_dev->device.kobj,
1122                                         &ap_dev_attr_group);
1123                 if (!rc) {
1124                         spin_lock_bh(&ap_dev->lock);
1125                         ap_dev->unregistered = 0;
1126                         spin_unlock_bh(&ap_dev->lock);
1127                 }
1128                 else
1129                         device_unregister(&ap_dev->device);
1130         }
1131 }
1132
1133 static void
1134 ap_config_timeout(unsigned long ptr)
1135 {
1136         queue_work(ap_work_queue, &ap_config_work);
1137         ap_config_timer.expires = jiffies + ap_config_time * HZ;
1138         add_timer(&ap_config_timer);
1139 }
1140
1141 /**
1142  * ap_schedule_poll_timer(): Schedule poll timer.
1143  *
1144  * Set up the timer to run the poll tasklet
1145  */
1146 static inline void ap_schedule_poll_timer(void)
1147 {
1148         if (ap_using_interrupts() || ap_suspend_flag)
1149                 return;
1150         if (hrtimer_is_queued(&ap_poll_timer))
1151                 return;
1152         hrtimer_start(&ap_poll_timer, ktime_set(0, poll_timeout),
1153                       HRTIMER_MODE_ABS);
1154 }
1155
1156 /**
1157  * ap_poll_read(): Receive pending reply messages from an AP device.
1158  * @ap_dev: pointer to the AP device
1159  * @flags: pointer to control flags, bit 2^0 is set if another poll is
1160  *         required, bit 2^1 is set if the poll timer needs to get armed
1161  *
1162  * Returns 0 if the device is still present, -ENODEV if not.
1163  */
1164 static int ap_poll_read(struct ap_device *ap_dev, unsigned long *flags)
1165 {
1166         struct ap_queue_status status;
1167         struct ap_message *ap_msg;
1168
1169         if (ap_dev->queue_count <= 0)
1170                 return 0;
1171         status = __ap_recv(ap_dev->qid, &ap_dev->reply->psmid,
1172                            ap_dev->reply->message, ap_dev->reply->length);
1173         switch (status.response_code) {
1174         case AP_RESPONSE_NORMAL:
1175                 atomic_dec(&ap_poll_requests);
1176                 ap_decrease_queue_count(ap_dev);
1177                 list_for_each_entry(ap_msg, &ap_dev->pendingq, list) {
1178                         if (ap_msg->psmid != ap_dev->reply->psmid)
1179                                 continue;
1180                         list_del_init(&ap_msg->list);
1181                         ap_dev->pendingq_count--;
1182                         ap_dev->drv->receive(ap_dev, ap_msg, ap_dev->reply);
1183                         break;
1184                 }
1185                 if (ap_dev->queue_count > 0)
1186                         *flags |= 1;
1187                 break;
1188         case AP_RESPONSE_NO_PENDING_REPLY:
1189                 if (status.queue_empty) {
1190                         /* The card shouldn't forget requests but who knows. */
1191                         atomic_sub(ap_dev->queue_count, &ap_poll_requests);
1192                         ap_dev->queue_count = 0;
1193                         list_splice_init(&ap_dev->pendingq, &ap_dev->requestq);
1194                         ap_dev->requestq_count += ap_dev->pendingq_count;
1195                         ap_dev->pendingq_count = 0;
1196                 } else
1197                         *flags |= 2;
1198                 break;
1199         default:
1200                 return -ENODEV;
1201         }
1202         return 0;
1203 }
1204
1205 /**
1206  * ap_poll_write(): Send messages from the request queue to an AP device.
1207  * @ap_dev: pointer to the AP device
1208  * @flags: pointer to control flags, bit 2^0 is set if another poll is
1209  *         required, bit 2^1 is set if the poll timer needs to get armed
1210  *
1211  * Returns 0 if the device is still present, -ENODEV if not.
1212  */
1213 static int ap_poll_write(struct ap_device *ap_dev, unsigned long *flags)
1214 {
1215         struct ap_queue_status status;
1216         struct ap_message *ap_msg;
1217
1218         if (ap_dev->requestq_count <= 0 ||
1219             ap_dev->queue_count >= ap_dev->queue_depth)
1220                 return 0;
1221         /* Start the next request on the queue. */
1222         ap_msg = list_entry(ap_dev->requestq.next, struct ap_message, list);
1223         status = __ap_send(ap_dev->qid, ap_msg->psmid,
1224                            ap_msg->message, ap_msg->length);
1225         switch (status.response_code) {
1226         case AP_RESPONSE_NORMAL:
1227                 atomic_inc(&ap_poll_requests);
1228                 ap_increase_queue_count(ap_dev);
1229                 list_move_tail(&ap_msg->list, &ap_dev->pendingq);
1230                 ap_dev->requestq_count--;
1231                 ap_dev->pendingq_count++;
1232                 if (ap_dev->queue_count < ap_dev->queue_depth &&
1233                     ap_dev->requestq_count > 0)
1234                         *flags |= 1;
1235                 *flags |= 2;
1236                 break;
1237         case AP_RESPONSE_Q_FULL:
1238         case AP_RESPONSE_RESET_IN_PROGRESS:
1239                 *flags |= 2;
1240                 break;
1241         case AP_RESPONSE_MESSAGE_TOO_BIG:
1242                 return -EINVAL;
1243         default:
1244                 return -ENODEV;
1245         }
1246         return 0;
1247 }
1248
1249 /**
1250  * ap_poll_queue(): Poll AP device for pending replies and send new messages.
1251  * @ap_dev: pointer to the bus device
1252  * @flags: pointer to control flags, bit 2^0 is set if another poll is
1253  *         required, bit 2^1 is set if the poll timer needs to get armed
1254  *
1255  * Poll AP device for pending replies and send new messages. If either
1256  * ap_poll_read or ap_poll_write returns -ENODEV unregister the device.
1257  * Returns 0.
1258  */
1259 static inline int ap_poll_queue(struct ap_device *ap_dev, unsigned long *flags)
1260 {
1261         int rc;
1262
1263         rc = ap_poll_read(ap_dev, flags);
1264         if (rc)
1265                 return rc;
1266         return ap_poll_write(ap_dev, flags);
1267 }
1268
1269 /**
1270  * __ap_queue_message(): Queue a message to a device.
1271  * @ap_dev: pointer to the AP device
1272  * @ap_msg: the message to be queued
1273  *
1274  * Queue a message to a device. Returns 0 if successful.
1275  */
1276 static int __ap_queue_message(struct ap_device *ap_dev, struct ap_message *ap_msg)
1277 {
1278         struct ap_queue_status status;
1279
1280         if (list_empty(&ap_dev->requestq) &&
1281             ap_dev->queue_count < ap_dev->queue_depth) {
1282                 status = __ap_send(ap_dev->qid, ap_msg->psmid,
1283                                    ap_msg->message, ap_msg->length);
1284                 switch (status.response_code) {
1285                 case AP_RESPONSE_NORMAL:
1286                         list_add_tail(&ap_msg->list, &ap_dev->pendingq);
1287                         atomic_inc(&ap_poll_requests);
1288                         ap_dev->pendingq_count++;
1289                         ap_increase_queue_count(ap_dev);
1290                         ap_dev->total_request_count++;
1291                         break;
1292                 case AP_RESPONSE_Q_FULL:
1293                 case AP_RESPONSE_RESET_IN_PROGRESS:
1294                         list_add_tail(&ap_msg->list, &ap_dev->requestq);
1295                         ap_dev->requestq_count++;
1296                         ap_dev->total_request_count++;
1297                         return -EBUSY;
1298                 case AP_RESPONSE_MESSAGE_TOO_BIG:
1299                         ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-EINVAL));
1300                         return -EINVAL;
1301                 default:        /* Device is gone. */
1302                         ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
1303                         return -ENODEV;
1304                 }
1305         } else {
1306                 list_add_tail(&ap_msg->list, &ap_dev->requestq);
1307                 ap_dev->requestq_count++;
1308                 ap_dev->total_request_count++;
1309                 return -EBUSY;
1310         }
1311         ap_schedule_poll_timer();
1312         return 0;
1313 }
1314
1315 void ap_queue_message(struct ap_device *ap_dev, struct ap_message *ap_msg)
1316 {
1317         unsigned long flags;
1318         int rc;
1319
1320         spin_lock_bh(&ap_dev->lock);
1321         if (!ap_dev->unregistered) {
1322                 /* Make room on the queue by polling for finished requests. */
1323                 rc = ap_poll_queue(ap_dev, &flags);
1324                 if (!rc)
1325                         rc = __ap_queue_message(ap_dev, ap_msg);
1326                 if (!rc)
1327                         wake_up(&ap_poll_wait);
1328                 if (rc == -ENODEV)
1329                         ap_dev->unregistered = 1;
1330         } else {
1331                 ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
1332                 rc = -ENODEV;
1333         }
1334         spin_unlock_bh(&ap_dev->lock);
1335         if (rc == -ENODEV)
1336                 device_unregister(&ap_dev->device);
1337 }
1338 EXPORT_SYMBOL(ap_queue_message);
1339
1340 /**
1341  * ap_cancel_message(): Cancel a crypto request.
1342  * @ap_dev: The AP device that has the message queued
1343  * @ap_msg: The message that is to be removed
1344  *
1345  * Cancel a crypto request. This is done by removing the request
1346  * from the device pending or request queue. Note that the
1347  * request stays on the AP queue. When it finishes the message
1348  * reply will be discarded because the psmid can't be found.
1349  */
1350 void ap_cancel_message(struct ap_device *ap_dev, struct ap_message *ap_msg)
1351 {
1352         struct ap_message *tmp;
1353
1354         spin_lock_bh(&ap_dev->lock);
1355         if (!list_empty(&ap_msg->list)) {
1356                 list_for_each_entry(tmp, &ap_dev->pendingq, list)
1357                         if (tmp->psmid == ap_msg->psmid) {
1358                                 ap_dev->pendingq_count--;
1359                                 goto found;
1360                         }
1361                 ap_dev->requestq_count--;
1362         found:
1363                 list_del_init(&ap_msg->list);
1364         }
1365         spin_unlock_bh(&ap_dev->lock);
1366 }
1367 EXPORT_SYMBOL(ap_cancel_message);
1368
1369 /**
1370  * ap_poll_timeout(): AP receive polling for finished AP requests.
1371  * @unused: Unused pointer.
1372  *
1373  * Schedules the AP tasklet using a high resolution timer.
1374  */
1375 static enum hrtimer_restart ap_poll_timeout(struct hrtimer *unused)
1376 {
1377         tasklet_schedule(&ap_tasklet);
1378         return HRTIMER_NORESTART;
1379 }
1380
1381 /**
1382  * ap_reset(): Reset a not responding AP device.
1383  * @ap_dev: Pointer to the AP device
1384  *
1385  * Reset a not responding AP device and move all requests from the
1386  * pending queue to the request queue.
1387  */
1388 static void ap_reset(struct ap_device *ap_dev)
1389 {
1390         int rc;
1391
1392         ap_dev->reset = AP_RESET_IGNORE;
1393         atomic_sub(ap_dev->queue_count, &ap_poll_requests);
1394         ap_dev->queue_count = 0;
1395         list_splice_init(&ap_dev->pendingq, &ap_dev->requestq);
1396         ap_dev->requestq_count += ap_dev->pendingq_count;
1397         ap_dev->pendingq_count = 0;
1398         rc = ap_init_queue(ap_dev->qid);
1399         if (rc == -ENODEV)
1400                 ap_dev->unregistered = 1;
1401 }
1402
1403 static int __ap_poll_device(struct ap_device *ap_dev, unsigned long *flags)
1404 {
1405         spin_lock(&ap_dev->lock);
1406         if (!ap_dev->unregistered) {
1407                 if (ap_poll_queue(ap_dev, flags))
1408                         ap_dev->unregistered = 1;
1409                 if (ap_dev->reset == AP_RESET_DO)
1410                         ap_reset(ap_dev);
1411         }
1412         spin_unlock(&ap_dev->lock);
1413         return 0;
1414 }
1415
1416 /**
1417  * ap_poll_all(): Poll all AP devices.
1418  * @dummy: Unused variable
1419  *
1420  * Poll all AP devices on the bus in a round robin fashion. Continue
1421  * polling until bit 2^0 of the control flags is not set. If bit 2^1
1422  * of the control flags has been set arm the poll timer.
1423  */
1424 static void ap_poll_all(unsigned long dummy)
1425 {
1426         unsigned long flags;
1427         struct ap_device *ap_dev;
1428
1429         /* Reset the indicator if interrupts are used. Thus new interrupts can
1430          * be received. Doing it in the beginning of the tasklet is therefor
1431          * important that no requests on any AP get lost.
1432          */
1433         if (ap_using_interrupts())
1434                 xchg((u8 *)ap_interrupt_indicator, 0);
1435         do {
1436                 flags = 0;
1437                 spin_lock(&ap_device_list_lock);
1438                 list_for_each_entry(ap_dev, &ap_device_list, list) {
1439                         __ap_poll_device(ap_dev, &flags);
1440                 }
1441                 spin_unlock(&ap_device_list_lock);
1442         } while (flags & 1);
1443         if (flags & 2)
1444                 ap_schedule_poll_timer();
1445 }
1446
1447 /**
1448  * ap_poll_thread(): Thread that polls for finished requests.
1449  * @data: Unused pointer
1450  *
1451  * AP bus poll thread. The purpose of this thread is to poll for
1452  * finished requests in a loop if there is a "free" cpu - that is
1453  * a cpu that doesn't have anything better to do. The polling stops
1454  * as soon as there is another task or if all messages have been
1455  * delivered.
1456  */
1457 static int ap_poll_thread(void *data)
1458 {
1459         DECLARE_WAITQUEUE(wait, current);
1460         unsigned long flags;
1461         int requests;
1462         struct ap_device *ap_dev;
1463
1464         set_user_nice(current, 19);
1465         while (1) {
1466                 if (ap_suspend_flag)
1467                         return 0;
1468                 if (need_resched()) {
1469                         schedule();
1470                         continue;
1471                 }
1472                 add_wait_queue(&ap_poll_wait, &wait);
1473                 set_current_state(TASK_INTERRUPTIBLE);
1474                 if (kthread_should_stop())
1475                         break;
1476                 requests = atomic_read(&ap_poll_requests);
1477                 if (requests <= 0)
1478                         schedule();
1479                 set_current_state(TASK_RUNNING);
1480                 remove_wait_queue(&ap_poll_wait, &wait);
1481
1482                 flags = 0;
1483                 spin_lock_bh(&ap_device_list_lock);
1484                 list_for_each_entry(ap_dev, &ap_device_list, list) {
1485                         __ap_poll_device(ap_dev, &flags);
1486                 }
1487                 spin_unlock_bh(&ap_device_list_lock);
1488         }
1489         set_current_state(TASK_RUNNING);
1490         remove_wait_queue(&ap_poll_wait, &wait);
1491         return 0;
1492 }
1493
1494 static int ap_poll_thread_start(void)
1495 {
1496         int rc;
1497
1498         if (ap_using_interrupts() || ap_suspend_flag)
1499                 return 0;
1500         mutex_lock(&ap_poll_thread_mutex);
1501         if (!ap_poll_kthread) {
1502                 ap_poll_kthread = kthread_run(ap_poll_thread, NULL, "appoll");
1503                 rc = IS_ERR(ap_poll_kthread) ? PTR_ERR(ap_poll_kthread) : 0;
1504                 if (rc)
1505                         ap_poll_kthread = NULL;
1506         }
1507         else
1508                 rc = 0;
1509         mutex_unlock(&ap_poll_thread_mutex);
1510         return rc;
1511 }
1512
1513 static void ap_poll_thread_stop(void)
1514 {
1515         mutex_lock(&ap_poll_thread_mutex);
1516         if (ap_poll_kthread) {
1517                 kthread_stop(ap_poll_kthread);
1518                 ap_poll_kthread = NULL;
1519         }
1520         mutex_unlock(&ap_poll_thread_mutex);
1521 }
1522
1523 /**
1524  * ap_request_timeout(): Handling of request timeouts
1525  * @data: Holds the AP device.
1526  *
1527  * Handles request timeouts.
1528  */
1529 static void ap_request_timeout(unsigned long data)
1530 {
1531         struct ap_device *ap_dev = (struct ap_device *) data;
1532
1533         if (ap_dev->reset == AP_RESET_ARMED) {
1534                 ap_dev->reset = AP_RESET_DO;
1535
1536                 if (ap_using_interrupts())
1537                         tasklet_schedule(&ap_tasklet);
1538         }
1539 }
1540
1541 static void ap_reset_domain(void)
1542 {
1543         int i;
1544
1545         if (ap_domain_index != -1)
1546                 for (i = 0; i < AP_DEVICES; i++)
1547                         ap_reset_queue(AP_MKQID(i, ap_domain_index));
1548 }
1549
1550 static void ap_reset_all(void)
1551 {
1552         int i, j;
1553
1554         for (i = 0; i < AP_DOMAINS; i++)
1555                 for (j = 0; j < AP_DEVICES; j++)
1556                         ap_reset_queue(AP_MKQID(j, i));
1557 }
1558
1559 static struct reset_call ap_reset_call = {
1560         .fn = ap_reset_all,
1561 };
1562
1563 /**
1564  * ap_module_init(): The module initialization code.
1565  *
1566  * Initializes the module.
1567  */
1568 int __init ap_module_init(void)
1569 {
1570         int rc, i;
1571
1572         if (ap_domain_index < -1 || ap_domain_index >= AP_DOMAINS) {
1573                 pr_warning("%d is not a valid cryptographic domain\n",
1574                            ap_domain_index);
1575                 return -EINVAL;
1576         }
1577         if (ap_instructions_available() != 0) {
1578                 pr_warning("The hardware system does not support "
1579                            "AP instructions\n");
1580                 return -ENODEV;
1581         }
1582         if (ap_interrupts_available()) {
1583                 isc_register(AP_ISC);
1584                 ap_interrupt_indicator = s390_register_adapter_interrupt(
1585                         &ap_interrupt_handler, NULL, AP_ISC);
1586                 if (IS_ERR(ap_interrupt_indicator)) {
1587                         ap_interrupt_indicator = NULL;
1588                         isc_unregister(AP_ISC);
1589                 }
1590         }
1591
1592         register_reset_call(&ap_reset_call);
1593
1594         /* Create /sys/bus/ap. */
1595         rc = bus_register(&ap_bus_type);
1596         if (rc)
1597                 goto out;
1598         for (i = 0; ap_bus_attrs[i]; i++) {
1599                 rc = bus_create_file(&ap_bus_type, ap_bus_attrs[i]);
1600                 if (rc)
1601                         goto out_bus;
1602         }
1603
1604         /* Create /sys/devices/ap. */
1605         ap_root_device = root_device_register("ap");
1606         rc = IS_ERR(ap_root_device) ? PTR_ERR(ap_root_device) : 0;
1607         if (rc)
1608                 goto out_bus;
1609
1610         ap_work_queue = create_singlethread_workqueue("kapwork");
1611         if (!ap_work_queue) {
1612                 rc = -ENOMEM;
1613                 goto out_root;
1614         }
1615
1616         if (ap_select_domain() == 0)
1617                 ap_scan_bus(NULL);
1618
1619         /* Setup the AP bus rescan timer. */
1620         init_timer(&ap_config_timer);
1621         ap_config_timer.function = ap_config_timeout;
1622         ap_config_timer.data = 0;
1623         ap_config_timer.expires = jiffies + ap_config_time * HZ;
1624         add_timer(&ap_config_timer);
1625
1626         /* Setup the high resultion poll timer.
1627          * If we are running under z/VM adjust polling to z/VM polling rate.
1628          */
1629         if (MACHINE_IS_VM)
1630                 poll_timeout = 1500000;
1631         hrtimer_init(&ap_poll_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1632         ap_poll_timer.function = ap_poll_timeout;
1633
1634         /* Start the low priority AP bus poll thread. */
1635         if (ap_thread_flag) {
1636                 rc = ap_poll_thread_start();
1637                 if (rc)
1638                         goto out_work;
1639         }
1640
1641         return 0;
1642
1643 out_work:
1644         del_timer_sync(&ap_config_timer);
1645         hrtimer_cancel(&ap_poll_timer);
1646         destroy_workqueue(ap_work_queue);
1647 out_root:
1648         root_device_unregister(ap_root_device);
1649 out_bus:
1650         while (i--)
1651                 bus_remove_file(&ap_bus_type, ap_bus_attrs[i]);
1652         bus_unregister(&ap_bus_type);
1653 out:
1654         unregister_reset_call(&ap_reset_call);
1655         if (ap_using_interrupts()) {
1656                 s390_unregister_adapter_interrupt(ap_interrupt_indicator, AP_ISC);
1657                 isc_unregister(AP_ISC);
1658         }
1659         return rc;
1660 }
1661
1662 static int __ap_match_all(struct device *dev, void *data)
1663 {
1664         return 1;
1665 }
1666
1667 /**
1668  * ap_modules_exit(): The module termination code
1669  *
1670  * Terminates the module.
1671  */
1672 void ap_module_exit(void)
1673 {
1674         int i;
1675         struct device *dev;
1676
1677         ap_reset_domain();
1678         ap_poll_thread_stop();
1679         del_timer_sync(&ap_config_timer);
1680         hrtimer_cancel(&ap_poll_timer);
1681         destroy_workqueue(ap_work_queue);
1682         tasklet_kill(&ap_tasklet);
1683         root_device_unregister(ap_root_device);
1684         while ((dev = bus_find_device(&ap_bus_type, NULL, NULL,
1685                     __ap_match_all)))
1686         {
1687                 device_unregister(dev);
1688                 put_device(dev);
1689         }
1690         for (i = 0; ap_bus_attrs[i]; i++)
1691                 bus_remove_file(&ap_bus_type, ap_bus_attrs[i]);
1692         bus_unregister(&ap_bus_type);
1693         unregister_reset_call(&ap_reset_call);
1694         if (ap_using_interrupts()) {
1695                 s390_unregister_adapter_interrupt(ap_interrupt_indicator, AP_ISC);
1696                 isc_unregister(AP_ISC);
1697         }
1698 }
1699
1700 #ifndef CONFIG_ZCRYPT_MONOLITHIC
1701 module_init(ap_module_init);
1702 module_exit(ap_module_exit);
1703 #endif