drivers/s390/crypto/ap_bus.c

   1 /*
   2  * linux/drivers/s390/crypto/ap_bus.c
   3  *
   4  * Copyright (C) 2006 IBM Corporation
   5  * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
   6  *            Martin Schwidefsky <schwidefsky@de.ibm.com>
   7  *            Ralph Wuerthner <rwuerthn@de.ibm.com>
   8  *            Felix Beck <felix.beck@de.ibm.com>
   9  *
  10  * Adjunct processor bus.
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License as published by
  14  * the Free Software Foundation; either version 2, or (at your option)
  15  * any later version.
  16  *
  17  * This program is distributed in the hope that it will be useful,
  18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  20  * GNU General Public License for more details.
  21  *
  22  * You should have received a copy of the GNU General Public License
  23  * along with this program; if not, write to the Free Software
  24  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  25  */
  26
  27 #define KMSG_COMPONENT "ap"
  28 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  29
  30 #include <linux/module.h>
  31 #include <linux/init.h>
  32 #include <linux/delay.h>
  33 #include <linux/err.h>
  34 #include <linux/interrupt.h>
  35 #include <linux/workqueue.h>
  36 #include <linux/notifier.h>
  37 #include <linux/kthread.h>
  38 #include <linux/mutex.h>
  39 #include <asm/reset.h>
  40 #include <asm/airq.h>
  41 #include <asm/atomic.h>
  42 #include <asm/system.h>
  43 #include <asm/isc.h>
  44 #include <linux/hrtimer.h>
  45 #include <linux/ktime.h>
  46
  47 #include "ap_bus.h"
  48
  49 /* Some prototypes. */
  50 static void ap_scan_bus(struct work_struct *);
  51 static void ap_poll_all(unsigned long);
  52 static enum hrtimer_restart ap_poll_timeout(struct hrtimer *);
  53 static int ap_poll_thread_start(void);
  54 static void ap_poll_thread_stop(void);
  55 static void ap_request_timeout(unsigned long);
  56 static inline void ap_schedule_poll_timer(void);
  57 static int __ap_poll_device(struct ap_device *ap_dev, unsigned long *flags);
  58 static int ap_device_remove(struct device *dev);
  59 static int ap_device_probe(struct device *dev);
  60 static void ap_interrupt_handler(void *unused1, void *unused2);
  61 static void ap_reset(struct ap_device *ap_dev);
  62 static void ap_config_timeout(unsigned long ptr);
  63
  64 /*
  65  * Module description.
  66  */
  67 MODULE_AUTHOR("IBM Corporation");
  68 MODULE_DESCRIPTION("Adjunct Processor Bus driver, "
  69                    "Copyright 2006 IBM Corporation");
  70 MODULE_LICENSE("GPL");
  71
  72 /*
  73  * Module parameter
  74  */
  75 int ap_domain_index = -1;       /* Adjunct Processor Domain Index */
  76 module_param_named(domain, ap_domain_index, int, 0000);
  77 MODULE_PARM_DESC(domain, "domain index for ap devices");
  78 EXPORT_SYMBOL(ap_domain_index);
  79
  80 static int ap_thread_flag = 0;
  81 module_param_named(poll_thread, ap_thread_flag, int, 0000);
  82 MODULE_PARM_DESC(poll_thread, "Turn on/off poll thread, default is 0 (off).");
  83
  84 static struct device *ap_root_device = NULL;
  85 static DEFINE_SPINLOCK(ap_device_list_lock);
  86 static LIST_HEAD(ap_device_list);
  87
  88 /*
  89  * Workqueue & timer for bus rescan.
  90  */
  91 static struct workqueue_struct *ap_work_queue;
  92 static struct timer_list ap_config_timer;
  93 static int ap_config_time = AP_CONFIG_TIME;
  94 static DECLARE_WORK(ap_config_work, ap_scan_bus);
  95
  96 /*
  97  * Tasklet & timer for AP request polling and interrupts
  98  */
  99 static DECLARE_TASKLET(ap_tasklet, ap_poll_all, 0);
 100 static atomic_t ap_poll_requests = ATOMIC_INIT(0);
 101 static DECLARE_WAIT_QUEUE_HEAD(ap_poll_wait);
 102 static struct task_struct *ap_poll_kthread = NULL;
 103 static DEFINE_MUTEX(ap_poll_thread_mutex);
 104 static void *ap_interrupt_indicator;
 105 static struct hrtimer ap_poll_timer;
 106 /* In LPAR poll with 4kHz frequency. Poll every 250000 nanoseconds.
 107  * If z/VM change to 1500000 nanoseconds to adjust to z/VM polling.*/
 108 static unsigned long long poll_timeout = 250000;
 109
 110 /* Suspend flag */
 111 static int ap_suspend_flag;
 112 static struct bus_type ap_bus_type;
 113
 114 /**
 115  * ap_using_interrupts() - Returns non-zero if interrupt support is
 116  * available.
 117  */
 118 static inline int ap_using_interrupts(void)
 119 {
 120         return ap_interrupt_indicator != NULL;
 121 }
 122
 123 /**
 124  * ap_intructions_available() - Test if AP instructions are available.
 125  *
 126  * Returns 0 if the AP instructions are installed.
 127  */
 128 static inline int ap_instructions_available(void)
 129 {
 130         register unsigned long reg0 asm ("0") = AP_MKQID(0,0);
 131         register unsigned long reg1 asm ("1") = -ENODEV;
 132         register unsigned long reg2 asm ("2") = 0UL;
 133
 134         asm volatile(
 135                 "   .long 0xb2af0000\n"         /* PQAP(TAPQ) */
 136                 "0: la    %1,0\n"
 137                 "1:\n"
 138                 EX_TABLE(0b, 1b)
 139                 : "+d" (reg0), "+d" (reg1), "+d" (reg2) : : "cc" );
 140         return reg1;
 141 }
 142
 143 /**
 144  * ap_interrupts_available(): Test if AP interrupts are available.
 145  *
 146  * Returns 1 if AP interrupts are available.
 147  */
 148 static int ap_interrupts_available(void)
 149 {
 150         unsigned long long facility_bits[2];
 151
 152         if (stfle(facility_bits, 2) <= 1)
 153                 return 0;
 154         if (!(facility_bits[0] & (1ULL << 61)) ||
 155             !(facility_bits[1] & (1ULL << 62)))
 156                 return 0;
 157         return 1;
 158 }
 159
 160 /**
 161  * ap_test_queue(): Test adjunct processor queue.
 162  * @qid: The AP queue number
 163  * @queue_depth: Pointer to queue depth value
 164  * @device_type: Pointer to device type value
 165  *
 166  * Returns AP queue status structure.
 167  */
 168 static inline struct ap_queue_status
 169 ap_test_queue(ap_qid_t qid, int *queue_depth, int *device_type)
 170 {
 171         register unsigned long reg0 asm ("0") = qid;
 172         register struct ap_queue_status reg1 asm ("1");
 173         register unsigned long reg2 asm ("2") = 0UL;
 174
 175         asm volatile(".long 0xb2af0000"         /* PQAP(TAPQ) */
 176                      : "+d" (reg0), "=d" (reg1), "+d" (reg2) : : "cc");
 177         *device_type = (int) (reg2 >> 24);
 178         *queue_depth = (int) (reg2 & 0xff);
 179         return reg1;
 180 }
 181
 182 /**
 183  * ap_reset_queue(): Reset adjunct processor queue.
 184  * @qid: The AP queue number
 185  *
 186  * Returns AP queue status structure.
 187  */
 188 static inline struct ap_queue_status ap_reset_queue(ap_qid_t qid)
 189 {
 190         register unsigned long reg0 asm ("0") = qid | 0x01000000UL;
 191         register struct ap_queue_status reg1 asm ("1");
 192         register unsigned long reg2 asm ("2") = 0UL;
 193
 194         asm volatile(
 195                 ".long 0xb2af0000"              /* PQAP(RAPQ) */
 196                 : "+d" (reg0), "=d" (reg1), "+d" (reg2) : : "cc");
 197         return reg1;
 198 }
 199
 200 #ifdef CONFIG_64BIT
 201 /**
 202  * ap_queue_interruption_control(): Enable interruption for a specific AP.
 203  * @qid: The AP queue number
 204  * @ind: The notification indicator byte
 205  *
 206  * Returns AP queue status.
 207  */
 208 static inline struct ap_queue_status
 209 ap_queue_interruption_control(ap_qid_t qid, void *ind)
 210 {
 211         register unsigned long reg0 asm ("0") = qid | 0x03000000UL;
 212         register unsigned long reg1_in asm ("1") = 0x0000800000000000UL | AP_ISC;
 213         register struct ap_queue_status reg1_out asm ("1");
 214         register void *reg2 asm ("2") = ind;
 215         asm volatile(
 216                 ".long 0xb2af0000"              /* PQAP(RAPQ) */
 217                 : "+d" (reg0), "+d" (reg1_in), "=d" (reg1_out), "+d" (reg2)
 218                 :
 219                 : "cc" );
 220         return reg1_out;
 221 }
 222 #endif
 223
 224 /**
 225  * ap_queue_enable_interruption(): Enable interruption on an AP.
 226  * @qid: The AP queue number
 227  * @ind: the notification indicator byte
 228  *
 229  * Enables interruption on AP queue via ap_queue_interruption_control(). Based
 230  * on the return value it waits a while and tests the AP queue if interrupts
 231  * have been switched on using ap_test_queue().
 232  */
 233 static int ap_queue_enable_interruption(ap_qid_t qid, void *ind)
 234 {
 235 #ifdef CONFIG_64BIT
 236         struct ap_queue_status status;
 237         int t_depth, t_device_type, rc, i;
 238
 239         rc = -EBUSY;
 240         status = ap_queue_interruption_control(qid, ind);
 241
 242         for (i = 0; i < AP_MAX_RESET; i++) {
 243                 switch (status.response_code) {
 244                 case AP_RESPONSE_NORMAL:
 245                         if (status.int_enabled)
 246                                 return 0;
 247                         break;
 248                 case AP_RESPONSE_RESET_IN_PROGRESS:
 249                 case AP_RESPONSE_BUSY:
 250                         break;
 251                 case AP_RESPONSE_Q_NOT_AVAIL:
 252                 case AP_RESPONSE_DECONFIGURED:
 253                 case AP_RESPONSE_CHECKSTOPPED:
 254                 case AP_RESPONSE_INVALID_ADDRESS:
 255                         return -ENODEV;
 256                 case AP_RESPONSE_OTHERWISE_CHANGED:
 257                         if (status.int_enabled)
 258                                 return 0;
 259                         break;
 260                 default:
 261                         break;
 262                 }
 263                 if (i < AP_MAX_RESET - 1) {
 264                         udelay(5);
 265                         status = ap_test_queue(qid, &t_depth, &t_device_type);
 266                 }
 267         }
 268         return rc;
 269 #else
 270         return -EINVAL;
 271 #endif
 272 }
 273
 274 /**
 275  * __ap_send(): Send message to adjunct processor queue.
 276  * @qid: The AP queue number
 277  * @psmid: The program supplied message identifier
 278  * @msg: The message text
 279  * @length: The message length
 280  *
 281  * Returns AP queue status structure.
 282  * Condition code 1 on NQAP can't happen because the L bit is 1.
 283  * Condition code 2 on NQAP also means the send is incomplete,
 284  * because a segment boundary was reached. The NQAP is repeated.
 285  */
 286 static inline struct ap_queue_status
 287 __ap_send(ap_qid_t qid, unsigned long long psmid, void *msg, size_t length)
 288 {
 289         typedef struct { char _[length]; } msgblock;
 290         register unsigned long reg0 asm ("0") = qid | 0x40000000UL;
 291         register struct ap_queue_status reg1 asm ("1");
 292         register unsigned long reg2 asm ("2") = (unsigned long) msg;
 293         register unsigned long reg3 asm ("3") = (unsigned long) length;
 294         register unsigned long reg4 asm ("4") = (unsigned int) (psmid >> 32);
 295         register unsigned long reg5 asm ("5") = (unsigned int) psmid;
 296
 297         asm volatile (
 298                 "0: .long 0xb2ad0042\n"         /* DQAP */
 299                 "   brc   2,0b"
 300                 : "+d" (reg0), "=d" (reg1), "+d" (reg2), "+d" (reg3)
 301                 : "d" (reg4), "d" (reg5), "m" (*(msgblock *) msg)
 302                 : "cc" );
 303         return reg1;
 304 }
 305
 306 int ap_send(ap_qid_t qid, unsigned long long psmid, void *msg, size_t length)
 307 {
 308         struct ap_queue_status status;
 309
 310         status = __ap_send(qid, psmid, msg, length);
 311         switch (status.response_code) {
 312         case AP_RESPONSE_NORMAL:
 313                 return 0;
 314         case AP_RESPONSE_Q_FULL:
 315         case AP_RESPONSE_RESET_IN_PROGRESS:
 316                 return -EBUSY;
 317         default:        /* Device is gone. */
 318                 return -ENODEV;
 319         }
 320 }
 321 EXPORT_SYMBOL(ap_send);
 322
 323 /**
 324  * __ap_recv(): Receive message from adjunct processor queue.
 325  * @qid: The AP queue number
 326  * @psmid: Pointer to program supplied message identifier
 327  * @msg: The message text
 328  * @length: The message length
 329  *
 330  * Returns AP queue status structure.
 331  * Condition code 1 on DQAP means the receive has taken place
 332  * but only partially.  The response is incomplete, hence the
 333  * DQAP is repeated.
 334  * Condition code 2 on DQAP also means the receive is incomplete,
 335  * this time because a segment boundary was reached. Again, the
 336  * DQAP is repeated.
 337  * Note that gpr2 is used by the DQAP instruction to keep track of
 338  * any 'residual' length, in case the instruction gets interrupted.
 339  * Hence it gets zeroed before the instruction.
 340  */
 341 static inline struct ap_queue_status
 342 __ap_recv(ap_qid_t qid, unsigned long long *psmid, void *msg, size_t length)
 343 {
 344         typedef struct { char _[length]; } msgblock;
 345         register unsigned long reg0 asm("0") = qid | 0x80000000UL;
 346         register struct ap_queue_status reg1 asm ("1");
 347         register unsigned long reg2 asm("2") = 0UL;
 348         register unsigned long reg4 asm("4") = (unsigned long) msg;
 349         register unsigned long reg5 asm("5") = (unsigned long) length;
 350         register unsigned long reg6 asm("6") = 0UL;
 351         register unsigned long reg7 asm("7") = 0UL;
 352
 353
 354         asm volatile(
 355                 "0: .long 0xb2ae0064\n"
 356                 "   brc   6,0b\n"
 357                 : "+d" (reg0), "=d" (reg1), "+d" (reg2),
 358                 "+d" (reg4), "+d" (reg5), "+d" (reg6), "+d" (reg7),
 359                 "=m" (*(msgblock *) msg) : : "cc" );
 360         *psmid = (((unsigned long long) reg6) << 32) + reg7;
 361         return reg1;
 362 }
 363
 364 int ap_recv(ap_qid_t qid, unsigned long long *psmid, void *msg, size_t length)
 365 {
 366         struct ap_queue_status status;
 367
 368         status = __ap_recv(qid, psmid, msg, length);
 369         switch (status.response_code) {
 370         case AP_RESPONSE_NORMAL:
 371                 return 0;
 372         case AP_RESPONSE_NO_PENDING_REPLY:
 373                 if (status.queue_empty)
 374                         return -ENOENT;
 375                 return -EBUSY;
 376         case AP_RESPONSE_RESET_IN_PROGRESS:
 377                 return -EBUSY;
 378         default:
 379                 return -ENODEV;
 380         }
 381 }
 382 EXPORT_SYMBOL(ap_recv);
 383
 384 /**
 385  * ap_query_queue(): Check if an AP queue is available.
 386  * @qid: The AP queue number
 387  * @queue_depth: Pointer to queue depth value
 388  * @device_type: Pointer to device type value
 389  *
 390  * The test is repeated for AP_MAX_RESET times.
 391  */
 392 static int ap_query_queue(ap_qid_t qid, int *queue_depth, int *device_type)
 393 {
 394         struct ap_queue_status status;
 395         int t_depth, t_device_type, rc, i;
 396
 397         rc = -EBUSY;
 398         for (i = 0; i < AP_MAX_RESET; i++) {
 399                 status = ap_test_queue(qid, &t_depth, &t_device_type);
 400                 switch (status.response_code) {
 401                 case AP_RESPONSE_NORMAL:
 402                         *queue_depth = t_depth + 1;
 403                         *device_type = t_device_type;
 404                         rc = 0;
 405                         break;
 406                 case AP_RESPONSE_Q_NOT_AVAIL:
 407                         rc = -ENODEV;
 408                         break;
 409                 case AP_RESPONSE_RESET_IN_PROGRESS:
 410                         break;
 411                 case AP_RESPONSE_DECONFIGURED:
 412                         rc = -ENODEV;
 413                         break;
 414                 case AP_RESPONSE_CHECKSTOPPED:
 415                         rc = -ENODEV;
 416                         break;
 417                 case AP_RESPONSE_INVALID_ADDRESS:
 418                         rc = -ENODEV;
 419                         break;
 420                 case AP_RESPONSE_OTHERWISE_CHANGED:
 421                         break;
 422                 case AP_RESPONSE_BUSY:
 423                         break;
 424                 default:
 425                         BUG();
 426                 }
 427                 if (rc != -EBUSY)
 428                         break;
 429                 if (i < AP_MAX_RESET - 1)
 430                         udelay(5);
 431         }
 432         return rc;
 433 }
 434
 435 /**
 436  * ap_init_queue(): Reset an AP queue.
 437  * @qid: The AP queue number
 438  *
 439  * Reset an AP queue and wait for it to become available again.
 440  */
 441 static int ap_init_queue(ap_qid_t qid)
 442 {
 443         struct ap_queue_status status;
 444         int rc, dummy, i;
 445
 446         rc = -ENODEV;
 447         status = ap_reset_queue(qid);
 448         for (i = 0; i < AP_MAX_RESET; i++) {
 449                 switch (status.response_code) {
 450                 case AP_RESPONSE_NORMAL:
 451                         if (status.queue_empty)
 452                                 rc = 0;
 453                         break;
 454                 case AP_RESPONSE_Q_NOT_AVAIL:
 455                 case AP_RESPONSE_DECONFIGURED:
 456                 case AP_RESPONSE_CHECKSTOPPED:
 457                         i = AP_MAX_RESET;       /* return with -ENODEV */
 458                         break;
 459                 case AP_RESPONSE_RESET_IN_PROGRESS:
 460                         rc = -EBUSY;
 461                 case AP_RESPONSE_BUSY:
 462                 default:
 463                         break;
 464                 }
 465                 if (rc != -ENODEV && rc != -EBUSY)
 466                         break;
 467                 if (i < AP_MAX_RESET - 1) {
 468                         udelay(5);
 469                         status = ap_test_queue(qid, &dummy, &dummy);
 470                 }
 471         }
 472         if (rc == 0 && ap_using_interrupts()) {
 473                 rc = ap_queue_enable_interruption(qid, ap_interrupt_indicator);
 474                 /* If interruption mode is supported by the machine,
 475                 * but an AP can not be enabled for interruption then
 476                 * the AP will be discarded.    */
 477                 if (rc)
 478                         pr_err("Registering adapter interrupts for "
 479                                "AP %d failed\n", AP_QID_DEVICE(qid));
 480         }
 481         return rc;
 482 }
 483
 484 /**
 485  * ap_increase_queue_count(): Arm request timeout.
 486  * @ap_dev: Pointer to an AP device.
 487  *
 488  * Arm request timeout if an AP device was idle and a new request is submitted.
 489  */
 490 static void ap_increase_queue_count(struct ap_device *ap_dev)
 491 {
 492         int timeout = ap_dev->drv->request_timeout;
 493
 494         ap_dev->queue_count++;
 495         if (ap_dev->queue_count == 1) {
 496                 mod_timer(&ap_dev->timeout, jiffies + timeout);
 497                 ap_dev->reset = AP_RESET_ARMED;
 498         }
 499 }
 500
 501 /**
 502  * ap_decrease_queue_count(): Decrease queue count.
 503  * @ap_dev: Pointer to an AP device.
 504  *
 505  * If AP device is still alive, re-schedule request timeout if there are still
 506  * pending requests.
 507  */
 508 static void ap_decrease_queue_count(struct ap_device *ap_dev)
 509 {
 510         int timeout = ap_dev->drv->request_timeout;
 511
 512         ap_dev->queue_count--;
 513         if (ap_dev->queue_count > 0)
 514                 mod_timer(&ap_dev->timeout, jiffies + timeout);
 515         else
 516                 /*
 517                  * The timeout timer should to be disabled now - since
 518                  * del_timer_sync() is very expensive, we just tell via the
 519                  * reset flag to ignore the pending timeout timer.
 520                  */
 521                 ap_dev->reset = AP_RESET_IGNORE;
 522 }
 523
 524 /*
 525  * AP device related attributes.
 526  */
 527 static ssize_t ap_hwtype_show(struct device *dev,
 528                               struct device_attribute *attr, char *buf)
 529 {
 530         struct ap_device *ap_dev = to_ap_dev(dev);
 531         return snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->device_type);
 532 }
 533
 534 static DEVICE_ATTR(hwtype, 0444, ap_hwtype_show, NULL);
 535 static ssize_t ap_depth_show(struct device *dev, struct device_attribute *attr,
 536                              char *buf)
 537 {
 538         struct ap_device *ap_dev = to_ap_dev(dev);
 539         return snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->queue_depth);
 540 }
 541
 542 static DEVICE_ATTR(depth, 0444, ap_depth_show, NULL);
 543 static ssize_t ap_request_count_show(struct device *dev,
 544                                      struct device_attribute *attr,
 545                                      char *buf)
 546 {
 547         struct ap_device *ap_dev = to_ap_dev(dev);
 548         int rc;
 549
 550         spin_lock_bh(&ap_dev->lock);
 551         rc = snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->total_request_count);
 552         spin_unlock_bh(&ap_dev->lock);
 553         return rc;
 554 }
 555
 556 static DEVICE_ATTR(request_count, 0444, ap_request_count_show, NULL);
 557
 558 static ssize_t ap_modalias_show(struct device *dev,
 559                                 struct device_attribute *attr, char *buf)
 560 {
 561         return sprintf(buf, "ap:t%02X", to_ap_dev(dev)->device_type);
 562 }
 563
 564 static DEVICE_ATTR(modalias, 0444, ap_modalias_show, NULL);
 565
 566 static struct attribute *ap_dev_attrs[] = {
 567         &dev_attr_hwtype.attr,
 568         &dev_attr_depth.attr,
 569         &dev_attr_request_count.attr,
 570         &dev_attr_modalias.attr,
 571         NULL
 572 };
 573 static struct attribute_group ap_dev_attr_group = {
 574         .attrs = ap_dev_attrs
 575 };
 576
 577 /**
 578  * ap_bus_match()
 579  * @dev: Pointer to device
 580  * @drv: Pointer to device_driver
 581  *
 582  * AP bus driver registration/unregistration.
 583  */
 584 static int ap_bus_match(struct device *dev, struct device_driver *drv)
 585 {
 586         struct ap_device *ap_dev = to_ap_dev(dev);
 587         struct ap_driver *ap_drv = to_ap_drv(drv);
 588         struct ap_device_id *id;
 589
 590         /*
 591          * Compare device type of the device with the list of
 592          * supported types of the device_driver.
 593          */
 594         for (id = ap_drv->ids; id->match_flags; id++) {
 595                 if ((id->match_flags & AP_DEVICE_ID_MATCH_DEVICE_TYPE) &&
 596                     (id->dev_type != ap_dev->device_type))
 597                         continue;
 598                 return 1;
 599         }
 600         return 0;
 601 }
 602
 603 /**
 604  * ap_uevent(): Uevent function for AP devices.
 605  * @dev: Pointer to device
 606  * @env: Pointer to kobj_uevent_env
 607  *
 608  * It sets up a single environment variable DEV_TYPE which contains the
 609  * hardware device type.
 610  */
 611 static int ap_uevent (struct device *dev, struct kobj_uevent_env *env)
 612 {
 613         struct ap_device *ap_dev = to_ap_dev(dev);
 614         int retval = 0;
 615
 616         if (!ap_dev)
 617                 return -ENODEV;
 618
 619         /* Set up DEV_TYPE environment variable. */
 620         retval = add_uevent_var(env, "DEV_TYPE=%04X", ap_dev->device_type);
 621         if (retval)
 622                 return retval;
 623
 624         /* Add MODALIAS= */
 625         retval = add_uevent_var(env, "MODALIAS=ap:t%02X", ap_dev->device_type);
 626
 627         return retval;
 628 }
 629
 630 static int ap_bus_suspend(struct device *dev, pm_message_t state)
 631 {
 632         struct ap_device *ap_dev = to_ap_dev(dev);
 633         unsigned long flags;
 634
 635         if (!ap_suspend_flag) {
 636                 ap_suspend_flag = 1;
 637
 638                 /* Disable scanning for devices, thus we do not want to scan
 639                  * for them after removing.
 640                  */
 641                 del_timer_sync(&ap_config_timer);
 642                 if (ap_work_queue != NULL) {
 643                         destroy_workqueue(ap_work_queue);
 644                         ap_work_queue = NULL;
 645                 }
 646                 tasklet_disable(&ap_tasklet);
 647         }
 648         /* Poll on the device until all requests are finished. */
 649         do {
 650                 flags = 0;
 651                 __ap_poll_device(ap_dev, &flags);
 652         } while ((flags & 1) || (flags & 2));
 653
 654         ap_device_remove(dev);
 655         return 0;
 656 }
 657
 658 static int ap_bus_resume(struct device *dev)
 659 {
 660         int rc = 0;
 661         struct ap_device *ap_dev = to_ap_dev(dev);
 662
 663         if (ap_suspend_flag) {
 664                 ap_suspend_flag = 0;
 665                 if (!ap_interrupts_available())
 666                         ap_interrupt_indicator = NULL;
 667                 ap_device_probe(dev);
 668                 ap_reset(ap_dev);
 669                 setup_timer(&ap_dev->timeout, ap_request_timeout,
 670                             (unsigned long) ap_dev);
 671                 ap_scan_bus(NULL);
 672                 init_timer(&ap_config_timer);
 673                 ap_config_timer.function = ap_config_timeout;
 674                 ap_config_timer.data = 0;
 675                 ap_config_timer.expires = jiffies + ap_config_time * HZ;
 676                 add_timer(&ap_config_timer);
 677                 ap_work_queue = create_singlethread_workqueue("kapwork");
 678                 if (!ap_work_queue)
 679                         return -ENOMEM;
 680                 tasklet_enable(&ap_tasklet);
 681                 if (!ap_using_interrupts())
 682                         ap_schedule_poll_timer();
 683                 else
 684                         tasklet_schedule(&ap_tasklet);
 685                 if (ap_thread_flag)
 686                         rc = ap_poll_thread_start();
 687         } else {
 688                 ap_device_probe(dev);
 689                 ap_reset(ap_dev);
 690                 setup_timer(&ap_dev->timeout, ap_request_timeout,
 691                             (unsigned long) ap_dev);
 692         }
 693
 694         return rc;
 695 }
 696
 697 static struct bus_type ap_bus_type = {
 698         .name = "ap",
 699         .match = &ap_bus_match,
 700         .uevent = &ap_uevent,
 701         .suspend = ap_bus_suspend,
 702         .resume = ap_bus_resume
 703 };
 704
 705 static int ap_device_probe(struct device *dev)
 706 {
 707         struct ap_device *ap_dev = to_ap_dev(dev);
 708         struct ap_driver *ap_drv = to_ap_drv(dev->driver);
 709         int rc;
 710
 711         ap_dev->drv = ap_drv;
 712         rc = ap_drv->probe ? ap_drv->probe(ap_dev) : -ENODEV;
 713         if (!rc) {
 714                 spin_lock_bh(&ap_device_list_lock);
 715                 list_add(&ap_dev->list, &ap_device_list);
 716                 spin_unlock_bh(&ap_device_list_lock);
 717         }
 718         return rc;
 719 }
 720
 721 /**
 722  * __ap_flush_queue(): Flush requests.
 723  * @ap_dev: Pointer to the AP device
 724  *
 725  * Flush all requests from the request/pending queue of an AP device.
 726  */
 727 static void __ap_flush_queue(struct ap_device *ap_dev)
 728 {
 729         struct ap_message *ap_msg, *next;
 730
 731         list_for_each_entry_safe(ap_msg, next, &ap_dev->pendingq, list) {
 732                 list_del_init(&ap_msg->list);
 733                 ap_dev->pendingq_count--;
 734                 ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
 735         }
 736         list_for_each_entry_safe(ap_msg, next, &ap_dev->requestq, list) {
 737                 list_del_init(&ap_msg->list);
 738                 ap_dev->requestq_count--;
 739                 ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
 740         }
 741 }
 742
 743 void ap_flush_queue(struct ap_device *ap_dev)
 744 {
 745         spin_lock_bh(&ap_dev->lock);
 746         __ap_flush_queue(ap_dev);
 747         spin_unlock_bh(&ap_dev->lock);
 748 }
 749 EXPORT_SYMBOL(ap_flush_queue);
 750
 751 static int ap_device_remove(struct device *dev)
 752 {
 753         struct ap_device *ap_dev = to_ap_dev(dev);
 754         struct ap_driver *ap_drv = ap_dev->drv;
 755
 756         ap_flush_queue(ap_dev);
 757         del_timer_sync(&ap_dev->timeout);
 758         spin_lock_bh(&ap_device_list_lock);
 759         list_del_init(&ap_dev->list);
 760         spin_unlock_bh(&ap_device_list_lock);
 761         if (ap_drv->remove)
 762                 ap_drv->remove(ap_dev);
 763         spin_lock_bh(&ap_dev->lock);
 764         atomic_sub(ap_dev->queue_count, &ap_poll_requests);
 765         spin_unlock_bh(&ap_dev->lock);
 766         return 0;
 767 }
 768
 769 int ap_driver_register(struct ap_driver *ap_drv, struct module *owner,
 770                        char *name)
 771 {
 772         struct device_driver *drv = &ap_drv->driver;
 773
 774         drv->bus = &ap_bus_type;
 775         drv->probe = ap_device_probe;
 776         drv->remove = ap_device_remove;
 777         drv->owner = owner;
 778         drv->name = name;
 779         return driver_register(drv);
 780 }
 781 EXPORT_SYMBOL(ap_driver_register);
 782
 783 void ap_driver_unregister(struct ap_driver *ap_drv)
 784 {
 785         driver_unregister(&ap_drv->driver);
 786 }
 787 EXPORT_SYMBOL(ap_driver_unregister);
 788
 789 /*
 790  * AP bus attributes.
 791  */
 792 static ssize_t ap_domain_show(struct bus_type *bus, char *buf)
 793 {
 794         return snprintf(buf, PAGE_SIZE, "%d\n", ap_domain_index);
 795 }
 796
 797 static BUS_ATTR(ap_domain, 0444, ap_domain_show, NULL);
 798
 799 static ssize_t ap_config_time_show(struct bus_type *bus, char *buf)
 800 {
 801         return snprintf(buf, PAGE_SIZE, "%d\n", ap_config_time);
 802 }
 803
 804 static ssize_t ap_interrupts_show(struct bus_type *bus, char *buf)
 805 {
 806         return snprintf(buf, PAGE_SIZE, "%d\n",
 807                         ap_using_interrupts() ? 1 : 0);
 808 }
 809
 810 static BUS_ATTR(ap_interrupts, 0444, ap_interrupts_show, NULL);
 811
 812 static ssize_t ap_config_time_store(struct bus_type *bus,
 813                                     const char *buf, size_t count)
 814 {
 815         int time;
 816
 817         if (sscanf(buf, "%d\n", &time) != 1 || time < 5 || time > 120)
 818                 return -EINVAL;
 819         ap_config_time = time;
 820         if (!timer_pending(&ap_config_timer) ||
 821             !mod_timer(&ap_config_timer, jiffies + ap_config_time * HZ)) {
 822                 ap_config_timer.expires = jiffies + ap_config_time * HZ;
 823                 add_timer(&ap_config_timer);
 824         }
 825         return count;
 826 }
 827
 828 static BUS_ATTR(config_time, 0644, ap_config_time_show, ap_config_time_store);
 829
 830 static ssize_t ap_poll_thread_show(struct bus_type *bus, char *buf)
 831 {
 832         return snprintf(buf, PAGE_SIZE, "%d\n", ap_poll_kthread ? 1 : 0);
 833 }
 834
 835 static ssize_t ap_poll_thread_store(struct bus_type *bus,
 836                                     const char *buf, size_t count)
 837 {
 838         int flag, rc;
 839
 840         if (sscanf(buf, "%d\n", &flag) != 1)
 841                 return -EINVAL;
 842         if (flag) {
 843                 rc = ap_poll_thread_start();
 844                 if (rc)
 845                         return rc;
 846         }
 847         else
 848                 ap_poll_thread_stop();
 849         return count;
 850 }
 851
 852 static BUS_ATTR(poll_thread, 0644, ap_poll_thread_show, ap_poll_thread_store);
 853
 854 static ssize_t poll_timeout_show(struct bus_type *bus, char *buf)
 855 {
 856         return snprintf(buf, PAGE_SIZE, "%llu\n", poll_timeout);
 857 }
 858
 859 static ssize_t poll_timeout_store(struct bus_type *bus, const char *buf,
 860                                   size_t count)
 861 {
 862         unsigned long long time;
 863         ktime_t hr_time;
 864
 865         /* 120 seconds = maximum poll interval */
 866         if (sscanf(buf, "%llu\n", &time) != 1 || time < 1 ||
 867             time > 120000000000ULL)
 868                 return -EINVAL;
 869         poll_timeout = time;
 870         hr_time = ktime_set(0, poll_timeout);
 871
 872         if (!hrtimer_is_queued(&ap_poll_timer) ||
 873             !hrtimer_forward(&ap_poll_timer, hrtimer_get_expires(&ap_poll_timer), hr_time)) {
 874                 hrtimer_set_expires(&ap_poll_timer, hr_time);
 875                 hrtimer_start_expires(&ap_poll_timer, HRTIMER_MODE_ABS);
 876         }
 877         return count;
 878 }
 879
 880 static BUS_ATTR(poll_timeout, 0644, poll_timeout_show, poll_timeout_store);
 881
 882 static struct bus_attribute *const ap_bus_attrs[] = {
 883         &bus_attr_ap_domain,
 884         &bus_attr_config_time,
 885         &bus_attr_poll_thread,
 886         &bus_attr_ap_interrupts,
 887         &bus_attr_poll_timeout,
 888         NULL,
 889 };
 890
 891 /**
 892  * ap_select_domain(): Select an AP domain.
 893  *
 894  * Pick one of the 16 AP domains.
 895  */
 896 static int ap_select_domain(void)
 897 {
 898         int queue_depth, device_type, count, max_count, best_domain;
 899         int rc, i, j;
 900
 901         /*
 902          * We want to use a single domain. Either the one specified with
 903          * the "domain=" parameter or the domain with the maximum number
 904          * of devices.
 905          */
 906         if (ap_domain_index >= 0 && ap_domain_index < AP_DOMAINS)
 907                 /* Domain has already been selected. */
 908                 return 0;
 909         best_domain = -1;
 910         max_count = 0;
 911         for (i = 0; i < AP_DOMAINS; i++) {
 912                 count = 0;
 913                 for (j = 0; j < AP_DEVICES; j++) {
 914                         ap_qid_t qid = AP_MKQID(j, i);
 915                         rc = ap_query_queue(qid, &queue_depth, &device_type);
 916                         if (rc)
 917                                 continue;
 918                         count++;
 919                 }
 920                 if (count > max_count) {
 921                         max_count = count;
 922                         best_domain = i;
 923                 }
 924         }
 925         if (best_domain >= 0){
 926                 ap_domain_index = best_domain;
 927                 return 0;
 928         }
 929         return -ENODEV;
 930 }
 931
 932 /**
 933  * ap_probe_device_type(): Find the device type of an AP.
 934  * @ap_dev: pointer to the AP device.
 935  *
 936  * Find the device type if query queue returned a device type of 0.
 937  */
 938 static int ap_probe_device_type(struct ap_device *ap_dev)
 939 {
 940         static unsigned char msg[] = {
 941                 0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,
 942                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 943                 0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,
 944                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 945                 0x01,0x00,0x43,0x43,0x41,0x2d,0x41,0x50,
 946                 0x50,0x4c,0x20,0x20,0x20,0x01,0x01,0x01,
 947                 0x00,0x00,0x00,0x00,0x50,0x4b,0x00,0x00,
 948                 0x00,0x00,0x01,0x1c,0x00,0x00,0x00,0x00,
 949                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 950                 0x00,0x00,0x05,0xb8,0x00,0x00,0x00,0x00,
 951                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 952                 0x70,0x00,0x41,0x00,0x00,0x00,0x00,0x00,
 953                 0x00,0x00,0x54,0x32,0x01,0x00,0xa0,0x00,
 954                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 955                 0x00,0x00,0x00,0x00,0xb8,0x05,0x00,0x00,
 956                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 957                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 958                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 959                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 960                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 961                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 962                 0x00,0x00,0x0a,0x00,0x00,0x00,0x00,0x00,
 963                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 964                 0x00,0x00,0x00,0x00,0x00,0x00,0x08,0x00,
 965                 0x49,0x43,0x53,0x46,0x20,0x20,0x20,0x20,
 966                 0x50,0x4b,0x0a,0x00,0x50,0x4b,0x43,0x53,
 967                 0x2d,0x31,0x2e,0x32,0x37,0x00,0x11,0x22,
 968                 0x33,0x44,0x55,0x66,0x77,0x88,0x99,0x00,
 969                 0x11,0x22,0x33,0x44,0x55,0x66,0x77,0x88,
 970                 0x99,0x00,0x11,0x22,0x33,0x44,0x55,0x66,
 971                 0x77,0x88,0x99,0x00,0x11,0x22,0x33,0x44,
 972                 0x55,0x66,0x77,0x88,0x99,0x00,0x11,0x22,
 973                 0x33,0x44,0x55,0x66,0x77,0x88,0x99,0x00,
 974                 0x11,0x22,0x33,0x5d,0x00,0x5b,0x00,0x77,
 975                 0x88,0x1e,0x00,0x00,0x57,0x00,0x00,0x00,
 976                 0x00,0x04,0x00,0x00,0x4f,0x00,0x00,0x00,
 977                 0x03,0x02,0x00,0x00,0x40,0x01,0x00,0x01,
 978                 0xce,0x02,0x68,0x2d,0x5f,0xa9,0xde,0x0c,
 979                 0xf6,0xd2,0x7b,0x58,0x4b,0xf9,0x28,0x68,
 980                 0x3d,0xb4,0xf4,0xef,0x78,0xd5,0xbe,0x66,
 981                 0x63,0x42,0xef,0xf8,0xfd,0xa4,0xf8,0xb0,
 982                 0x8e,0x29,0xc2,0xc9,0x2e,0xd8,0x45,0xb8,
 983                 0x53,0x8c,0x6f,0x4e,0x72,0x8f,0x6c,0x04,
 984                 0x9c,0x88,0xfc,0x1e,0xc5,0x83,0x55,0x57,
 985                 0xf7,0xdd,0xfd,0x4f,0x11,0x36,0x95,0x5d,
 986         };
 987         struct ap_queue_status status;
 988         unsigned long long psmid;
 989         char *reply;
 990         int rc, i;
 991
 992         reply = (void *) get_zeroed_page(GFP_KERNEL);
 993         if (!reply) {
 994                 rc = -ENOMEM;
 995                 goto out;
 996         }
 997
 998         status = __ap_send(ap_dev->qid, 0x0102030405060708ULL,
 999                            msg, sizeof(msg));
1000         if (status.response_code != AP_RESPONSE_NORMAL) {
1001                 rc = -ENODEV;
1002                 goto out_free;
1003         }
1004
1005         /* Wait for the test message to complete. */
1006         for (i = 0; i < 6; i++) {
1007                 mdelay(300);
1008                 status = __ap_recv(ap_dev->qid, &psmid, reply, 4096);
1009                 if (status.response_code == AP_RESPONSE_NORMAL &&
1010                     psmid == 0x0102030405060708ULL)
1011                         break;
1012         }
1013         if (i < 6) {
1014                 /* Got an answer. */
1015                 if (reply[0] == 0x00 && reply[1] == 0x86)
1016                         ap_dev->device_type = AP_DEVICE_TYPE_PCICC;
1017                 else
1018                         ap_dev->device_type = AP_DEVICE_TYPE_PCICA;
1019                 rc = 0;
1020         } else
1021                 rc = -ENODEV;
1022
1023 out_free:
1024         free_page((unsigned long) reply);
1025 out:
1026         return rc;
1027 }
1028
1029 static void ap_interrupt_handler(void *unused1, void *unused2)
1030 {
1031         tasklet_schedule(&ap_tasklet);
1032 }
1033
1034 /**
1035  * __ap_scan_bus(): Scan the AP bus.
1036  * @dev: Pointer to device
1037  * @data: Pointer to data
1038  *
1039  * Scan the AP bus for new devices.
1040  */
1041 static int __ap_scan_bus(struct device *dev, void *data)
1042 {
1043         return to_ap_dev(dev)->qid == (ap_qid_t)(unsigned long) data;
1044 }
1045
1046 static void ap_device_release(struct device *dev)
1047 {
1048         struct ap_device *ap_dev = to_ap_dev(dev);
1049
1050         kfree(ap_dev);
1051 }
1052
1053 static void ap_scan_bus(struct work_struct *unused)
1054 {
1055         struct ap_device *ap_dev;
1056         struct device *dev;
1057         ap_qid_t qid;
1058         int queue_depth, device_type;
1059         int rc, i;
1060
1061         if (ap_select_domain() != 0)
1062                 return;
1063         for (i = 0; i < AP_DEVICES; i++) {
1064                 qid = AP_MKQID(i, ap_domain_index);
1065                 dev = bus_find_device(&ap_bus_type, NULL,
1066                                       (void *)(unsigned long)qid,
1067                                       __ap_scan_bus);
1068                 rc = ap_query_queue(qid, &queue_depth, &device_type);
1069                 if (dev) {
1070                         if (rc == -EBUSY) {
1071                                 set_current_state(TASK_UNINTERRUPTIBLE);
1072                                 schedule_timeout(AP_RESET_TIMEOUT);
1073                                 rc = ap_query_queue(qid, &queue_depth,
1074                                                     &device_type);
1075                         }
1076                         ap_dev = to_ap_dev(dev);
1077                         spin_lock_bh(&ap_dev->lock);
1078                         if (rc || ap_dev->unregistered) {
1079                                 spin_unlock_bh(&ap_dev->lock);
1080                                 device_unregister(dev);
1081                                 put_device(dev);
1082                                 continue;
1083                         }
1084                         spin_unlock_bh(&ap_dev->lock);
1085                         put_device(dev);
1086                         continue;
1087                 }
1088                 if (rc)
1089                         continue;
1090                 rc = ap_init_queue(qid);
1091                 if (rc)
1092                         continue;
1093                 ap_dev = kzalloc(sizeof(*ap_dev), GFP_KERNEL);
1094                 if (!ap_dev)
1095                         break;
1096                 ap_dev->qid = qid;
1097                 ap_dev->queue_depth = queue_depth;
1098                 ap_dev->unregistered = 1;
1099                 spin_lock_init(&ap_dev->lock);
1100                 INIT_LIST_HEAD(&ap_dev->pendingq);
1101                 INIT_LIST_HEAD(&ap_dev->requestq);
1102                 INIT_LIST_HEAD(&ap_dev->list);
1103                 setup_timer(&ap_dev->timeout, ap_request_timeout,
1104                             (unsigned long) ap_dev);
1105                 if (device_type == 0)
1106                         ap_probe_device_type(ap_dev);
1107                 else
1108                         ap_dev->device_type = device_type;
1109
1110                 ap_dev->device.bus = &ap_bus_type;
1111                 ap_dev->device.parent = ap_root_device;
1112                 dev_set_name(&ap_dev->device, "card%02x",
1113                              AP_QID_DEVICE(ap_dev->qid));
1114                 ap_dev->device.release = ap_device_release;
1115                 rc = device_register(&ap_dev->device);
1116                 if (rc) {
1117                         kfree(ap_dev);
1118                         continue;
1119                 }
1120                 /* Add device attributes. */
1121                 rc = sysfs_create_group(&ap_dev->device.kobj,
1122                                         &ap_dev_attr_group);
1123                 if (!rc) {
1124                         spin_lock_bh(&ap_dev->lock);
1125                         ap_dev->unregistered = 0;
1126                         spin_unlock_bh(&ap_dev->lock);
1127                 }
1128                 else
1129                         device_unregister(&ap_dev->device);
1130         }
1131 }
1132
1133 static void
1134 ap_config_timeout(unsigned long ptr)
1135 {
1136         queue_work(ap_work_queue, &ap_config_work);
1137         ap_config_timer.expires = jiffies + ap_config_time * HZ;
1138         add_timer(&ap_config_timer);
1139 }
1140
1141 /**
1142  * ap_schedule_poll_timer(): Schedule poll timer.
1143  *
1144  * Set up the timer to run the poll tasklet
1145  */
1146 static inline void ap_schedule_poll_timer(void)
1147 {
1148         ktime_t hr_time;
1149         if (ap_using_interrupts() || ap_suspend_flag)
1150                 return;
1151         if (hrtimer_is_queued(&ap_poll_timer))
1152                 return;
1153         if (ktime_to_ns(hrtimer_expires_remaining(&ap_poll_timer)) <= 0) {
1154                 hr_time = ktime_set(0, poll_timeout);
1155                 hrtimer_forward_now(&ap_poll_timer, hr_time);
1156                 hrtimer_restart(&ap_poll_timer);
1157         }
1158         return;
1159 }
1160
1161 /**
1162  * ap_poll_read(): Receive pending reply messages from an AP device.
1163  * @ap_dev: pointer to the AP device
1164  * @flags: pointer to control flags, bit 2^0 is set if another poll is
1165  *         required, bit 2^1 is set if the poll timer needs to get armed
1166  *
1167  * Returns 0 if the device is still present, -ENODEV if not.
1168  */
1169 static int ap_poll_read(struct ap_device *ap_dev, unsigned long *flags)
1170 {
1171         struct ap_queue_status status;
1172         struct ap_message *ap_msg;
1173
1174         if (ap_dev->queue_count <= 0)
1175                 return 0;
1176         status = __ap_recv(ap_dev->qid, &ap_dev->reply->psmid,
1177                            ap_dev->reply->message, ap_dev->reply->length);
1178         switch (status.response_code) {
1179         case AP_RESPONSE_NORMAL:
1180                 atomic_dec(&ap_poll_requests);
1181                 ap_decrease_queue_count(ap_dev);
1182                 list_for_each_entry(ap_msg, &ap_dev->pendingq, list) {
1183                         if (ap_msg->psmid != ap_dev->reply->psmid)
1184                                 continue;
1185                         list_del_init(&ap_msg->list);
1186                         ap_dev->pendingq_count--;
1187                         ap_dev->drv->receive(ap_dev, ap_msg, ap_dev->reply);
1188                         break;
1189                 }
1190                 if (ap_dev->queue_count > 0)
1191                         *flags |= 1;
1192                 break;
1193         case AP_RESPONSE_NO_PENDING_REPLY:
1194                 if (status.queue_empty) {
1195                         /* The card shouldn't forget requests but who knows. */
1196                         atomic_sub(ap_dev->queue_count, &ap_poll_requests);
1197                         ap_dev->queue_count = 0;
1198                         list_splice_init(&ap_dev->pendingq, &ap_dev->requestq);
1199                         ap_dev->requestq_count += ap_dev->pendingq_count;
1200                         ap_dev->pendingq_count = 0;
1201                 } else
1202                         *flags |= 2;
1203                 break;
1204         default:
1205                 return -ENODEV;
1206         }
1207         return 0;
1208 }
1209
1210 /**
1211  * ap_poll_write(): Send messages from the request queue to an AP device.
1212  * @ap_dev: pointer to the AP device
1213  * @flags: pointer to control flags, bit 2^0 is set if another poll is
1214  *         required, bit 2^1 is set if the poll timer needs to get armed
1215  *
1216  * Returns 0 if the device is still present, -ENODEV if not.
1217  */
1218 static int ap_poll_write(struct ap_device *ap_dev, unsigned long *flags)
1219 {
1220         struct ap_queue_status status;
1221         struct ap_message *ap_msg;
1222
1223         if (ap_dev->requestq_count <= 0 ||
1224             ap_dev->queue_count >= ap_dev->queue_depth)
1225                 return 0;
1226         /* Start the next request on the queue. */
1227         ap_msg = list_entry(ap_dev->requestq.next, struct ap_message, list);
1228         status = __ap_send(ap_dev->qid, ap_msg->psmid,
1229                            ap_msg->message, ap_msg->length);
1230         switch (status.response_code) {
1231         case AP_RESPONSE_NORMAL:
1232                 atomic_inc(&ap_poll_requests);
1233                 ap_increase_queue_count(ap_dev);
1234                 list_move_tail(&ap_msg->list, &ap_dev->pendingq);
1235                 ap_dev->requestq_count--;
1236                 ap_dev->pendingq_count++;
1237                 if (ap_dev->queue_count < ap_dev->queue_depth &&
1238                     ap_dev->requestq_count > 0)
1239                         *flags |= 1;
1240                 *flags |= 2;
1241                 break;
1242         case AP_RESPONSE_Q_FULL:
1243         case AP_RESPONSE_RESET_IN_PROGRESS:
1244                 *flags |= 2;
1245                 break;
1246         case AP_RESPONSE_MESSAGE_TOO_BIG:
1247                 return -EINVAL;
1248         default:
1249                 return -ENODEV;
1250         }
1251         return 0;
1252 }
1253
1254 /**
1255  * ap_poll_queue(): Poll AP device for pending replies and send new messages.
1256  * @ap_dev: pointer to the bus device
1257  * @flags: pointer to control flags, bit 2^0 is set if another poll is
1258  *         required, bit 2^1 is set if the poll timer needs to get armed
1259  *
1260  * Poll AP device for pending replies and send new messages. If either
1261  * ap_poll_read or ap_poll_write returns -ENODEV unregister the device.
1262  * Returns 0.
1263  */
1264 static inline int ap_poll_queue(struct ap_device *ap_dev, unsigned long *flags)
1265 {
1266         int rc;
1267
1268         rc = ap_poll_read(ap_dev, flags);
1269         if (rc)
1270                 return rc;
1271         return ap_poll_write(ap_dev, flags);
1272 }
1273
1274 /**
1275  * __ap_queue_message(): Queue a message to a device.
1276  * @ap_dev: pointer to the AP device
1277  * @ap_msg: the message to be queued
1278  *
1279  * Queue a message to a device. Returns 0 if successful.
1280  */
1281 static int __ap_queue_message(struct ap_device *ap_dev, struct ap_message *ap_msg)
1282 {
1283         struct ap_queue_status status;
1284
1285         if (list_empty(&ap_dev->requestq) &&
1286             ap_dev->queue_count < ap_dev->queue_depth) {
1287                 status = __ap_send(ap_dev->qid, ap_msg->psmid,
1288                                    ap_msg->message, ap_msg->length);
1289                 switch (status.response_code) {
1290                 case AP_RESPONSE_NORMAL:
1291                         list_add_tail(&ap_msg->list, &ap_dev->pendingq);
1292                         atomic_inc(&ap_poll_requests);
1293                         ap_dev->pendingq_count++;
1294                         ap_increase_queue_count(ap_dev);
1295                         ap_dev->total_request_count++;
1296                         break;
1297                 case AP_RESPONSE_Q_FULL:
1298                 case AP_RESPONSE_RESET_IN_PROGRESS:
1299                         list_add_tail(&ap_msg->list, &ap_dev->requestq);
1300                         ap_dev->requestq_count++;
1301                         ap_dev->total_request_count++;
1302                         return -EBUSY;
1303                 case AP_RESPONSE_MESSAGE_TOO_BIG:
1304                         ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-EINVAL));
1305                         return -EINVAL;
1306                 default:        /* Device is gone. */
1307                         ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
1308                         return -ENODEV;
1309                 }
1310         } else {
1311                 list_add_tail(&ap_msg->list, &ap_dev->requestq);
1312                 ap_dev->requestq_count++;
1313                 ap_dev->total_request_count++;
1314                 return -EBUSY;
1315         }
1316         ap_schedule_poll_timer();
1317         return 0;
1318 }
1319
1320 void ap_queue_message(struct ap_device *ap_dev, struct ap_message *ap_msg)
1321 {
1322         unsigned long flags;
1323         int rc;
1324
1325         spin_lock_bh(&ap_dev->lock);
1326         if (!ap_dev->unregistered) {
1327                 /* Make room on the queue by polling for finished requests. */
1328                 rc = ap_poll_queue(ap_dev, &flags);
1329                 if (!rc)
1330                         rc = __ap_queue_message(ap_dev, ap_msg);
1331                 if (!rc)
1332                         wake_up(&ap_poll_wait);
1333                 if (rc == -ENODEV)
1334                         ap_dev->unregistered = 1;
1335         } else {
1336                 ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
1337                 rc = -ENODEV;
1338         }
1339         spin_unlock_bh(&ap_dev->lock);
1340         if (rc == -ENODEV)
1341                 device_unregister(&ap_dev->device);
1342 }
1343 EXPORT_SYMBOL(ap_queue_message);
1344
1345 /**
1346  * ap_cancel_message(): Cancel a crypto request.
1347  * @ap_dev: The AP device that has the message queued
1348  * @ap_msg: The message that is to be removed
1349  *
1350  * Cancel a crypto request. This is done by removing the request
1351  * from the device pending or request queue. Note that the
1352  * request stays on the AP queue. When it finishes the message
1353  * reply will be discarded because the psmid can't be found.
1354  */
1355 void ap_cancel_message(struct ap_device *ap_dev, struct ap_message *ap_msg)
1356 {
1357         struct ap_message *tmp;
1358
1359         spin_lock_bh(&ap_dev->lock);
1360         if (!list_empty(&ap_msg->list)) {
1361                 list_for_each_entry(tmp, &ap_dev->pendingq, list)
1362                         if (tmp->psmid == ap_msg->psmid) {
1363                                 ap_dev->pendingq_count--;
1364                                 goto found;
1365                         }
1366                 ap_dev->requestq_count--;
1367         found:
1368                 list_del_init(&ap_msg->list);
1369         }
1370         spin_unlock_bh(&ap_dev->lock);
1371 }
1372 EXPORT_SYMBOL(ap_cancel_message);
1373
1374 /**
1375  * ap_poll_timeout(): AP receive polling for finished AP requests.
1376  * @unused: Unused pointer.
1377  *
1378  * Schedules the AP tasklet using a high resolution timer.
1379  */
1380 static enum hrtimer_restart ap_poll_timeout(struct hrtimer *unused)
1381 {
1382         tasklet_schedule(&ap_tasklet);
1383         return HRTIMER_NORESTART;
1384 }
1385
1386 /**
1387  * ap_reset(): Reset a not responding AP device.
1388  * @ap_dev: Pointer to the AP device
1389  *
1390  * Reset a not responding AP device and move all requests from the
1391  * pending queue to the request queue.
1392  */
1393 static void ap_reset(struct ap_device *ap_dev)
1394 {
1395         int rc;
1396
1397         ap_dev->reset = AP_RESET_IGNORE;
1398         atomic_sub(ap_dev->queue_count, &ap_poll_requests);
1399         ap_dev->queue_count = 0;
1400         list_splice_init(&ap_dev->pendingq, &ap_dev->requestq);
1401         ap_dev->requestq_count += ap_dev->pendingq_count;
1402         ap_dev->pendingq_count = 0;
1403         rc = ap_init_queue(ap_dev->qid);
1404         if (rc == -ENODEV)
1405                 ap_dev->unregistered = 1;
1406 }
1407
1408 static int __ap_poll_device(struct ap_device *ap_dev, unsigned long *flags)
1409 {
1410         spin_lock(&ap_dev->lock);
1411         if (!ap_dev->unregistered) {
1412                 if (ap_poll_queue(ap_dev, flags))
1413                         ap_dev->unregistered = 1;
1414                 if (ap_dev->reset == AP_RESET_DO)
1415                         ap_reset(ap_dev);
1416         }
1417         spin_unlock(&ap_dev->lock);
1418         return 0;
1419 }
1420
1421 /**
1422  * ap_poll_all(): Poll all AP devices.
1423  * @dummy: Unused variable
1424  *
1425  * Poll all AP devices on the bus in a round robin fashion. Continue
1426  * polling until bit 2^0 of the control flags is not set. If bit 2^1
1427  * of the control flags has been set arm the poll timer.
1428  */
1429 static void ap_poll_all(unsigned long dummy)
1430 {
1431         unsigned long flags;
1432         struct ap_device *ap_dev;
1433
1434         /* Reset the indicator if interrupts are used. Thus new interrupts can
1435          * be received. Doing it in the beginning of the tasklet is therefor
1436          * important that no requests on any AP get lost.
1437          */
1438         if (ap_using_interrupts())
1439                 xchg((u8 *)ap_interrupt_indicator, 0);
1440         do {
1441                 flags = 0;
1442                 spin_lock(&ap_device_list_lock);
1443                 list_for_each_entry(ap_dev, &ap_device_list, list) {
1444                         __ap_poll_device(ap_dev, &flags);
1445                 }
1446                 spin_unlock(&ap_device_list_lock);
1447         } while (flags & 1);
1448         if (flags & 2)
1449                 ap_schedule_poll_timer();
1450 }
1451
1452 /**
1453  * ap_poll_thread(): Thread that polls for finished requests.
1454  * @data: Unused pointer
1455  *
1456  * AP bus poll thread. The purpose of this thread is to poll for
1457  * finished requests in a loop if there is a "free" cpu - that is
1458  * a cpu that doesn't have anything better to do. The polling stops
1459  * as soon as there is another task or if all messages have been
1460  * delivered.
1461  */
1462 static int ap_poll_thread(void *data)
1463 {
1464         DECLARE_WAITQUEUE(wait, current);
1465         unsigned long flags;
1466         int requests;
1467         struct ap_device *ap_dev;
1468
1469         set_user_nice(current, 19);
1470         while (1) {
1471                 if (ap_suspend_flag)
1472                         return 0;
1473                 if (need_resched()) {
1474                         schedule();
1475                         continue;
1476                 }
1477                 add_wait_queue(&ap_poll_wait, &wait);
1478                 set_current_state(TASK_INTERRUPTIBLE);
1479                 if (kthread_should_stop())
1480                         break;
1481                 requests = atomic_read(&ap_poll_requests);
1482                 if (requests <= 0)
1483                         schedule();
1484                 set_current_state(TASK_RUNNING);
1485                 remove_wait_queue(&ap_poll_wait, &wait);
1486
1487                 flags = 0;
1488                 spin_lock_bh(&ap_device_list_lock);
1489                 list_for_each_entry(ap_dev, &ap_device_list, list) {
1490                         __ap_poll_device(ap_dev, &flags);
1491                 }
1492                 spin_unlock_bh(&ap_device_list_lock);
1493         }
1494         set_current_state(TASK_RUNNING);
1495         remove_wait_queue(&ap_poll_wait, &wait);
1496         return 0;
1497 }
1498
1499 static int ap_poll_thread_start(void)
1500 {
1501         int rc;
1502
1503         if (ap_using_interrupts() || ap_suspend_flag)
1504                 return 0;
1505         mutex_lock(&ap_poll_thread_mutex);
1506         if (!ap_poll_kthread) {
1507                 ap_poll_kthread = kthread_run(ap_poll_thread, NULL, "appoll");
1508                 rc = IS_ERR(ap_poll_kthread) ? PTR_ERR(ap_poll_kthread) : 0;
1509                 if (rc)
1510                         ap_poll_kthread = NULL;
1511         }
1512         else
1513                 rc = 0;
1514         mutex_unlock(&ap_poll_thread_mutex);
1515         return rc;
1516 }
1517
1518 static void ap_poll_thread_stop(void)
1519 {
1520         mutex_lock(&ap_poll_thread_mutex);
1521         if (ap_poll_kthread) {
1522                 kthread_stop(ap_poll_kthread);
1523                 ap_poll_kthread = NULL;
1524         }
1525         mutex_unlock(&ap_poll_thread_mutex);
1526 }
1527
1528 /**
1529  * ap_request_timeout(): Handling of request timeouts
1530  * @data: Holds the AP device.
1531  *
1532  * Handles request timeouts.
1533  */
1534 static void ap_request_timeout(unsigned long data)
1535 {
1536         struct ap_device *ap_dev = (struct ap_device *) data;
1537
1538         if (ap_dev->reset == AP_RESET_ARMED) {
1539                 ap_dev->reset = AP_RESET_DO;
1540
1541                 if (ap_using_interrupts())
1542                         tasklet_schedule(&ap_tasklet);
1543         }
1544 }
1545
1546 static void ap_reset_domain(void)
1547 {
1548         int i;
1549
1550         if (ap_domain_index != -1)
1551                 for (i = 0; i < AP_DEVICES; i++)
1552                         ap_reset_queue(AP_MKQID(i, ap_domain_index));
1553 }
1554
1555 static void ap_reset_all(void)
1556 {
1557         int i, j;
1558
1559         for (i = 0; i < AP_DOMAINS; i++)
1560                 for (j = 0; j < AP_DEVICES; j++)
1561                         ap_reset_queue(AP_MKQID(j, i));
1562 }
1563
1564 static struct reset_call ap_reset_call = {
1565         .fn = ap_reset_all,
1566 };
1567
1568 /**
1569  * ap_module_init(): The module initialization code.
1570  *
1571  * Initializes the module.
1572  */
1573 int __init ap_module_init(void)
1574 {
1575         int rc, i;
1576
1577         if (ap_domain_index < -1 || ap_domain_index >= AP_DOMAINS) {
1578                 pr_warning("%d is not a valid cryptographic domain\n",
1579                            ap_domain_index);
1580                 return -EINVAL;
1581         }
1582         if (ap_instructions_available() != 0) {
1583                 pr_warning("The hardware system does not support "
1584                            "AP instructions\n");
1585                 return -ENODEV;
1586         }
1587         if (ap_interrupts_available()) {
1588                 isc_register(AP_ISC);
1589                 ap_interrupt_indicator = s390_register_adapter_interrupt(
1590                         &ap_interrupt_handler, NULL, AP_ISC);
1591                 if (IS_ERR(ap_interrupt_indicator)) {
1592                         ap_interrupt_indicator = NULL;
1593                         isc_unregister(AP_ISC);
1594                 }
1595         }
1596
1597         register_reset_call(&ap_reset_call);
1598
1599         /* Create /sys/bus/ap. */
1600         rc = bus_register(&ap_bus_type);
1601         if (rc)
1602                 goto out;
1603         for (i = 0; ap_bus_attrs[i]; i++) {
1604                 rc = bus_create_file(&ap_bus_type, ap_bus_attrs[i]);
1605                 if (rc)
1606                         goto out_bus;
1607         }
1608
1609         /* Create /sys/devices/ap. */
1610         ap_root_device = root_device_register("ap");
1611         rc = IS_ERR(ap_root_device) ? PTR_ERR(ap_root_device) : 0;
1612         if (rc)
1613                 goto out_bus;
1614
1615         ap_work_queue = create_singlethread_workqueue("kapwork");
1616         if (!ap_work_queue) {
1617                 rc = -ENOMEM;
1618                 goto out_root;
1619         }
1620
1621         if (ap_select_domain() == 0)
1622                 ap_scan_bus(NULL);
1623
1624         /* Setup the AP bus rescan timer. */
1625         init_timer(&ap_config_timer);
1626         ap_config_timer.function = ap_config_timeout;
1627         ap_config_timer.data = 0;
1628         ap_config_timer.expires = jiffies + ap_config_time * HZ;
1629         add_timer(&ap_config_timer);
1630
1631         /* Setup the high resultion poll timer.
1632          * If we are running under z/VM adjust polling to z/VM polling rate.
1633          */
1634         if (MACHINE_IS_VM)
1635                 poll_timeout = 1500000;
1636         hrtimer_init(&ap_poll_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1637         ap_poll_timer.function = ap_poll_timeout;
1638
1639         /* Start the low priority AP bus poll thread. */
1640         if (ap_thread_flag) {
1641                 rc = ap_poll_thread_start();
1642                 if (rc)
1643                         goto out_work;
1644         }
1645
1646         return 0;
1647
1648 out_work:
1649         del_timer_sync(&ap_config_timer);
1650         hrtimer_cancel(&ap_poll_timer);
1651         destroy_workqueue(ap_work_queue);
1652 out_root:
1653         root_device_unregister(ap_root_device);
1654 out_bus:
1655         while (i--)
1656                 bus_remove_file(&ap_bus_type, ap_bus_attrs[i]);
1657         bus_unregister(&ap_bus_type);
1658 out:
1659         unregister_reset_call(&ap_reset_call);
1660         if (ap_using_interrupts()) {
1661                 s390_unregister_adapter_interrupt(ap_interrupt_indicator, AP_ISC);
1662                 isc_unregister(AP_ISC);
1663         }
1664         return rc;
1665 }
1666
1667 static int __ap_match_all(struct device *dev, void *data)
1668 {
1669         return 1;
1670 }
1671
1672 /**
1673  * ap_modules_exit(): The module termination code
1674  *
1675  * Terminates the module.
1676  */
1677 void ap_module_exit(void)
1678 {
1679         int i;
1680         struct device *dev;
1681
1682         ap_reset_domain();
1683         ap_poll_thread_stop();
1684         del_timer_sync(&ap_config_timer);
1685         hrtimer_cancel(&ap_poll_timer);
1686         destroy_workqueue(ap_work_queue);
1687         tasklet_kill(&ap_tasklet);
1688         root_device_unregister(ap_root_device);
1689         while ((dev = bus_find_device(&ap_bus_type, NULL, NULL,
1690                     __ap_match_all)))
1691         {
1692                 device_unregister(dev);
1693                 put_device(dev);
1694         }
1695         for (i = 0; ap_bus_attrs[i]; i++)
1696                 bus_remove_file(&ap_bus_type, ap_bus_attrs[i]);
1697         bus_unregister(&ap_bus_type);
1698         unregister_reset_call(&ap_reset_call);
1699         if (ap_using_interrupts()) {
1700                 s390_unregister_adapter_interrupt(ap_interrupt_indicator, AP_ISC);
1701                 isc_unregister(AP_ISC);
1702         }
1703 }
1704
1705 #ifndef CONFIG_ZCRYPT_MONOLITHIC
1706 module_init(ap_module_init);
1707 module_exit(ap_module_exit);
1708 #endif