drivers/s390/crypto/ap_bus.c

   1 /*
   2  * linux/drivers/s390/crypto/ap_bus.c
   3  *
   4  * Copyright (C) 2006 IBM Corporation
   5  * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
   6  *            Martin Schwidefsky <schwidefsky@de.ibm.com>
   7  *            Ralph Wuerthner <rwuerthn@de.ibm.com>
   8  *            Felix Beck <felix.beck@de.ibm.com>
   9  *
  10  * Adjunct processor bus.
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License as published by
  14  * the Free Software Foundation; either version 2, or (at your option)
  15  * any later version.
  16  *
  17  * This program is distributed in the hope that it will be useful,
  18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  20  * GNU General Public License for more details.
  21  *
  22  * You should have received a copy of the GNU General Public License
  23  * along with this program; if not, write to the Free Software
  24  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  25  */
  26
  27 #define KMSG_COMPONENT "ap"
  28 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  29
  30 #include <linux/module.h>
  31 #include <linux/init.h>
  32 #include <linux/delay.h>
  33 #include <linux/err.h>
  34 #include <linux/interrupt.h>
  35 #include <linux/workqueue.h>
  36 #include <linux/notifier.h>
  37 #include <linux/kthread.h>
  38 #include <linux/mutex.h>
  39 #include <asm/reset.h>
  40 #include <asm/airq.h>
  41 #include <asm/atomic.h>
  42 #include <asm/system.h>
  43 #include <asm/isc.h>
  44 #include <linux/hrtimer.h>
  45 #include <linux/ktime.h>
  46
  47 #include "ap_bus.h"
  48
  49 /* Some prototypes. */
  50 static void ap_scan_bus(struct work_struct *);
  51 static void ap_poll_all(unsigned long);
  52 static enum hrtimer_restart ap_poll_timeout(struct hrtimer *);
  53 static int ap_poll_thread_start(void);
  54 static void ap_poll_thread_stop(void);
  55 static void ap_request_timeout(unsigned long);
  56 static inline void ap_schedule_poll_timer(void);
  57 static int __ap_poll_device(struct ap_device *ap_dev, unsigned long *flags);
  58 static int ap_device_remove(struct device *dev);
  59 static int ap_device_probe(struct device *dev);
  60 static void ap_interrupt_handler(void *unused1, void *unused2);
  61 static void ap_reset(struct ap_device *ap_dev);
  62 static void ap_config_timeout(unsigned long ptr);
  63
  64 /*
  65  * Module description.
  66  */
  67 MODULE_AUTHOR("IBM Corporation");
  68 MODULE_DESCRIPTION("Adjunct Processor Bus driver, "
  69                    "Copyright 2006 IBM Corporation");
  70 MODULE_LICENSE("GPL");
  71
  72 /*
  73  * Module parameter
  74  */
  75 int ap_domain_index = -1;       /* Adjunct Processor Domain Index */
  76 module_param_named(domain, ap_domain_index, int, 0000);
  77 MODULE_PARM_DESC(domain, "domain index for ap devices");
  78 EXPORT_SYMBOL(ap_domain_index);
  79
  80 static int ap_thread_flag = 0;
  81 module_param_named(poll_thread, ap_thread_flag, int, 0000);
  82 MODULE_PARM_DESC(poll_thread, "Turn on/off poll thread, default is 0 (off).");
  83
  84 static struct device *ap_root_device = NULL;
  85 static DEFINE_SPINLOCK(ap_device_list_lock);
  86 static LIST_HEAD(ap_device_list);
  87
  88 /*
  89  * Workqueue & timer for bus rescan.
  90  */
  91 static struct workqueue_struct *ap_work_queue;
  92 static struct timer_list ap_config_timer;
  93 static int ap_config_time = AP_CONFIG_TIME;
  94 static DECLARE_WORK(ap_config_work, ap_scan_bus);
  95
  96 /*
  97  * Tasklet & timer for AP request polling and interrupts
  98  */
  99 static DECLARE_TASKLET(ap_tasklet, ap_poll_all, 0);
 100 static atomic_t ap_poll_requests = ATOMIC_INIT(0);
 101 static DECLARE_WAIT_QUEUE_HEAD(ap_poll_wait);
 102 static struct task_struct *ap_poll_kthread = NULL;
 103 static DEFINE_MUTEX(ap_poll_thread_mutex);
 104 static void *ap_interrupt_indicator;
 105 static struct hrtimer ap_poll_timer;
 106 /* In LPAR poll with 4kHz frequency. Poll every 250000 nanoseconds.
 107  * If z/VM change to 1500000 nanoseconds to adjust to z/VM polling.*/
 108 static unsigned long long poll_timeout = 250000;
 109
 110 /* Suspend flag */
 111 static int ap_suspend_flag;
 112 static struct bus_type ap_bus_type;
 113
 114 /**
 115  * ap_using_interrupts() - Returns non-zero if interrupt support is
 116  * available.
 117  */
 118 static inline int ap_using_interrupts(void)
 119 {
 120         return ap_interrupt_indicator != NULL;
 121 }
 122
 123 /**
 124  * ap_intructions_available() - Test if AP instructions are available.
 125  *
 126  * Returns 0 if the AP instructions are installed.
 127  */
 128 static inline int ap_instructions_available(void)
 129 {
 130         register unsigned long reg0 asm ("0") = AP_MKQID(0,0);
 131         register unsigned long reg1 asm ("1") = -ENODEV;
 132         register unsigned long reg2 asm ("2") = 0UL;
 133
 134         asm volatile(
 135                 "   .long 0xb2af0000\n"         /* PQAP(TAPQ) */
 136                 "0: la    %1,0\n"
 137                 "1:\n"
 138                 EX_TABLE(0b, 1b)
 139                 : "+d" (reg0), "+d" (reg1), "+d" (reg2) : : "cc" );
 140         return reg1;
 141 }
 142
 143 /**
 144  * ap_interrupts_available(): Test if AP interrupts are available.
 145  *
 146  * Returns 1 if AP interrupts are available.
 147  */
 148 static int ap_interrupts_available(void)
 149 {
 150         unsigned long long facility_bits[2];
 151
 152         if (stfle(facility_bits, 2) <= 1)
 153                 return 0;
 154         if (!(facility_bits[0] & (1ULL << 61)) ||
 155             !(facility_bits[1] & (1ULL << 62)))
 156                 return 0;
 157         return 1;
 158 }
 159
 160 /**
 161  * ap_test_queue(): Test adjunct processor queue.
 162  * @qid: The AP queue number
 163  * @queue_depth: Pointer to queue depth value
 164  * @device_type: Pointer to device type value
 165  *
 166  * Returns AP queue status structure.
 167  */
 168 static inline struct ap_queue_status
 169 ap_test_queue(ap_qid_t qid, int *queue_depth, int *device_type)
 170 {
 171         register unsigned long reg0 asm ("0") = qid;
 172         register struct ap_queue_status reg1 asm ("1");
 173         register unsigned long reg2 asm ("2") = 0UL;
 174
 175         asm volatile(".long 0xb2af0000"         /* PQAP(TAPQ) */
 176                      : "+d" (reg0), "=d" (reg1), "+d" (reg2) : : "cc");
 177         *device_type = (int) (reg2 >> 24);
 178         *queue_depth = (int) (reg2 & 0xff);
 179         return reg1;
 180 }
 181
 182 /**
 183  * ap_reset_queue(): Reset adjunct processor queue.
 184  * @qid: The AP queue number
 185  *
 186  * Returns AP queue status structure.
 187  */
 188 static inline struct ap_queue_status ap_reset_queue(ap_qid_t qid)
 189 {
 190         register unsigned long reg0 asm ("0") = qid | 0x01000000UL;
 191         register struct ap_queue_status reg1 asm ("1");
 192         register unsigned long reg2 asm ("2") = 0UL;
 193
 194         asm volatile(
 195                 ".long 0xb2af0000"              /* PQAP(RAPQ) */
 196                 : "+d" (reg0), "=d" (reg1), "+d" (reg2) : : "cc");
 197         return reg1;
 198 }
 199
 200 #ifdef CONFIG_64BIT
 201 /**
 202  * ap_queue_interruption_control(): Enable interruption for a specific AP.
 203  * @qid: The AP queue number
 204  * @ind: The notification indicator byte
 205  *
 206  * Returns AP queue status.
 207  */
 208 static inline struct ap_queue_status
 209 ap_queue_interruption_control(ap_qid_t qid, void *ind)
 210 {
 211         register unsigned long reg0 asm ("0") = qid | 0x03000000UL;
 212         register unsigned long reg1_in asm ("1") = 0x0000800000000000UL | AP_ISC;
 213         register struct ap_queue_status reg1_out asm ("1");
 214         register void *reg2 asm ("2") = ind;
 215         asm volatile(
 216                 ".long 0xb2af0000"              /* PQAP(RAPQ) */
 217                 : "+d" (reg0), "+d" (reg1_in), "=d" (reg1_out), "+d" (reg2)
 218                 :
 219                 : "cc" );
 220         return reg1_out;
 221 }
 222 #endif
 223
 224 /**
 225  * ap_queue_enable_interruption(): Enable interruption on an AP.
 226  * @qid: The AP queue number
 227  * @ind: the notification indicator byte
 228  *
 229  * Enables interruption on AP queue via ap_queue_interruption_control(). Based
 230  * on the return value it waits a while and tests the AP queue if interrupts
 231  * have been switched on using ap_test_queue().
 232  */
 233 static int ap_queue_enable_interruption(ap_qid_t qid, void *ind)
 234 {
 235 #ifdef CONFIG_64BIT
 236         struct ap_queue_status status;
 237         int t_depth, t_device_type, rc, i;
 238
 239         rc = -EBUSY;
 240         status = ap_queue_interruption_control(qid, ind);
 241
 242         for (i = 0; i < AP_MAX_RESET; i++) {
 243                 switch (status.response_code) {
 244                 case AP_RESPONSE_NORMAL:
 245                         if (status.int_enabled)
 246                                 return 0;
 247                         break;
 248                 case AP_RESPONSE_RESET_IN_PROGRESS:
 249                 case AP_RESPONSE_BUSY:
 250                         break;
 251                 case AP_RESPONSE_Q_NOT_AVAIL:
 252                 case AP_RESPONSE_DECONFIGURED:
 253                 case AP_RESPONSE_CHECKSTOPPED:
 254                 case AP_RESPONSE_INVALID_ADDRESS:
 255                         return -ENODEV;
 256                 case AP_RESPONSE_OTHERWISE_CHANGED:
 257                         if (status.int_enabled)
 258                                 return 0;
 259                         break;
 260                 default:
 261                         break;
 262                 }
 263                 if (i < AP_MAX_RESET - 1) {
 264                         udelay(5);
 265                         status = ap_test_queue(qid, &t_depth, &t_device_type);
 266                 }
 267         }
 268         return rc;
 269 #else
 270         return -EINVAL;
 271 #endif
 272 }
 273
 274 /**
 275  * __ap_send(): Send message to adjunct processor queue.
 276  * @qid: The AP queue number
 277  * @psmid: The program supplied message identifier
 278  * @msg: The message text
 279  * @length: The message length
 280  *
 281  * Returns AP queue status structure.
 282  * Condition code 1 on NQAP can't happen because the L bit is 1.
 283  * Condition code 2 on NQAP also means the send is incomplete,
 284  * because a segment boundary was reached. The NQAP is repeated.
 285  */
 286 static inline struct ap_queue_status
 287 __ap_send(ap_qid_t qid, unsigned long long psmid, void *msg, size_t length)
 288 {
 289         typedef struct { char _[length]; } msgblock;
 290         register unsigned long reg0 asm ("0") = qid | 0x40000000UL;
 291         register struct ap_queue_status reg1 asm ("1");
 292         register unsigned long reg2 asm ("2") = (unsigned long) msg;
 293         register unsigned long reg3 asm ("3") = (unsigned long) length;
 294         register unsigned long reg4 asm ("4") = (unsigned int) (psmid >> 32);
 295         register unsigned long reg5 asm ("5") = (unsigned int) psmid;
 296
 297         asm volatile (
 298                 "0: .long 0xb2ad0042\n"         /* DQAP */
 299                 "   brc   2,0b"
 300                 : "+d" (reg0), "=d" (reg1), "+d" (reg2), "+d" (reg3)
 301                 : "d" (reg4), "d" (reg5), "m" (*(msgblock *) msg)
 302                 : "cc" );
 303         return reg1;
 304 }
 305
 306 int ap_send(ap_qid_t qid, unsigned long long psmid, void *msg, size_t length)
 307 {
 308         struct ap_queue_status status;
 309
 310         status = __ap_send(qid, psmid, msg, length);
 311         switch (status.response_code) {
 312         case AP_RESPONSE_NORMAL:
 313                 return 0;
 314         case AP_RESPONSE_Q_FULL:
 315         case AP_RESPONSE_RESET_IN_PROGRESS:
 316                 return -EBUSY;
 317         default:        /* Device is gone. */
 318                 return -ENODEV;
 319         }
 320 }
 321 EXPORT_SYMBOL(ap_send);
 322
 323 /**
 324  * __ap_recv(): Receive message from adjunct processor queue.
 325  * @qid: The AP queue number
 326  * @psmid: Pointer to program supplied message identifier
 327  * @msg: The message text
 328  * @length: The message length
 329  *
 330  * Returns AP queue status structure.
 331  * Condition code 1 on DQAP means the receive has taken place
 332  * but only partially.  The response is incomplete, hence the
 333  * DQAP is repeated.
 334  * Condition code 2 on DQAP also means the receive is incomplete,
 335  * this time because a segment boundary was reached. Again, the
 336  * DQAP is repeated.
 337  * Note that gpr2 is used by the DQAP instruction to keep track of
 338  * any 'residual' length, in case the instruction gets interrupted.
 339  * Hence it gets zeroed before the instruction.
 340  */
 341 static inline struct ap_queue_status
 342 __ap_recv(ap_qid_t qid, unsigned long long *psmid, void *msg, size_t length)
 343 {
 344         typedef struct { char _[length]; } msgblock;
 345         register unsigned long reg0 asm("0") = qid | 0x80000000UL;
 346         register struct ap_queue_status reg1 asm ("1");
 347         register unsigned long reg2 asm("2") = 0UL;
 348         register unsigned long reg4 asm("4") = (unsigned long) msg;
 349         register unsigned long reg5 asm("5") = (unsigned long) length;
 350         register unsigned long reg6 asm("6") = 0UL;
 351         register unsigned long reg7 asm("7") = 0UL;
 352
 353
 354         asm volatile(
 355                 "0: .long 0xb2ae0064\n"
 356                 "   brc   6,0b\n"
 357                 : "+d" (reg0), "=d" (reg1), "+d" (reg2),
 358                 "+d" (reg4), "+d" (reg5), "+d" (reg6), "+d" (reg7),
 359                 "=m" (*(msgblock *) msg) : : "cc" );
 360         *psmid = (((unsigned long long) reg6) << 32) + reg7;
 361         return reg1;
 362 }
 363
 364 int ap_recv(ap_qid_t qid, unsigned long long *psmid, void *msg, size_t length)
 365 {
 366         struct ap_queue_status status;
 367
 368         status = __ap_recv(qid, psmid, msg, length);
 369         switch (status.response_code) {
 370         case AP_RESPONSE_NORMAL:
 371                 return 0;
 372         case AP_RESPONSE_NO_PENDING_REPLY:
 373                 if (status.queue_empty)
 374                         return -ENOENT;
 375                 return -EBUSY;
 376         case AP_RESPONSE_RESET_IN_PROGRESS:
 377                 return -EBUSY;
 378         default:
 379                 return -ENODEV;
 380         }
 381 }
 382 EXPORT_SYMBOL(ap_recv);
 383
 384 /**
 385  * ap_query_queue(): Check if an AP queue is available.
 386  * @qid: The AP queue number
 387  * @queue_depth: Pointer to queue depth value
 388  * @device_type: Pointer to device type value
 389  *
 390  * The test is repeated for AP_MAX_RESET times.
 391  */
 392 static int ap_query_queue(ap_qid_t qid, int *queue_depth, int *device_type)
 393 {
 394         struct ap_queue_status status;
 395         int t_depth, t_device_type, rc, i;
 396
 397         rc = -EBUSY;
 398         for (i = 0; i < AP_MAX_RESET; i++) {
 399                 status = ap_test_queue(qid, &t_depth, &t_device_type);
 400                 switch (status.response_code) {
 401                 case AP_RESPONSE_NORMAL:
 402                         *queue_depth = t_depth + 1;
 403                         *device_type = t_device_type;
 404                         rc = 0;
 405                         break;
 406                 case AP_RESPONSE_Q_NOT_AVAIL:
 407                         rc = -ENODEV;
 408                         break;
 409                 case AP_RESPONSE_RESET_IN_PROGRESS:
 410                         break;
 411                 case AP_RESPONSE_DECONFIGURED:
 412                         rc = -ENODEV;
 413                         break;
 414                 case AP_RESPONSE_CHECKSTOPPED:
 415                         rc = -ENODEV;
 416                         break;
 417                 case AP_RESPONSE_INVALID_ADDRESS:
 418                         rc = -ENODEV;
 419                         break;
 420                 case AP_RESPONSE_OTHERWISE_CHANGED:
 421                         break;
 422                 case AP_RESPONSE_BUSY:
 423                         break;
 424                 default:
 425                         BUG();
 426                 }
 427                 if (rc != -EBUSY)
 428                         break;
 429                 if (i < AP_MAX_RESET - 1)
 430                         udelay(5);
 431         }
 432         return rc;
 433 }
 434
 435 /**
 436  * ap_init_queue(): Reset an AP queue.
 437  * @qid: The AP queue number
 438  *
 439  * Reset an AP queue and wait for it to become available again.
 440  */
 441 static int ap_init_queue(ap_qid_t qid)
 442 {
 443         struct ap_queue_status status;
 444         int rc, dummy, i;
 445
 446         rc = -ENODEV;
 447         status = ap_reset_queue(qid);
 448         for (i = 0; i < AP_MAX_RESET; i++) {
 449                 switch (status.response_code) {
 450                 case AP_RESPONSE_NORMAL:
 451                         if (status.queue_empty)
 452                                 rc = 0;
 453                         break;
 454                 case AP_RESPONSE_Q_NOT_AVAIL:
 455                 case AP_RESPONSE_DECONFIGURED:
 456                 case AP_RESPONSE_CHECKSTOPPED:
 457                         i = AP_MAX_RESET;       /* return with -ENODEV */
 458                         break;
 459                 case AP_RESPONSE_RESET_IN_PROGRESS:
 460                         rc = -EBUSY;
 461                 case AP_RESPONSE_BUSY:
 462                 default:
 463                         break;
 464                 }
 465                 if (rc != -ENODEV && rc != -EBUSY)
 466                         break;
 467                 if (i < AP_MAX_RESET - 1) {
 468                         udelay(5);
 469                         status = ap_test_queue(qid, &dummy, &dummy);
 470                 }
 471         }
 472         if (rc == 0 && ap_using_interrupts()) {
 473                 rc = ap_queue_enable_interruption(qid, ap_interrupt_indicator);
 474                 /* If interruption mode is supported by the machine,
 475                 * but an AP can not be enabled for interruption then
 476                 * the AP will be discarded.    */
 477                 if (rc)
 478                         pr_err("Registering adapter interrupts for "
 479                                "AP %d failed\n", AP_QID_DEVICE(qid));
 480         }
 481         return rc;
 482 }
 483
 484 /**
 485  * ap_increase_queue_count(): Arm request timeout.
 486  * @ap_dev: Pointer to an AP device.
 487  *
 488  * Arm request timeout if an AP device was idle and a new request is submitted.
 489  */
 490 static void ap_increase_queue_count(struct ap_device *ap_dev)
 491 {
 492         int timeout = ap_dev->drv->request_timeout;
 493
 494         ap_dev->queue_count++;
 495         if (ap_dev->queue_count == 1) {
 496                 mod_timer(&ap_dev->timeout, jiffies + timeout);
 497                 ap_dev->reset = AP_RESET_ARMED;
 498         }
 499 }
 500
 501 /**
 502  * ap_decrease_queue_count(): Decrease queue count.
 503  * @ap_dev: Pointer to an AP device.
 504  *
 505  * If AP device is still alive, re-schedule request timeout if there are still
 506  * pending requests.
 507  */
 508 static void ap_decrease_queue_count(struct ap_device *ap_dev)
 509 {
 510         int timeout = ap_dev->drv->request_timeout;
 511
 512         ap_dev->queue_count--;
 513         if (ap_dev->queue_count > 0)
 514                 mod_timer(&ap_dev->timeout, jiffies + timeout);
 515         else
 516                 /*
 517                  * The timeout timer should to be disabled now - since
 518                  * del_timer_sync() is very expensive, we just tell via the
 519                  * reset flag to ignore the pending timeout timer.
 520                  */
 521                 ap_dev->reset = AP_RESET_IGNORE;
 522 }
 523
 524 /*
 525  * AP device related attributes.
 526  */
 527 static ssize_t ap_hwtype_show(struct device *dev,
 528                               struct device_attribute *attr, char *buf)
 529 {
 530         struct ap_device *ap_dev = to_ap_dev(dev);
 531         return snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->device_type);
 532 }
 533
 534 static DEVICE_ATTR(hwtype, 0444, ap_hwtype_show, NULL);
 535 static ssize_t ap_depth_show(struct device *dev, struct device_attribute *attr,
 536                              char *buf)
 537 {
 538         struct ap_device *ap_dev = to_ap_dev(dev);
 539         return snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->queue_depth);
 540 }
 541
 542 static DEVICE_ATTR(depth, 0444, ap_depth_show, NULL);
 543 static ssize_t ap_request_count_show(struct device *dev,
 544                                      struct device_attribute *attr,
 545                                      char *buf)
 546 {
 547         struct ap_device *ap_dev = to_ap_dev(dev);
 548         int rc;
 549
 550         spin_lock_bh(&ap_dev->lock);
 551         rc = snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->total_request_count);
 552         spin_unlock_bh(&ap_dev->lock);
 553         return rc;
 554 }
 555
 556 static DEVICE_ATTR(request_count, 0444, ap_request_count_show, NULL);
 557
 558 static ssize_t ap_modalias_show(struct device *dev,
 559                                 struct device_attribute *attr, char *buf)
 560 {
 561         return sprintf(buf, "ap:t%02X", to_ap_dev(dev)->device_type);
 562 }
 563
 564 static DEVICE_ATTR(modalias, 0444, ap_modalias_show, NULL);
 565
 566 static struct attribute *ap_dev_attrs[] = {
 567         &dev_attr_hwtype.attr,
 568         &dev_attr_depth.attr,
 569         &dev_attr_request_count.attr,
 570         &dev_attr_modalias.attr,
 571         NULL
 572 };
 573 static struct attribute_group ap_dev_attr_group = {
 574         .attrs = ap_dev_attrs
 575 };
 576
 577 /**
 578  * ap_bus_match()
 579  * @dev: Pointer to device
 580  * @drv: Pointer to device_driver
 581  *
 582  * AP bus driver registration/unregistration.
 583  */
 584 static int ap_bus_match(struct device *dev, struct device_driver *drv)
 585 {
 586         struct ap_device *ap_dev = to_ap_dev(dev);
 587         struct ap_driver *ap_drv = to_ap_drv(drv);
 588         struct ap_device_id *id;
 589
 590         /*
 591          * Compare device type of the device with the list of
 592          * supported types of the device_driver.
 593          */
 594         for (id = ap_drv->ids; id->match_flags; id++) {
 595                 if ((id->match_flags & AP_DEVICE_ID_MATCH_DEVICE_TYPE) &&
 596                     (id->dev_type != ap_dev->device_type))
 597                         continue;
 598                 return 1;
 599         }
 600         return 0;
 601 }
 602
 603 /**
 604  * ap_uevent(): Uevent function for AP devices.
 605  * @dev: Pointer to device
 606  * @env: Pointer to kobj_uevent_env
 607  *
 608  * It sets up a single environment variable DEV_TYPE which contains the
 609  * hardware device type.
 610  */
 611 static int ap_uevent (struct device *dev, struct kobj_uevent_env *env)
 612 {
 613         struct ap_device *ap_dev = to_ap_dev(dev);
 614         int retval = 0;
 615
 616         if (!ap_dev)
 617                 return -ENODEV;
 618
 619         /* Set up DEV_TYPE environment variable. */
 620         retval = add_uevent_var(env, "DEV_TYPE=%04X", ap_dev->device_type);
 621         if (retval)
 622                 return retval;
 623
 624         /* Add MODALIAS= */
 625         retval = add_uevent_var(env, "MODALIAS=ap:t%02X", ap_dev->device_type);
 626
 627         return retval;
 628 }
 629
 630 static int ap_bus_suspend(struct device *dev, pm_message_t state)
 631 {
 632         struct ap_device *ap_dev = to_ap_dev(dev);
 633         unsigned long flags;
 634
 635         if (!ap_suspend_flag) {
 636                 ap_suspend_flag = 1;
 637
 638                 /* Disable scanning for devices, thus we do not want to scan
 639                  * for them after removing.
 640                  */
 641                 del_timer_sync(&ap_config_timer);
 642                 if (ap_work_queue != NULL) {
 643                         destroy_workqueue(ap_work_queue);
 644                         ap_work_queue = NULL;
 645                 }
 646                 tasklet_disable(&ap_tasklet);
 647         }
 648         /* Poll on the device until all requests are finished. */
 649         do {
 650                 flags = 0;
 651                 spin_lock_bh(&ap_dev->lock);
 652                 __ap_poll_device(ap_dev, &flags);
 653                 spin_unlock_bh(&ap_dev->lock);
 654         } while ((flags & 1) || (flags & 2));
 655
 656         ap_device_remove(dev);
 657         return 0;
 658 }
 659
 660 static int ap_bus_resume(struct device *dev)
 661 {
 662         int rc = 0;
 663         struct ap_device *ap_dev = to_ap_dev(dev);
 664
 665         if (ap_suspend_flag) {
 666                 ap_suspend_flag = 0;
 667                 if (!ap_interrupts_available())
 668                         ap_interrupt_indicator = NULL;
 669                 ap_device_probe(dev);
 670                 ap_reset(ap_dev);
 671                 setup_timer(&ap_dev->timeout, ap_request_timeout,
 672                             (unsigned long) ap_dev);
 673                 ap_scan_bus(NULL);
 674                 init_timer(&ap_config_timer);
 675                 ap_config_timer.function = ap_config_timeout;
 676                 ap_config_timer.data = 0;
 677                 ap_config_timer.expires = jiffies + ap_config_time * HZ;
 678                 add_timer(&ap_config_timer);
 679                 ap_work_queue = create_singlethread_workqueue("kapwork");
 680                 if (!ap_work_queue)
 681                         return -ENOMEM;
 682                 tasklet_enable(&ap_tasklet);
 683                 if (!ap_using_interrupts())
 684                         ap_schedule_poll_timer();
 685                 else
 686                         tasklet_schedule(&ap_tasklet);
 687                 if (ap_thread_flag)
 688                         rc = ap_poll_thread_start();
 689         } else {
 690                 ap_device_probe(dev);
 691                 ap_reset(ap_dev);
 692                 setup_timer(&ap_dev->timeout, ap_request_timeout,
 693                             (unsigned long) ap_dev);
 694         }
 695
 696         return rc;
 697 }
 698
 699 static struct bus_type ap_bus_type = {
 700         .name = "ap",
 701         .match = &ap_bus_match,
 702         .uevent = &ap_uevent,
 703         .suspend = ap_bus_suspend,
 704         .resume = ap_bus_resume
 705 };
 706
 707 static int ap_device_probe(struct device *dev)
 708 {
 709         struct ap_device *ap_dev = to_ap_dev(dev);
 710         struct ap_driver *ap_drv = to_ap_drv(dev->driver);
 711         int rc;
 712
 713         ap_dev->drv = ap_drv;
 714         rc = ap_drv->probe ? ap_drv->probe(ap_dev) : -ENODEV;
 715         if (!rc) {
 716                 spin_lock_bh(&ap_device_list_lock);
 717                 list_add(&ap_dev->list, &ap_device_list);
 718                 spin_unlock_bh(&ap_device_list_lock);
 719         }
 720         return rc;
 721 }
 722
 723 /**
 724  * __ap_flush_queue(): Flush requests.
 725  * @ap_dev: Pointer to the AP device
 726  *
 727  * Flush all requests from the request/pending queue of an AP device.
 728  */
 729 static void __ap_flush_queue(struct ap_device *ap_dev)
 730 {
 731         struct ap_message *ap_msg, *next;
 732
 733         list_for_each_entry_safe(ap_msg, next, &ap_dev->pendingq, list) {
 734                 list_del_init(&ap_msg->list);
 735                 ap_dev->pendingq_count--;
 736                 ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
 737         }
 738         list_for_each_entry_safe(ap_msg, next, &ap_dev->requestq, list) {
 739                 list_del_init(&ap_msg->list);
 740                 ap_dev->requestq_count--;
 741                 ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
 742         }
 743 }
 744
 745 void ap_flush_queue(struct ap_device *ap_dev)
 746 {
 747         spin_lock_bh(&ap_dev->lock);
 748         __ap_flush_queue(ap_dev);
 749         spin_unlock_bh(&ap_dev->lock);
 750 }
 751 EXPORT_SYMBOL(ap_flush_queue);
 752
 753 static int ap_device_remove(struct device *dev)
 754 {
 755         struct ap_device *ap_dev = to_ap_dev(dev);
 756         struct ap_driver *ap_drv = ap_dev->drv;
 757
 758         ap_flush_queue(ap_dev);
 759         del_timer_sync(&ap_dev->timeout);
 760         spin_lock_bh(&ap_device_list_lock);
 761         list_del_init(&ap_dev->list);
 762         spin_unlock_bh(&ap_device_list_lock);
 763         if (ap_drv->remove)
 764                 ap_drv->remove(ap_dev);
 765         spin_lock_bh(&ap_dev->lock);
 766         atomic_sub(ap_dev->queue_count, &ap_poll_requests);
 767         spin_unlock_bh(&ap_dev->lock);
 768         return 0;
 769 }
 770
 771 int ap_driver_register(struct ap_driver *ap_drv, struct module *owner,
 772                        char *name)
 773 {
 774         struct device_driver *drv = &ap_drv->driver;
 775
 776         drv->bus = &ap_bus_type;
 777         drv->probe = ap_device_probe;
 778         drv->remove = ap_device_remove;
 779         drv->owner = owner;
 780         drv->name = name;
 781         return driver_register(drv);
 782 }
 783 EXPORT_SYMBOL(ap_driver_register);
 784
 785 void ap_driver_unregister(struct ap_driver *ap_drv)
 786 {
 787         driver_unregister(&ap_drv->driver);
 788 }
 789 EXPORT_SYMBOL(ap_driver_unregister);
 790
 791 /*
 792  * AP bus attributes.
 793  */
 794 static ssize_t ap_domain_show(struct bus_type *bus, char *buf)
 795 {
 796         return snprintf(buf, PAGE_SIZE, "%d\n", ap_domain_index);
 797 }
 798
 799 static BUS_ATTR(ap_domain, 0444, ap_domain_show, NULL);
 800
 801 static ssize_t ap_config_time_show(struct bus_type *bus, char *buf)
 802 {
 803         return snprintf(buf, PAGE_SIZE, "%d\n", ap_config_time);
 804 }
 805
 806 static ssize_t ap_interrupts_show(struct bus_type *bus, char *buf)
 807 {
 808         return snprintf(buf, PAGE_SIZE, "%d\n",
 809                         ap_using_interrupts() ? 1 : 0);
 810 }
 811
 812 static BUS_ATTR(ap_interrupts, 0444, ap_interrupts_show, NULL);
 813
 814 static ssize_t ap_config_time_store(struct bus_type *bus,
 815                                     const char *buf, size_t count)
 816 {
 817         int time;
 818
 819         if (sscanf(buf, "%d\n", &time) != 1 || time < 5 || time > 120)
 820                 return -EINVAL;
 821         ap_config_time = time;
 822         if (!timer_pending(&ap_config_timer) ||
 823             !mod_timer(&ap_config_timer, jiffies + ap_config_time * HZ)) {
 824                 ap_config_timer.expires = jiffies + ap_config_time * HZ;
 825                 add_timer(&ap_config_timer);
 826         }
 827         return count;
 828 }
 829
 830 static BUS_ATTR(config_time, 0644, ap_config_time_show, ap_config_time_store);
 831
 832 static ssize_t ap_poll_thread_show(struct bus_type *bus, char *buf)
 833 {
 834         return snprintf(buf, PAGE_SIZE, "%d\n", ap_poll_kthread ? 1 : 0);
 835 }
 836
 837 static ssize_t ap_poll_thread_store(struct bus_type *bus,
 838                                     const char *buf, size_t count)
 839 {
 840         int flag, rc;
 841
 842         if (sscanf(buf, "%d\n", &flag) != 1)
 843                 return -EINVAL;
 844         if (flag) {
 845                 rc = ap_poll_thread_start();
 846                 if (rc)
 847                         return rc;
 848         }
 849         else
 850                 ap_poll_thread_stop();
 851         return count;
 852 }
 853
 854 static BUS_ATTR(poll_thread, 0644, ap_poll_thread_show, ap_poll_thread_store);
 855
 856 static ssize_t poll_timeout_show(struct bus_type *bus, char *buf)
 857 {
 858         return snprintf(buf, PAGE_SIZE, "%llu\n", poll_timeout);
 859 }
 860
 861 static ssize_t poll_timeout_store(struct bus_type *bus, const char *buf,
 862                                   size_t count)
 863 {
 864         unsigned long long time;
 865         ktime_t hr_time;
 866
 867         /* 120 seconds = maximum poll interval */
 868         if (sscanf(buf, "%llu\n", &time) != 1 || time < 1 ||
 869             time > 120000000000ULL)
 870                 return -EINVAL;
 871         poll_timeout = time;
 872         hr_time = ktime_set(0, poll_timeout);
 873
 874         if (!hrtimer_is_queued(&ap_poll_timer) ||
 875             !hrtimer_forward(&ap_poll_timer, hrtimer_get_expires(&ap_poll_timer), hr_time)) {
 876                 hrtimer_set_expires(&ap_poll_timer, hr_time);
 877                 hrtimer_start_expires(&ap_poll_timer, HRTIMER_MODE_ABS);
 878         }
 879         return count;
 880 }
 881
 882 static BUS_ATTR(poll_timeout, 0644, poll_timeout_show, poll_timeout_store);
 883
 884 static struct bus_attribute *const ap_bus_attrs[] = {
 885         &bus_attr_ap_domain,
 886         &bus_attr_config_time,
 887         &bus_attr_poll_thread,
 888         &bus_attr_ap_interrupts,
 889         &bus_attr_poll_timeout,
 890         NULL,
 891 };
 892
 893 /**
 894  * ap_select_domain(): Select an AP domain.
 895  *
 896  * Pick one of the 16 AP domains.
 897  */
 898 static int ap_select_domain(void)
 899 {
 900         int queue_depth, device_type, count, max_count, best_domain;
 901         int rc, i, j;
 902
 903         /*
 904          * We want to use a single domain. Either the one specified with
 905          * the "domain=" parameter or the domain with the maximum number
 906          * of devices.
 907          */
 908         if (ap_domain_index >= 0 && ap_domain_index < AP_DOMAINS)
 909                 /* Domain has already been selected. */
 910                 return 0;
 911         best_domain = -1;
 912         max_count = 0;
 913         for (i = 0; i < AP_DOMAINS; i++) {
 914                 count = 0;
 915                 for (j = 0; j < AP_DEVICES; j++) {
 916                         ap_qid_t qid = AP_MKQID(j, i);
 917                         rc = ap_query_queue(qid, &queue_depth, &device_type);
 918                         if (rc)
 919                                 continue;
 920                         count++;
 921                 }
 922                 if (count > max_count) {
 923                         max_count = count;
 924                         best_domain = i;
 925                 }
 926         }
 927         if (best_domain >= 0){
 928                 ap_domain_index = best_domain;
 929                 return 0;
 930         }
 931         return -ENODEV;
 932 }
 933
 934 /**
 935  * ap_probe_device_type(): Find the device type of an AP.
 936  * @ap_dev: pointer to the AP device.
 937  *
 938  * Find the device type if query queue returned a device type of 0.
 939  */
 940 static int ap_probe_device_type(struct ap_device *ap_dev)
 941 {
 942         static unsigned char msg[] = {
 943                 0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,
 944                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 945                 0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,
 946                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 947                 0x01,0x00,0x43,0x43,0x41,0x2d,0x41,0x50,
 948                 0x50,0x4c,0x20,0x20,0x20,0x01,0x01,0x01,
 949                 0x00,0x00,0x00,0x00,0x50,0x4b,0x00,0x00,
 950                 0x00,0x00,0x01,0x1c,0x00,0x00,0x00,0x00,
 951                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 952                 0x00,0x00,0x05,0xb8,0x00,0x00,0x00,0x00,
 953                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 954                 0x70,0x00,0x41,0x00,0x00,0x00,0x00,0x00,
 955                 0x00,0x00,0x54,0x32,0x01,0x00,0xa0,0x00,
 956                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 957                 0x00,0x00,0x00,0x00,0xb8,0x05,0x00,0x00,
 958                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 959                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 960                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 961                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 962                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 963                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 964                 0x00,0x00,0x0a,0x00,0x00,0x00,0x00,0x00,
 965                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 966                 0x00,0x00,0x00,0x00,0x00,0x00,0x08,0x00,
 967                 0x49,0x43,0x53,0x46,0x20,0x20,0x20,0x20,
 968                 0x50,0x4b,0x0a,0x00,0x50,0x4b,0x43,0x53,
 969                 0x2d,0x31,0x2e,0x32,0x37,0x00,0x11,0x22,
 970                 0x33,0x44,0x55,0x66,0x77,0x88,0x99,0x00,
 971                 0x11,0x22,0x33,0x44,0x55,0x66,0x77,0x88,
 972                 0x99,0x00,0x11,0x22,0x33,0x44,0x55,0x66,
 973                 0x77,0x88,0x99,0x00,0x11,0x22,0x33,0x44,
 974                 0x55,0x66,0x77,0x88,0x99,0x00,0x11,0x22,
 975                 0x33,0x44,0x55,0x66,0x77,0x88,0x99,0x00,
 976                 0x11,0x22,0x33,0x5d,0x00,0x5b,0x00,0x77,
 977                 0x88,0x1e,0x00,0x00,0x57,0x00,0x00,0x00,
 978                 0x00,0x04,0x00,0x00,0x4f,0x00,0x00,0x00,
 979                 0x03,0x02,0x00,0x00,0x40,0x01,0x00,0x01,
 980                 0xce,0x02,0x68,0x2d,0x5f,0xa9,0xde,0x0c,
 981                 0xf6,0xd2,0x7b,0x58,0x4b,0xf9,0x28,0x68,
 982                 0x3d,0xb4,0xf4,0xef,0x78,0xd5,0xbe,0x66,
 983                 0x63,0x42,0xef,0xf8,0xfd,0xa4,0xf8,0xb0,
 984                 0x8e,0x29,0xc2,0xc9,0x2e,0xd8,0x45,0xb8,
 985                 0x53,0x8c,0x6f,0x4e,0x72,0x8f,0x6c,0x04,
 986                 0x9c,0x88,0xfc,0x1e,0xc5,0x83,0x55,0x57,
 987                 0xf7,0xdd,0xfd,0x4f,0x11,0x36,0x95,0x5d,
 988         };
 989         struct ap_queue_status status;
 990         unsigned long long psmid;
 991         char *reply;
 992         int rc, i;
 993
 994         reply = (void *) get_zeroed_page(GFP_KERNEL);
 995         if (!reply) {
 996                 rc = -ENOMEM;
 997                 goto out;
 998         }
 999
1000         status = __ap_send(ap_dev->qid, 0x0102030405060708ULL,
1001                            msg, sizeof(msg));
1002         if (status.response_code != AP_RESPONSE_NORMAL) {
1003                 rc = -ENODEV;
1004                 goto out_free;
1005         }
1006
1007         /* Wait for the test message to complete. */
1008         for (i = 0; i < 6; i++) {
1009                 mdelay(300);
1010                 status = __ap_recv(ap_dev->qid, &psmid, reply, 4096);
1011                 if (status.response_code == AP_RESPONSE_NORMAL &&
1012                     psmid == 0x0102030405060708ULL)
1013                         break;
1014         }
1015         if (i < 6) {
1016                 /* Got an answer. */
1017                 if (reply[0] == 0x00 && reply[1] == 0x86)
1018                         ap_dev->device_type = AP_DEVICE_TYPE_PCICC;
1019                 else
1020                         ap_dev->device_type = AP_DEVICE_TYPE_PCICA;
1021                 rc = 0;
1022         } else
1023                 rc = -ENODEV;
1024
1025 out_free:
1026         free_page((unsigned long) reply);
1027 out:
1028         return rc;
1029 }
1030
1031 static void ap_interrupt_handler(void *unused1, void *unused2)
1032 {
1033         tasklet_schedule(&ap_tasklet);
1034 }
1035
1036 /**
1037  * __ap_scan_bus(): Scan the AP bus.
1038  * @dev: Pointer to device
1039  * @data: Pointer to data
1040  *
1041  * Scan the AP bus for new devices.
1042  */
1043 static int __ap_scan_bus(struct device *dev, void *data)
1044 {
1045         return to_ap_dev(dev)->qid == (ap_qid_t)(unsigned long) data;
1046 }
1047
1048 static void ap_device_release(struct device *dev)
1049 {
1050         struct ap_device *ap_dev = to_ap_dev(dev);
1051
1052         kfree(ap_dev);
1053 }
1054
1055 static void ap_scan_bus(struct work_struct *unused)
1056 {
1057         struct ap_device *ap_dev;
1058         struct device *dev;
1059         ap_qid_t qid;
1060         int queue_depth, device_type;
1061         int rc, i;
1062
1063         if (ap_select_domain() != 0)
1064                 return;
1065         for (i = 0; i < AP_DEVICES; i++) {
1066                 qid = AP_MKQID(i, ap_domain_index);
1067                 dev = bus_find_device(&ap_bus_type, NULL,
1068                                       (void *)(unsigned long)qid,
1069                                       __ap_scan_bus);
1070                 rc = ap_query_queue(qid, &queue_depth, &device_type);
1071                 if (dev) {
1072                         if (rc == -EBUSY) {
1073                                 set_current_state(TASK_UNINTERRUPTIBLE);
1074                                 schedule_timeout(AP_RESET_TIMEOUT);
1075                                 rc = ap_query_queue(qid, &queue_depth,
1076                                                     &device_type);
1077                         }
1078                         ap_dev = to_ap_dev(dev);
1079                         spin_lock_bh(&ap_dev->lock);
1080                         if (rc || ap_dev->unregistered) {
1081                                 spin_unlock_bh(&ap_dev->lock);
1082                                 device_unregister(dev);
1083                                 put_device(dev);
1084                                 continue;
1085                         }
1086                         spin_unlock_bh(&ap_dev->lock);
1087                         put_device(dev);
1088                         continue;
1089                 }
1090                 if (rc)
1091                         continue;
1092                 rc = ap_init_queue(qid);
1093                 if (rc)
1094                         continue;
1095                 ap_dev = kzalloc(sizeof(*ap_dev), GFP_KERNEL);
1096                 if (!ap_dev)
1097                         break;
1098                 ap_dev->qid = qid;
1099                 ap_dev->queue_depth = queue_depth;
1100                 ap_dev->unregistered = 1;
1101                 spin_lock_init(&ap_dev->lock);
1102                 INIT_LIST_HEAD(&ap_dev->pendingq);
1103                 INIT_LIST_HEAD(&ap_dev->requestq);
1104                 INIT_LIST_HEAD(&ap_dev->list);
1105                 setup_timer(&ap_dev->timeout, ap_request_timeout,
1106                             (unsigned long) ap_dev);
1107                 if (device_type == 0)
1108                         ap_probe_device_type(ap_dev);
1109                 else
1110                         ap_dev->device_type = device_type;
1111
1112                 ap_dev->device.bus = &ap_bus_type;
1113                 ap_dev->device.parent = ap_root_device;
1114                 if (dev_set_name(&ap_dev->device, "card%02x",
1115                                  AP_QID_DEVICE(ap_dev->qid))) {
1116                         kfree(ap_dev);
1117                         continue;
1118                 }
1119                 ap_dev->device.release = ap_device_release;
1120                 rc = device_register(&ap_dev->device);
1121                 if (rc) {
1122                         put_device(&ap_dev->device);
1123                         continue;
1124                 }
1125                 /* Add device attributes. */
1126                 rc = sysfs_create_group(&ap_dev->device.kobj,
1127                                         &ap_dev_attr_group);
1128                 if (!rc) {
1129                         spin_lock_bh(&ap_dev->lock);
1130                         ap_dev->unregistered = 0;
1131                         spin_unlock_bh(&ap_dev->lock);
1132                 }
1133                 else
1134                         device_unregister(&ap_dev->device);
1135         }
1136 }
1137
1138 static void
1139 ap_config_timeout(unsigned long ptr)
1140 {
1141         queue_work(ap_work_queue, &ap_config_work);
1142         ap_config_timer.expires = jiffies + ap_config_time * HZ;
1143         add_timer(&ap_config_timer);
1144 }
1145
1146 /**
1147  * ap_schedule_poll_timer(): Schedule poll timer.
1148  *
1149  * Set up the timer to run the poll tasklet
1150  */
1151 static inline void ap_schedule_poll_timer(void)
1152 {
1153         ktime_t hr_time;
1154         if (ap_using_interrupts() || ap_suspend_flag)
1155                 return;
1156         if (hrtimer_is_queued(&ap_poll_timer))
1157                 return;
1158         if (ktime_to_ns(hrtimer_expires_remaining(&ap_poll_timer)) <= 0) {
1159                 hr_time = ktime_set(0, poll_timeout);
1160                 hrtimer_forward_now(&ap_poll_timer, hr_time);
1161                 hrtimer_restart(&ap_poll_timer);
1162         }
1163         return;
1164 }
1165
1166 /**
1167  * ap_poll_read(): Receive pending reply messages from an AP device.
1168  * @ap_dev: pointer to the AP device
1169  * @flags: pointer to control flags, bit 2^0 is set if another poll is
1170  *         required, bit 2^1 is set if the poll timer needs to get armed
1171  *
1172  * Returns 0 if the device is still present, -ENODEV if not.
1173  */
1174 static int ap_poll_read(struct ap_device *ap_dev, unsigned long *flags)
1175 {
1176         struct ap_queue_status status;
1177         struct ap_message *ap_msg;
1178
1179         if (ap_dev->queue_count <= 0)
1180                 return 0;
1181         status = __ap_recv(ap_dev->qid, &ap_dev->reply->psmid,
1182                            ap_dev->reply->message, ap_dev->reply->length);
1183         switch (status.response_code) {
1184         case AP_RESPONSE_NORMAL:
1185                 atomic_dec(&ap_poll_requests);
1186                 ap_decrease_queue_count(ap_dev);
1187                 list_for_each_entry(ap_msg, &ap_dev->pendingq, list) {
1188                         if (ap_msg->psmid != ap_dev->reply->psmid)
1189                                 continue;
1190                         list_del_init(&ap_msg->list);
1191                         ap_dev->pendingq_count--;
1192                         ap_dev->drv->receive(ap_dev, ap_msg, ap_dev->reply);
1193                         break;
1194                 }
1195                 if (ap_dev->queue_count > 0)
1196                         *flags |= 1;
1197                 break;
1198         case AP_RESPONSE_NO_PENDING_REPLY:
1199                 if (status.queue_empty) {
1200                         /* The card shouldn't forget requests but who knows. */
1201                         atomic_sub(ap_dev->queue_count, &ap_poll_requests);
1202                         ap_dev->queue_count = 0;
1203                         list_splice_init(&ap_dev->pendingq, &ap_dev->requestq);
1204                         ap_dev->requestq_count += ap_dev->pendingq_count;
1205                         ap_dev->pendingq_count = 0;
1206                 } else
1207                         *flags |= 2;
1208                 break;
1209         default:
1210                 return -ENODEV;
1211         }
1212         return 0;
1213 }
1214
1215 /**
1216  * ap_poll_write(): Send messages from the request queue to an AP device.
1217  * @ap_dev: pointer to the AP device
1218  * @flags: pointer to control flags, bit 2^0 is set if another poll is
1219  *         required, bit 2^1 is set if the poll timer needs to get armed
1220  *
1221  * Returns 0 if the device is still present, -ENODEV if not.
1222  */
1223 static int ap_poll_write(struct ap_device *ap_dev, unsigned long *flags)
1224 {
1225         struct ap_queue_status status;
1226         struct ap_message *ap_msg;
1227
1228         if (ap_dev->requestq_count <= 0 ||
1229             ap_dev->queue_count >= ap_dev->queue_depth)
1230                 return 0;
1231         /* Start the next request on the queue. */
1232         ap_msg = list_entry(ap_dev->requestq.next, struct ap_message, list);
1233         status = __ap_send(ap_dev->qid, ap_msg->psmid,
1234                            ap_msg->message, ap_msg->length);
1235         switch (status.response_code) {
1236         case AP_RESPONSE_NORMAL:
1237                 atomic_inc(&ap_poll_requests);
1238                 ap_increase_queue_count(ap_dev);
1239                 list_move_tail(&ap_msg->list, &ap_dev->pendingq);
1240                 ap_dev->requestq_count--;
1241                 ap_dev->pendingq_count++;
1242                 if (ap_dev->queue_count < ap_dev->queue_depth &&
1243                     ap_dev->requestq_count > 0)
1244                         *flags |= 1;
1245                 *flags |= 2;
1246                 break;
1247         case AP_RESPONSE_Q_FULL:
1248         case AP_RESPONSE_RESET_IN_PROGRESS:
1249                 *flags |= 2;
1250                 break;
1251         case AP_RESPONSE_MESSAGE_TOO_BIG:
1252                 return -EINVAL;
1253         default:
1254                 return -ENODEV;
1255         }
1256         return 0;
1257 }
1258
1259 /**
1260  * ap_poll_queue(): Poll AP device for pending replies and send new messages.
1261  * @ap_dev: pointer to the bus device
1262  * @flags: pointer to control flags, bit 2^0 is set if another poll is
1263  *         required, bit 2^1 is set if the poll timer needs to get armed
1264  *
1265  * Poll AP device for pending replies and send new messages. If either
1266  * ap_poll_read or ap_poll_write returns -ENODEV unregister the device.
1267  * Returns 0.
1268  */
1269 static inline int ap_poll_queue(struct ap_device *ap_dev, unsigned long *flags)
1270 {
1271         int rc;
1272
1273         rc = ap_poll_read(ap_dev, flags);
1274         if (rc)
1275                 return rc;
1276         return ap_poll_write(ap_dev, flags);
1277 }
1278
1279 /**
1280  * __ap_queue_message(): Queue a message to a device.
1281  * @ap_dev: pointer to the AP device
1282  * @ap_msg: the message to be queued
1283  *
1284  * Queue a message to a device. Returns 0 if successful.
1285  */
1286 static int __ap_queue_message(struct ap_device *ap_dev, struct ap_message *ap_msg)
1287 {
1288         struct ap_queue_status status;
1289
1290         if (list_empty(&ap_dev->requestq) &&
1291             ap_dev->queue_count < ap_dev->queue_depth) {
1292                 status = __ap_send(ap_dev->qid, ap_msg->psmid,
1293                                    ap_msg->message, ap_msg->length);
1294                 switch (status.response_code) {
1295                 case AP_RESPONSE_NORMAL:
1296                         list_add_tail(&ap_msg->list, &ap_dev->pendingq);
1297                         atomic_inc(&ap_poll_requests);
1298                         ap_dev->pendingq_count++;
1299                         ap_increase_queue_count(ap_dev);
1300                         ap_dev->total_request_count++;
1301                         break;
1302                 case AP_RESPONSE_Q_FULL:
1303                 case AP_RESPONSE_RESET_IN_PROGRESS:
1304                         list_add_tail(&ap_msg->list, &ap_dev->requestq);
1305                         ap_dev->requestq_count++;
1306                         ap_dev->total_request_count++;
1307                         return -EBUSY;
1308                 case AP_RESPONSE_MESSAGE_TOO_BIG:
1309                         ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-EINVAL));
1310                         return -EINVAL;
1311                 default:        /* Device is gone. */
1312                         ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
1313                         return -ENODEV;
1314                 }
1315         } else {
1316                 list_add_tail(&ap_msg->list, &ap_dev->requestq);
1317                 ap_dev->requestq_count++;
1318                 ap_dev->total_request_count++;
1319                 return -EBUSY;
1320         }
1321         ap_schedule_poll_timer();
1322         return 0;
1323 }
1324
1325 void ap_queue_message(struct ap_device *ap_dev, struct ap_message *ap_msg)
1326 {
1327         unsigned long flags;
1328         int rc;
1329
1330         spin_lock_bh(&ap_dev->lock);
1331         if (!ap_dev->unregistered) {
1332                 /* Make room on the queue by polling for finished requests. */
1333                 rc = ap_poll_queue(ap_dev, &flags);
1334                 if (!rc)
1335                         rc = __ap_queue_message(ap_dev, ap_msg);
1336                 if (!rc)
1337                         wake_up(&ap_poll_wait);
1338                 if (rc == -ENODEV)
1339                         ap_dev->unregistered = 1;
1340         } else {
1341                 ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
1342                 rc = -ENODEV;
1343         }
1344         spin_unlock_bh(&ap_dev->lock);
1345         if (rc == -ENODEV)
1346                 device_unregister(&ap_dev->device);
1347 }
1348 EXPORT_SYMBOL(ap_queue_message);
1349
1350 /**
1351  * ap_cancel_message(): Cancel a crypto request.
1352  * @ap_dev: The AP device that has the message queued
1353  * @ap_msg: The message that is to be removed
1354  *
1355  * Cancel a crypto request. This is done by removing the request
1356  * from the device pending or request queue. Note that the
1357  * request stays on the AP queue. When it finishes the message
1358  * reply will be discarded because the psmid can't be found.
1359  */
1360 void ap_cancel_message(struct ap_device *ap_dev, struct ap_message *ap_msg)
1361 {
1362         struct ap_message *tmp;
1363
1364         spin_lock_bh(&ap_dev->lock);
1365         if (!list_empty(&ap_msg->list)) {
1366                 list_for_each_entry(tmp, &ap_dev->pendingq, list)
1367                         if (tmp->psmid == ap_msg->psmid) {
1368                                 ap_dev->pendingq_count--;
1369                                 goto found;
1370                         }
1371                 ap_dev->requestq_count--;
1372         found:
1373                 list_del_init(&ap_msg->list);
1374         }
1375         spin_unlock_bh(&ap_dev->lock);
1376 }
1377 EXPORT_SYMBOL(ap_cancel_message);
1378
1379 /**
1380  * ap_poll_timeout(): AP receive polling for finished AP requests.
1381  * @unused: Unused pointer.
1382  *
1383  * Schedules the AP tasklet using a high resolution timer.
1384  */
1385 static enum hrtimer_restart ap_poll_timeout(struct hrtimer *unused)
1386 {
1387         tasklet_schedule(&ap_tasklet);
1388         return HRTIMER_NORESTART;
1389 }
1390
1391 /**
1392  * ap_reset(): Reset a not responding AP device.
1393  * @ap_dev: Pointer to the AP device
1394  *
1395  * Reset a not responding AP device and move all requests from the
1396  * pending queue to the request queue.
1397  */
1398 static void ap_reset(struct ap_device *ap_dev)
1399 {
1400         int rc;
1401
1402         ap_dev->reset = AP_RESET_IGNORE;
1403         atomic_sub(ap_dev->queue_count, &ap_poll_requests);
1404         ap_dev->queue_count = 0;
1405         list_splice_init(&ap_dev->pendingq, &ap_dev->requestq);
1406         ap_dev->requestq_count += ap_dev->pendingq_count;
1407         ap_dev->pendingq_count = 0;
1408         rc = ap_init_queue(ap_dev->qid);
1409         if (rc == -ENODEV)
1410                 ap_dev->unregistered = 1;
1411 }
1412
1413 static int __ap_poll_device(struct ap_device *ap_dev, unsigned long *flags)
1414 {
1415         if (!ap_dev->unregistered) {
1416                 if (ap_poll_queue(ap_dev, flags))
1417                         ap_dev->unregistered = 1;
1418                 if (ap_dev->reset == AP_RESET_DO)
1419                         ap_reset(ap_dev);
1420         }
1421         return 0;
1422 }
1423
1424 /**
1425  * ap_poll_all(): Poll all AP devices.
1426  * @dummy: Unused variable
1427  *
1428  * Poll all AP devices on the bus in a round robin fashion. Continue
1429  * polling until bit 2^0 of the control flags is not set. If bit 2^1
1430  * of the control flags has been set arm the poll timer.
1431  */
1432 static void ap_poll_all(unsigned long dummy)
1433 {
1434         unsigned long flags;
1435         struct ap_device *ap_dev;
1436
1437         /* Reset the indicator if interrupts are used. Thus new interrupts can
1438          * be received. Doing it in the beginning of the tasklet is therefor
1439          * important that no requests on any AP get lost.
1440          */
1441         if (ap_using_interrupts())
1442                 xchg((u8 *)ap_interrupt_indicator, 0);
1443         do {
1444                 flags = 0;
1445                 spin_lock(&ap_device_list_lock);
1446                 list_for_each_entry(ap_dev, &ap_device_list, list) {
1447                         spin_lock(&ap_dev->lock);
1448                         __ap_poll_device(ap_dev, &flags);
1449                         spin_unlock(&ap_dev->lock);
1450                 }
1451                 spin_unlock(&ap_device_list_lock);
1452         } while (flags & 1);
1453         if (flags & 2)
1454                 ap_schedule_poll_timer();
1455 }
1456
1457 /**
1458  * ap_poll_thread(): Thread that polls for finished requests.
1459  * @data: Unused pointer
1460  *
1461  * AP bus poll thread. The purpose of this thread is to poll for
1462  * finished requests in a loop if there is a "free" cpu - that is
1463  * a cpu that doesn't have anything better to do. The polling stops
1464  * as soon as there is another task or if all messages have been
1465  * delivered.
1466  */
1467 static int ap_poll_thread(void *data)
1468 {
1469         DECLARE_WAITQUEUE(wait, current);
1470         unsigned long flags;
1471         int requests;
1472         struct ap_device *ap_dev;
1473
1474         set_user_nice(current, 19);
1475         while (1) {
1476                 if (ap_suspend_flag)
1477                         return 0;
1478                 if (need_resched()) {
1479                         schedule();
1480                         continue;
1481                 }
1482                 add_wait_queue(&ap_poll_wait, &wait);
1483                 set_current_state(TASK_INTERRUPTIBLE);
1484                 if (kthread_should_stop())
1485                         break;
1486                 requests = atomic_read(&ap_poll_requests);
1487                 if (requests <= 0)
1488                         schedule();
1489                 set_current_state(TASK_RUNNING);
1490                 remove_wait_queue(&ap_poll_wait, &wait);
1491
1492                 flags = 0;
1493                 spin_lock_bh(&ap_device_list_lock);
1494                 list_for_each_entry(ap_dev, &ap_device_list, list) {
1495                         spin_lock(&ap_dev->lock);
1496                         __ap_poll_device(ap_dev, &flags);
1497                         spin_unlock(&ap_dev->lock);
1498                 }
1499                 spin_unlock_bh(&ap_device_list_lock);
1500         }
1501         set_current_state(TASK_RUNNING);
1502         remove_wait_queue(&ap_poll_wait, &wait);
1503         return 0;
1504 }
1505
1506 static int ap_poll_thread_start(void)
1507 {
1508         int rc;
1509
1510         if (ap_using_interrupts() || ap_suspend_flag)
1511                 return 0;
1512         mutex_lock(&ap_poll_thread_mutex);
1513         if (!ap_poll_kthread) {
1514                 ap_poll_kthread = kthread_run(ap_poll_thread, NULL, "appoll");
1515                 rc = IS_ERR(ap_poll_kthread) ? PTR_ERR(ap_poll_kthread) : 0;
1516                 if (rc)
1517                         ap_poll_kthread = NULL;
1518         }
1519         else
1520                 rc = 0;
1521         mutex_unlock(&ap_poll_thread_mutex);
1522         return rc;
1523 }
1524
1525 static void ap_poll_thread_stop(void)
1526 {
1527         mutex_lock(&ap_poll_thread_mutex);
1528         if (ap_poll_kthread) {
1529                 kthread_stop(ap_poll_kthread);
1530                 ap_poll_kthread = NULL;
1531         }
1532         mutex_unlock(&ap_poll_thread_mutex);
1533 }
1534
1535 /**
1536  * ap_request_timeout(): Handling of request timeouts
1537  * @data: Holds the AP device.
1538  *
1539  * Handles request timeouts.
1540  */
1541 static void ap_request_timeout(unsigned long data)
1542 {
1543         struct ap_device *ap_dev = (struct ap_device *) data;
1544
1545         if (ap_dev->reset == AP_RESET_ARMED) {
1546                 ap_dev->reset = AP_RESET_DO;
1547
1548                 if (ap_using_interrupts())
1549                         tasklet_schedule(&ap_tasklet);
1550         }
1551 }
1552
1553 static void ap_reset_domain(void)
1554 {
1555         int i;
1556
1557         if (ap_domain_index != -1)
1558                 for (i = 0; i < AP_DEVICES; i++)
1559                         ap_reset_queue(AP_MKQID(i, ap_domain_index));
1560 }
1561
1562 static void ap_reset_all(void)
1563 {
1564         int i, j;
1565
1566         for (i = 0; i < AP_DOMAINS; i++)
1567                 for (j = 0; j < AP_DEVICES; j++)
1568                         ap_reset_queue(AP_MKQID(j, i));
1569 }
1570
1571 static struct reset_call ap_reset_call = {
1572         .fn = ap_reset_all,
1573 };
1574
1575 /**
1576  * ap_module_init(): The module initialization code.
1577  *
1578  * Initializes the module.
1579  */
1580 int __init ap_module_init(void)
1581 {
1582         int rc, i;
1583
1584         if (ap_domain_index < -1 || ap_domain_index >= AP_DOMAINS) {
1585                 pr_warning("%d is not a valid cryptographic domain\n",
1586                            ap_domain_index);
1587                 return -EINVAL;
1588         }
1589         if (ap_instructions_available() != 0) {
1590                 pr_warning("The hardware system does not support "
1591                            "AP instructions\n");
1592                 return -ENODEV;
1593         }
1594         if (ap_interrupts_available()) {
1595                 isc_register(AP_ISC);
1596                 ap_interrupt_indicator = s390_register_adapter_interrupt(
1597                         &ap_interrupt_handler, NULL, AP_ISC);
1598                 if (IS_ERR(ap_interrupt_indicator)) {
1599                         ap_interrupt_indicator = NULL;
1600                         isc_unregister(AP_ISC);
1601                 }
1602         }
1603
1604         register_reset_call(&ap_reset_call);
1605
1606         /* Create /sys/bus/ap. */
1607         rc = bus_register(&ap_bus_type);
1608         if (rc)
1609                 goto out;
1610         for (i = 0; ap_bus_attrs[i]; i++) {
1611                 rc = bus_create_file(&ap_bus_type, ap_bus_attrs[i]);
1612                 if (rc)
1613                         goto out_bus;
1614         }
1615
1616         /* Create /sys/devices/ap. */
1617         ap_root_device = root_device_register("ap");
1618         rc = IS_ERR(ap_root_device) ? PTR_ERR(ap_root_device) : 0;
1619         if (rc)
1620                 goto out_bus;
1621
1622         ap_work_queue = create_singlethread_workqueue("kapwork");
1623         if (!ap_work_queue) {
1624                 rc = -ENOMEM;
1625                 goto out_root;
1626         }
1627
1628         if (ap_select_domain() == 0)
1629                 ap_scan_bus(NULL);
1630
1631         /* Setup the AP bus rescan timer. */
1632         init_timer(&ap_config_timer);
1633         ap_config_timer.function = ap_config_timeout;
1634         ap_config_timer.data = 0;
1635         ap_config_timer.expires = jiffies + ap_config_time * HZ;
1636         add_timer(&ap_config_timer);
1637
1638         /* Setup the high resultion poll timer.
1639          * If we are running under z/VM adjust polling to z/VM polling rate.
1640          */
1641         if (MACHINE_IS_VM)
1642                 poll_timeout = 1500000;
1643         hrtimer_init(&ap_poll_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1644         ap_poll_timer.function = ap_poll_timeout;
1645
1646         /* Start the low priority AP bus poll thread. */
1647         if (ap_thread_flag) {
1648                 rc = ap_poll_thread_start();
1649                 if (rc)
1650                         goto out_work;
1651         }
1652
1653         return 0;
1654
1655 out_work:
1656         del_timer_sync(&ap_config_timer);
1657         hrtimer_cancel(&ap_poll_timer);
1658         destroy_workqueue(ap_work_queue);
1659 out_root:
1660         root_device_unregister(ap_root_device);
1661 out_bus:
1662         while (i--)
1663                 bus_remove_file(&ap_bus_type, ap_bus_attrs[i]);
1664         bus_unregister(&ap_bus_type);
1665 out:
1666         unregister_reset_call(&ap_reset_call);
1667         if (ap_using_interrupts()) {
1668                 s390_unregister_adapter_interrupt(ap_interrupt_indicator, AP_ISC);
1669                 isc_unregister(AP_ISC);
1670         }
1671         return rc;
1672 }
1673
1674 static int __ap_match_all(struct device *dev, void *data)
1675 {
1676         return 1;
1677 }
1678
1679 /**
1680  * ap_modules_exit(): The module termination code
1681  *
1682  * Terminates the module.
1683  */
1684 void ap_module_exit(void)
1685 {
1686         int i;
1687         struct device *dev;
1688
1689         ap_reset_domain();
1690         ap_poll_thread_stop();
1691         del_timer_sync(&ap_config_timer);
1692         hrtimer_cancel(&ap_poll_timer);
1693         destroy_workqueue(ap_work_queue);
1694         tasklet_kill(&ap_tasklet);
1695         root_device_unregister(ap_root_device);
1696         while ((dev = bus_find_device(&ap_bus_type, NULL, NULL,
1697                     __ap_match_all)))
1698         {
1699                 device_unregister(dev);
1700                 put_device(dev);
1701         }
1702         for (i = 0; ap_bus_attrs[i]; i++)
1703                 bus_remove_file(&ap_bus_type, ap_bus_attrs[i]);
1704         bus_unregister(&ap_bus_type);
1705         unregister_reset_call(&ap_reset_call);
1706         if (ap_using_interrupts()) {
1707                 s390_unregister_adapter_interrupt(ap_interrupt_indicator, AP_ISC);
1708                 isc_unregister(AP_ISC);
1709         }
1710 }
1711
1712 #ifndef CONFIG_ZCRYPT_MONOLITHIC
1713 module_init(ap_module_init);
1714 module_exit(ap_module_exit);
1715 #endif