drivers/xen/events.c

   1 /*
   2  * Xen event channels
   3  *
   4  * Xen models interrupts with abstract event channels.  Because each
   5  * domain gets 1024 event channels, but NR_IRQ is not that large, we
   6  * must dynamically map irqs<->event channels.  The event channels
   7  * interface with the rest of the kernel by defining a xen interrupt
   8  * chip.  When an event is recieved, it is mapped to an irq and sent
   9  * through the normal interrupt processing path.
  10  *
  11  * There are four kinds of events which can be mapped to an event
  12  * channel:
  13  *
  14  * 1. Inter-domain notifications.  This includes all the virtual
  15  *    device events, since they're driven by front-ends in another domain
  16  *    (typically dom0).
  17  * 2. VIRQs, typically used for timers.  These are per-cpu events.
  18  * 3. IPIs.
  19  * 4. PIRQs - Hardware interrupts.
  20  *
  21  * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
  22  */
  23
  24 #include <linux/linkage.h>
  25 #include <linux/interrupt.h>
  26 #include <linux/irq.h>
  27 #include <linux/module.h>
  28 #include <linux/string.h>
  29 #include <linux/bootmem.h>
  30 #include <linux/slab.h>
  31 #include <linux/irqnr.h>
  32
  33 #include <asm/desc.h>
  34 #include <asm/ptrace.h>
  35 #include <asm/irq.h>
  36 #include <asm/idle.h>
  37 #include <asm/io_apic.h>
  38 #include <asm/sync_bitops.h>
  39 #include <asm/xen/hypercall.h>
  40 #include <asm/xen/hypervisor.h>
  41
  42 #include <xen/xen.h>
  43 #include <xen/hvm.h>
  44 #include <xen/xen-ops.h>
  45 #include <xen/events.h>
  46 #include <xen/interface/xen.h>
  47 #include <xen/interface/event_channel.h>
  48 #include <xen/interface/hvm/hvm_op.h>
  49 #include <xen/interface/hvm/params.h>
  50
  51 /*
  52  * This lock protects updates to the following mapping and reference-count
  53  * arrays. The lock does not need to be acquired to read the mapping tables.
  54  */
  55 static DEFINE_SPINLOCK(irq_mapping_update_lock);
  56
  57 /* IRQ <-> VIRQ mapping. */
  58 static DEFINE_PER_CPU(int [NR_VIRQS], virq_to_irq) = {[0 ... NR_VIRQS-1] = -1};
  59
  60 /* IRQ <-> IPI mapping */
  61 static DEFINE_PER_CPU(int [XEN_NR_IPIS], ipi_to_irq) = {[0 ... XEN_NR_IPIS-1] = -1};
  62
  63 /* Interrupt types. */
  64 enum xen_irq_type {
  65         IRQT_UNBOUND = 0,
  66         IRQT_PIRQ,
  67         IRQT_VIRQ,
  68         IRQT_IPI,
  69         IRQT_EVTCHN
  70 };
  71
  72 /*
  73  * Packed IRQ information:
  74  * type - enum xen_irq_type
  75  * event channel - irq->event channel mapping
  76  * cpu - cpu this event channel is bound to
  77  * index - type-specific information:
  78  *    PIRQ - vector, with MSB being "needs EIO"
  79  *    VIRQ - virq number
  80  *    IPI - IPI vector
  81  *    EVTCHN -
  82  */
  83 struct irq_info
  84 {
  85         enum xen_irq_type type; /* type */
  86         unsigned short evtchn;  /* event channel */
  87         unsigned short cpu;     /* cpu bound */
  88
  89         union {
  90                 unsigned short virq;
  91                 enum ipi_vector ipi;
  92                 struct {
  93                         unsigned short gsi;
  94                         unsigned char vector;
  95                         unsigned char flags;
  96                 } pirq;
  97         } u;
  98 };
  99 #define PIRQ_NEEDS_EOI  (1 << 0)
 100
 101 static struct irq_info *irq_info;
 102
 103 static int *evtchn_to_irq;
 104 struct cpu_evtchn_s {
 105         unsigned long bits[NR_EVENT_CHANNELS/BITS_PER_LONG];
 106 };
 107
 108 static __initdata struct cpu_evtchn_s init_evtchn_mask = {
 109         .bits[0 ... (NR_EVENT_CHANNELS/BITS_PER_LONG)-1] = ~0ul,
 110 };
 111 static struct cpu_evtchn_s *cpu_evtchn_mask_p = &init_evtchn_mask;
 112
 113 static inline unsigned long *cpu_evtchn_mask(int cpu)
 114 {
 115         return cpu_evtchn_mask_p[cpu].bits;
 116 }
 117
 118 /* Xen will never allocate port zero for any purpose. */
 119 #define VALID_EVTCHN(chn)       ((chn) != 0)
 120
 121 static struct irq_chip xen_dynamic_chip;
 122 static struct irq_chip xen_percpu_chip;
 123 static struct irq_chip xen_pirq_chip;
 124
 125 /* Constructor for packed IRQ information. */
 126 static struct irq_info mk_unbound_info(void)
 127 {
 128         return (struct irq_info) { .type = IRQT_UNBOUND };
 129 }
 130
 131 static struct irq_info mk_evtchn_info(unsigned short evtchn)
 132 {
 133         return (struct irq_info) { .type = IRQT_EVTCHN, .evtchn = evtchn,
 134                         .cpu = 0 };
 135 }
 136
 137 static struct irq_info mk_ipi_info(unsigned short evtchn, enum ipi_vector ipi)
 138 {
 139         return (struct irq_info) { .type = IRQT_IPI, .evtchn = evtchn,
 140                         .cpu = 0, .u.ipi = ipi };
 141 }
 142
 143 static struct irq_info mk_virq_info(unsigned short evtchn, unsigned short virq)
 144 {
 145         return (struct irq_info) { .type = IRQT_VIRQ, .evtchn = evtchn,
 146                         .cpu = 0, .u.virq = virq };
 147 }
 148
 149 static struct irq_info mk_pirq_info(unsigned short evtchn,
 150                                     unsigned short gsi, unsigned short vector)
 151 {
 152         return (struct irq_info) { .type = IRQT_PIRQ, .evtchn = evtchn,
 153                         .cpu = 0, .u.pirq = { .gsi = gsi, .vector = vector } };
 154 }
 155
 156 /*
 157  * Accessors for packed IRQ information.
 158  */
 159 static struct irq_info *info_for_irq(unsigned irq)
 160 {
 161         return &irq_info[irq];
 162 }
 163
 164 static unsigned int evtchn_from_irq(unsigned irq)
 165 {
 166         return info_for_irq(irq)->evtchn;
 167 }
 168
 169 unsigned irq_from_evtchn(unsigned int evtchn)
 170 {
 171         return evtchn_to_irq[evtchn];
 172 }
 173 EXPORT_SYMBOL_GPL(irq_from_evtchn);
 174
 175 static enum ipi_vector ipi_from_irq(unsigned irq)
 176 {
 177         struct irq_info *info = info_for_irq(irq);
 178
 179         BUG_ON(info == NULL);
 180         BUG_ON(info->type != IRQT_IPI);
 181
 182         return info->u.ipi;
 183 }
 184
 185 static unsigned virq_from_irq(unsigned irq)
 186 {
 187         struct irq_info *info = info_for_irq(irq);
 188
 189         BUG_ON(info == NULL);
 190         BUG_ON(info->type != IRQT_VIRQ);
 191
 192         return info->u.virq;
 193 }
 194
 195 static unsigned gsi_from_irq(unsigned irq)
 196 {
 197         struct irq_info *info = info_for_irq(irq);
 198
 199         BUG_ON(info == NULL);
 200         BUG_ON(info->type != IRQT_PIRQ);
 201
 202         return info->u.pirq.gsi;
 203 }
 204
 205 static unsigned vector_from_irq(unsigned irq)
 206 {
 207         struct irq_info *info = info_for_irq(irq);
 208
 209         BUG_ON(info == NULL);
 210         BUG_ON(info->type != IRQT_PIRQ);
 211
 212         return info->u.pirq.vector;
 213 }
 214
 215 static enum xen_irq_type type_from_irq(unsigned irq)
 216 {
 217         return info_for_irq(irq)->type;
 218 }
 219
 220 static unsigned cpu_from_irq(unsigned irq)
 221 {
 222         return info_for_irq(irq)->cpu;
 223 }
 224
 225 static unsigned int cpu_from_evtchn(unsigned int evtchn)
 226 {
 227         int irq = evtchn_to_irq[evtchn];
 228         unsigned ret = 0;
 229
 230         if (irq != -1)
 231                 ret = cpu_from_irq(irq);
 232
 233         return ret;
 234 }
 235
 236 static bool pirq_needs_eoi(unsigned irq)
 237 {
 238         struct irq_info *info = info_for_irq(irq);
 239
 240         BUG_ON(info->type != IRQT_PIRQ);
 241
 242         return info->u.pirq.flags & PIRQ_NEEDS_EOI;
 243 }
 244
 245 static inline unsigned long active_evtchns(unsigned int cpu,
 246                                            struct shared_info *sh,
 247                                            unsigned int idx)
 248 {
 249         return (sh->evtchn_pending[idx] &
 250                 cpu_evtchn_mask(cpu)[idx] &
 251                 ~sh->evtchn_mask[idx]);
 252 }
 253
 254 static void bind_evtchn_to_cpu(unsigned int chn, unsigned int cpu)
 255 {
 256         int irq = evtchn_to_irq[chn];
 257
 258         BUG_ON(irq == -1);
 259 #ifdef CONFIG_SMP
 260         cpumask_copy(irq_to_desc(irq)->affinity, cpumask_of(cpu));
 261 #endif
 262
 263         __clear_bit(chn, cpu_evtchn_mask(cpu_from_irq(irq)));
 264         __set_bit(chn, cpu_evtchn_mask(cpu));
 265
 266         irq_info[irq].cpu = cpu;
 267 }
 268
 269 static void init_evtchn_cpu_bindings(void)
 270 {
 271 #ifdef CONFIG_SMP
 272         struct irq_desc *desc;
 273         int i;
 274
 275         /* By default all event channels notify CPU#0. */
 276         for_each_irq_desc(i, desc) {
 277                 cpumask_copy(desc->affinity, cpumask_of(0));
 278         }
 279 #endif
 280
 281         memset(cpu_evtchn_mask(0), ~0, sizeof(cpu_evtchn_mask(0)));
 282 }
 283
 284 static inline void clear_evtchn(int port)
 285 {
 286         struct shared_info *s = HYPERVISOR_shared_info;
 287         sync_clear_bit(port, &s->evtchn_pending[0]);
 288 }
 289
 290 static inline void set_evtchn(int port)
 291 {
 292         struct shared_info *s = HYPERVISOR_shared_info;
 293         sync_set_bit(port, &s->evtchn_pending[0]);
 294 }
 295
 296 static inline int test_evtchn(int port)
 297 {
 298         struct shared_info *s = HYPERVISOR_shared_info;
 299         return sync_test_bit(port, &s->evtchn_pending[0]);
 300 }
 301
 302
 303 /**
 304  * notify_remote_via_irq - send event to remote end of event channel via irq
 305  * @irq: irq of event channel to send event to
 306  *
 307  * Unlike notify_remote_via_evtchn(), this is safe to use across
 308  * save/restore. Notifications on a broken connection are silently
 309  * dropped.
 310  */
 311 void notify_remote_via_irq(int irq)
 312 {
 313         int evtchn = evtchn_from_irq(irq);
 314
 315         if (VALID_EVTCHN(evtchn))
 316                 notify_remote_via_evtchn(evtchn);
 317 }
 318 EXPORT_SYMBOL_GPL(notify_remote_via_irq);
 319
 320 static void mask_evtchn(int port)
 321 {
 322         struct shared_info *s = HYPERVISOR_shared_info;
 323         sync_set_bit(port, &s->evtchn_mask[0]);
 324 }
 325
 326 static void unmask_evtchn(int port)
 327 {
 328         struct shared_info *s = HYPERVISOR_shared_info;
 329         unsigned int cpu = get_cpu();
 330
 331         BUG_ON(!irqs_disabled());
 332
 333         /* Slow path (hypercall) if this is a non-local port. */
 334         if (unlikely(cpu != cpu_from_evtchn(port))) {
 335                 struct evtchn_unmask unmask = { .port = port };
 336                 (void)HYPERVISOR_event_channel_op(EVTCHNOP_unmask, &unmask);
 337         } else {
 338                 struct vcpu_info *vcpu_info = __get_cpu_var(xen_vcpu);
 339
 340                 sync_clear_bit(port, &s->evtchn_mask[0]);
 341
 342                 /*
 343                  * The following is basically the equivalent of
 344                  * 'hw_resend_irq'. Just like a real IO-APIC we 'lose
 345                  * the interrupt edge' if the channel is masked.
 346                  */
 347                 if (sync_test_bit(port, &s->evtchn_pending[0]) &&
 348                     !sync_test_and_set_bit(port / BITS_PER_LONG,
 349                                            &vcpu_info->evtchn_pending_sel))
 350                         vcpu_info->evtchn_upcall_pending = 1;
 351         }
 352
 353         put_cpu();
 354 }
 355
 356 static int get_nr_hw_irqs(void)
 357 {
 358         int ret = 1;
 359
 360 #ifdef CONFIG_X86_IO_APIC
 361         ret = get_nr_irqs_gsi();
 362 #endif
 363
 364         return ret;
 365 }
 366
 367 static int find_unbound_irq(void)
 368 {
 369         struct irq_data *data;
 370         int irq, res;
 371
 372         for (irq = 0; irq < nr_irqs; irq++) {
 373                 data = irq_get_irq_data(irq);
 374                 /* only 0->15 have init'd desc; handle irq > 16 */
 375                 if (!data)
 376                         break;
 377                 if (data->chip == &no_irq_chip)
 378                         break;
 379                 if (data->chip != &xen_dynamic_chip)
 380                         continue;
 381                 if (irq_info[irq].type == IRQT_UNBOUND)
 382                         return irq;
 383         }
 384
 385         if (irq == nr_irqs)
 386                 panic("No available IRQ to bind to: increase nr_irqs!\n");
 387
 388         res = irq_alloc_desc_at(irq, 0);
 389
 390         if (WARN_ON(res != irq))
 391                 return -1;
 392
 393         return irq;
 394 }
 395
 396 static bool identity_mapped_irq(unsigned irq)
 397 {
 398         /* identity map all the hardware irqs */
 399         return irq < get_nr_hw_irqs();
 400 }
 401
 402 static void pirq_unmask_notify(int irq)
 403 {
 404         struct physdev_eoi eoi = { .irq = irq };
 405
 406         if (unlikely(pirq_needs_eoi(irq))) {
 407                 int rc = HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi);
 408                 WARN_ON(rc);
 409         }
 410 }
 411
 412 static void pirq_query_unmask(int irq)
 413 {
 414         struct physdev_irq_status_query irq_status;
 415         struct irq_info *info = info_for_irq(irq);
 416
 417         BUG_ON(info->type != IRQT_PIRQ);
 418
 419         irq_status.irq = irq;
 420         if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
 421                 irq_status.flags = 0;
 422
 423         info->u.pirq.flags &= ~PIRQ_NEEDS_EOI;
 424         if (irq_status.flags & XENIRQSTAT_needs_eoi)
 425                 info->u.pirq.flags |= PIRQ_NEEDS_EOI;
 426 }
 427
 428 static bool probing_irq(int irq)
 429 {
 430         struct irq_desc *desc = irq_to_desc(irq);
 431
 432         return desc && desc->action == NULL;
 433 }
 434
 435 static unsigned int startup_pirq(unsigned int irq)
 436 {
 437         struct evtchn_bind_pirq bind_pirq;
 438         struct irq_info *info = info_for_irq(irq);
 439         int evtchn = evtchn_from_irq(irq);
 440
 441         BUG_ON(info->type != IRQT_PIRQ);
 442
 443         if (VALID_EVTCHN(evtchn))
 444                 goto out;
 445
 446         bind_pirq.pirq = irq;
 447         /* NB. We are happy to share unless we are probing. */
 448         bind_pirq.flags = probing_irq(irq) ? 0 : BIND_PIRQ__WILL_SHARE;
 449         if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind_pirq) != 0) {
 450                 if (!probing_irq(irq))
 451                         printk(KERN_INFO "Failed to obtain physical IRQ %d\n",
 452                                irq);
 453                 return 0;
 454         }
 455         evtchn = bind_pirq.port;
 456
 457         pirq_query_unmask(irq);
 458
 459         evtchn_to_irq[evtchn] = irq;
 460         bind_evtchn_to_cpu(evtchn, 0);
 461         info->evtchn = evtchn;
 462
 463 out:
 464         unmask_evtchn(evtchn);
 465         pirq_unmask_notify(irq);
 466
 467         return 0;
 468 }
 469
 470 static void shutdown_pirq(unsigned int irq)
 471 {
 472         struct evtchn_close close;
 473         struct irq_info *info = info_for_irq(irq);
 474         int evtchn = evtchn_from_irq(irq);
 475
 476         BUG_ON(info->type != IRQT_PIRQ);
 477
 478         if (!VALID_EVTCHN(evtchn))
 479                 return;
 480
 481         mask_evtchn(evtchn);
 482
 483         close.port = evtchn;
 484         if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0)
 485                 BUG();
 486
 487         bind_evtchn_to_cpu(evtchn, 0);
 488         evtchn_to_irq[evtchn] = -1;
 489         info->evtchn = 0;
 490 }
 491
 492 static void enable_pirq(unsigned int irq)
 493 {
 494         startup_pirq(irq);
 495 }
 496
 497 static void disable_pirq(unsigned int irq)
 498 {
 499 }
 500
 501 static void ack_pirq(unsigned int irq)
 502 {
 503         int evtchn = evtchn_from_irq(irq);
 504
 505         move_native_irq(irq);
 506
 507         if (VALID_EVTCHN(evtchn)) {
 508                 mask_evtchn(evtchn);
 509                 clear_evtchn(evtchn);
 510         }
 511 }
 512
 513 static void end_pirq(unsigned int irq)
 514 {
 515         int evtchn = evtchn_from_irq(irq);
 516         struct irq_desc *desc = irq_to_desc(irq);
 517
 518         if (WARN_ON(!desc))
 519                 return;
 520
 521         if ((desc->status & (IRQ_DISABLED|IRQ_PENDING)) ==
 522             (IRQ_DISABLED|IRQ_PENDING)) {
 523                 shutdown_pirq(irq);
 524         } else if (VALID_EVTCHN(evtchn)) {
 525                 unmask_evtchn(evtchn);
 526                 pirq_unmask_notify(irq);
 527         }
 528 }
 529
 530 static int find_irq_by_gsi(unsigned gsi)
 531 {
 532         int irq;
 533
 534         for (irq = 0; irq < nr_irqs; irq++) {
 535                 struct irq_info *info = info_for_irq(irq);
 536
 537                 if (info == NULL || info->type != IRQT_PIRQ)
 538                         continue;
 539
 540                 if (gsi_from_irq(irq) == gsi)
 541                         return irq;
 542         }
 543
 544         return -1;
 545 }
 546
 547 /*
 548  * Allocate a physical irq, along with a vector.  We don't assign an
 549  * event channel until the irq actually started up.  Return an
 550  * existing irq if we've already got one for the gsi.
 551  */
 552 int xen_allocate_pirq(unsigned gsi, char *name)
 553 {
 554         int irq;
 555         struct physdev_irq irq_op;
 556
 557         spin_lock(&irq_mapping_update_lock);
 558
 559         irq = find_irq_by_gsi(gsi);
 560         if (irq != -1) {
 561                 printk(KERN_INFO "xen_allocate_pirq: returning irq %d for gsi %u\n",
 562                        irq, gsi);
 563                 goto out;       /* XXX need refcount? */
 564         }
 565
 566         if (identity_mapped_irq(gsi)) {
 567                 irq = gsi;
 568                 irq_to_desc_alloc_node(irq, 0);
 569                 dynamic_irq_init(irq);
 570         } else
 571                 irq = find_unbound_irq();
 572
 573         set_irq_chip_and_handler_name(irq, &xen_pirq_chip,
 574                                       handle_level_irq, name);
 575
 576         irq_op.irq = irq;
 577         if (HYPERVISOR_physdev_op(PHYSDEVOP_alloc_irq_vector, &irq_op)) {
 578                 dynamic_irq_cleanup(irq);
 579                 irq = -ENOSPC;
 580                 goto out;
 581         }
 582
 583         irq_info[irq] = mk_pirq_info(0, gsi, irq_op.vector);
 584
 585 out:
 586         spin_unlock(&irq_mapping_update_lock);
 587
 588         return irq;
 589 }
 590
 591 int xen_vector_from_irq(unsigned irq)
 592 {
 593         return vector_from_irq(irq);
 594 }
 595
 596 int xen_gsi_from_irq(unsigned irq)
 597 {
 598         return gsi_from_irq(irq);
 599 }
 600
 601 int bind_evtchn_to_irq(unsigned int evtchn)
 602 {
 603         int irq;
 604
 605         spin_lock(&irq_mapping_update_lock);
 606
 607         irq = evtchn_to_irq[evtchn];
 608
 609         if (irq == -1) {
 610                 irq = find_unbound_irq();
 611
 612                 set_irq_chip_and_handler_name(irq, &xen_dynamic_chip,
 613                                               handle_edge_irq, "event");
 614
 615                 evtchn_to_irq[evtchn] = irq;
 616                 irq_info[irq] = mk_evtchn_info(evtchn);
 617         }
 618
 619         spin_unlock(&irq_mapping_update_lock);
 620
 621         return irq;
 622 }
 623 EXPORT_SYMBOL_GPL(bind_evtchn_to_irq);
 624
 625 static int bind_ipi_to_irq(unsigned int ipi, unsigned int cpu)
 626 {
 627         struct evtchn_bind_ipi bind_ipi;
 628         int evtchn, irq;
 629
 630         spin_lock(&irq_mapping_update_lock);
 631
 632         irq = per_cpu(ipi_to_irq, cpu)[ipi];
 633
 634         if (irq == -1) {
 635                 irq = find_unbound_irq();
 636                 if (irq < 0)
 637                         goto out;
 638
 639                 set_irq_chip_and_handler_name(irq, &xen_percpu_chip,
 640                                               handle_percpu_irq, "ipi");
 641
 642                 bind_ipi.vcpu = cpu;
 643                 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
 644                                                 &bind_ipi) != 0)
 645                         BUG();
 646                 evtchn = bind_ipi.port;
 647
 648                 evtchn_to_irq[evtchn] = irq;
 649                 irq_info[irq] = mk_ipi_info(evtchn, ipi);
 650                 per_cpu(ipi_to_irq, cpu)[ipi] = irq;
 651
 652                 bind_evtchn_to_cpu(evtchn, cpu);
 653         }
 654
 655  out:
 656         spin_unlock(&irq_mapping_update_lock);
 657         return irq;
 658 }
 659
 660
 661 static int bind_virq_to_irq(unsigned int virq, unsigned int cpu)
 662 {
 663         struct evtchn_bind_virq bind_virq;
 664         int evtchn, irq;
 665
 666         spin_lock(&irq_mapping_update_lock);
 667
 668         irq = per_cpu(virq_to_irq, cpu)[virq];
 669
 670         if (irq == -1) {
 671                 bind_virq.virq = virq;
 672                 bind_virq.vcpu = cpu;
 673                 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
 674                                                 &bind_virq) != 0)
 675                         BUG();
 676                 evtchn = bind_virq.port;
 677
 678                 irq = find_unbound_irq();
 679
 680                 set_irq_chip_and_handler_name(irq, &xen_percpu_chip,
 681                                               handle_percpu_irq, "virq");
 682
 683                 evtchn_to_irq[evtchn] = irq;
 684                 irq_info[irq] = mk_virq_info(evtchn, virq);
 685
 686                 per_cpu(virq_to_irq, cpu)[virq] = irq;
 687
 688                 bind_evtchn_to_cpu(evtchn, cpu);
 689         }
 690
 691         spin_unlock(&irq_mapping_update_lock);
 692
 693         return irq;
 694 }
 695
 696 static void unbind_from_irq(unsigned int irq)
 697 {
 698         struct evtchn_close close;
 699         int evtchn = evtchn_from_irq(irq);
 700
 701         spin_lock(&irq_mapping_update_lock);
 702
 703         if (VALID_EVTCHN(evtchn)) {
 704                 close.port = evtchn;
 705                 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0)
 706                         BUG();
 707
 708                 switch (type_from_irq(irq)) {
 709                 case IRQT_VIRQ:
 710                         per_cpu(virq_to_irq, cpu_from_evtchn(evtchn))
 711                                 [virq_from_irq(irq)] = -1;
 712                         break;
 713                 case IRQT_IPI:
 714                         per_cpu(ipi_to_irq, cpu_from_evtchn(evtchn))
 715                                 [ipi_from_irq(irq)] = -1;
 716                         break;
 717                 default:
 718                         break;
 719                 }
 720
 721                 /* Closed ports are implicitly re-bound to VCPU0. */
 722                 bind_evtchn_to_cpu(evtchn, 0);
 723
 724                 evtchn_to_irq[evtchn] = -1;
 725         }
 726
 727         if (irq_info[irq].type != IRQT_UNBOUND) {
 728                 irq_info[irq] = mk_unbound_info();
 729
 730                 irq_free_desc(irq);
 731         }
 732
 733         spin_unlock(&irq_mapping_update_lock);
 734 }
 735
 736 int bind_evtchn_to_irqhandler(unsigned int evtchn,
 737                               irq_handler_t handler,
 738                               unsigned long irqflags,
 739                               const char *devname, void *dev_id)
 740 {
 741         unsigned int irq;
 742         int retval;
 743
 744         irq = bind_evtchn_to_irq(evtchn);
 745         retval = request_irq(irq, handler, irqflags, devname, dev_id);
 746         if (retval != 0) {
 747                 unbind_from_irq(irq);
 748                 return retval;
 749         }
 750
 751         return irq;
 752 }
 753 EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler);
 754
 755 int bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu,
 756                             irq_handler_t handler,
 757                             unsigned long irqflags, const char *devname, void *dev_id)
 758 {
 759         unsigned int irq;
 760         int retval;
 761
 762         irq = bind_virq_to_irq(virq, cpu);
 763         retval = request_irq(irq, handler, irqflags, devname, dev_id);
 764         if (retval != 0) {
 765                 unbind_from_irq(irq);
 766                 return retval;
 767         }
 768
 769         return irq;
 770 }
 771 EXPORT_SYMBOL_GPL(bind_virq_to_irqhandler);
 772
 773 int bind_ipi_to_irqhandler(enum ipi_vector ipi,
 774                            unsigned int cpu,
 775                            irq_handler_t handler,
 776                            unsigned long irqflags,
 777                            const char *devname,
 778                            void *dev_id)
 779 {
 780         int irq, retval;
 781
 782         irq = bind_ipi_to_irq(ipi, cpu);
 783         if (irq < 0)
 784                 return irq;
 785
 786         irqflags |= IRQF_NO_SUSPEND;
 787         retval = request_irq(irq, handler, irqflags, devname, dev_id);
 788         if (retval != 0) {
 789                 unbind_from_irq(irq);
 790                 return retval;
 791         }
 792
 793         return irq;
 794 }
 795
 796 void unbind_from_irqhandler(unsigned int irq, void *dev_id)
 797 {
 798         free_irq(irq, dev_id);
 799         unbind_from_irq(irq);
 800 }
 801 EXPORT_SYMBOL_GPL(unbind_from_irqhandler);
 802
 803 void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector)
 804 {
 805         int irq = per_cpu(ipi_to_irq, cpu)[vector];
 806         BUG_ON(irq < 0);
 807         notify_remote_via_irq(irq);
 808 }
 809
 810 irqreturn_t xen_debug_interrupt(int irq, void *dev_id)
 811 {
 812         struct shared_info *sh = HYPERVISOR_shared_info;
 813         int cpu = smp_processor_id();
 814         int i;
 815         unsigned long flags;
 816         static DEFINE_SPINLOCK(debug_lock);
 817
 818         spin_lock_irqsave(&debug_lock, flags);
 819
 820         printk("vcpu %d\n  ", cpu);
 821
 822         for_each_online_cpu(i) {
 823                 struct vcpu_info *v = per_cpu(xen_vcpu, i);
 824                 printk("%d: masked=%d pending=%d event_sel %08lx\n  ", i,
 825                         (get_irq_regs() && i == cpu) ? xen_irqs_disabled(get_irq_regs()) : v->evtchn_upcall_mask,
 826                         v->evtchn_upcall_pending,
 827                         v->evtchn_pending_sel);
 828         }
 829         printk("pending:\n   ");
 830         for(i = ARRAY_SIZE(sh->evtchn_pending)-1; i >= 0; i--)
 831                 printk("%08lx%s", sh->evtchn_pending[i],
 832                         i % 8 == 0 ? "\n   " : " ");
 833         printk("\nmasks:\n   ");
 834         for(i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--)
 835                 printk("%08lx%s", sh->evtchn_mask[i],
 836                         i % 8 == 0 ? "\n   " : " ");
 837
 838         printk("\nunmasked:\n   ");
 839         for(i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--)
 840                 printk("%08lx%s", sh->evtchn_pending[i] & ~sh->evtchn_mask[i],
 841                         i % 8 == 0 ? "\n   " : " ");
 842
 843         printk("\npending list:\n");
 844         for(i = 0; i < NR_EVENT_CHANNELS; i++) {
 845                 if (sync_test_bit(i, sh->evtchn_pending)) {
 846                         printk("  %d: event %d -> irq %d\n",
 847                                cpu_from_evtchn(i), i,
 848                                evtchn_to_irq[i]);
 849                 }
 850         }
 851
 852         spin_unlock_irqrestore(&debug_lock, flags);
 853
 854         return IRQ_HANDLED;
 855 }
 856
 857 static DEFINE_PER_CPU(unsigned, xed_nesting_count);
 858
 859 /*
 860  * Search the CPUs pending events bitmasks.  For each one found, map
 861  * the event number to an irq, and feed it into do_IRQ() for
 862  * handling.
 863  *
 864  * Xen uses a two-level bitmap to speed searching.  The first level is
 865  * a bitset of words which contain pending event bits.  The second
 866  * level is a bitset of pending events themselves.
 867  */
 868 static void __xen_evtchn_do_upcall(void)
 869 {
 870         int cpu = get_cpu();
 871         struct shared_info *s = HYPERVISOR_shared_info;
 872         struct vcpu_info *vcpu_info = __get_cpu_var(xen_vcpu);
 873         unsigned count;
 874
 875         do {
 876                 unsigned long pending_words;
 877
 878                 vcpu_info->evtchn_upcall_pending = 0;
 879
 880                 if (__get_cpu_var(xed_nesting_count)++)
 881                         goto out;
 882
 883 #ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */
 884                 /* Clear master flag /before/ clearing selector flag. */
 885                 wmb();
 886 #endif
 887                 pending_words = xchg(&vcpu_info->evtchn_pending_sel, 0);
 888                 while (pending_words != 0) {
 889                         unsigned long pending_bits;
 890                         int word_idx = __ffs(pending_words);
 891                         pending_words &= ~(1UL << word_idx);
 892
 893                         while ((pending_bits = active_evtchns(cpu, s, word_idx)) != 0) {
 894                                 int bit_idx = __ffs(pending_bits);
 895                                 int port = (word_idx * BITS_PER_LONG) + bit_idx;
 896                                 int irq = evtchn_to_irq[port];
 897                                 struct irq_desc *desc;
 898
 899                                 if (irq != -1) {
 900                                         desc = irq_to_desc(irq);
 901                                         if (desc)
 902                                                 generic_handle_irq_desc(irq, desc);
 903                                 }
 904                         }
 905                 }
 906
 907                 BUG_ON(!irqs_disabled());
 908
 909                 count = __get_cpu_var(xed_nesting_count);
 910                 __get_cpu_var(xed_nesting_count) = 0;
 911         } while (count != 1 || vcpu_info->evtchn_upcall_pending);
 912
 913 out:
 914
 915         put_cpu();
 916 }
 917
 918 void xen_evtchn_do_upcall(struct pt_regs *regs)
 919 {
 920         struct pt_regs *old_regs = set_irq_regs(regs);
 921
 922         exit_idle();
 923         irq_enter();
 924
 925         __xen_evtchn_do_upcall();
 926
 927         irq_exit();
 928         set_irq_regs(old_regs);
 929 }
 930
 931 void xen_hvm_evtchn_do_upcall(void)
 932 {
 933         __xen_evtchn_do_upcall();
 934 }
 935 EXPORT_SYMBOL_GPL(xen_hvm_evtchn_do_upcall);
 936
 937 /* Rebind a new event channel to an existing irq. */
 938 void rebind_evtchn_irq(int evtchn, int irq)
 939 {
 940         struct irq_info *info = info_for_irq(irq);
 941
 942         /* Make sure the irq is masked, since the new event channel
 943            will also be masked. */
 944         disable_irq(irq);
 945
 946         spin_lock(&irq_mapping_update_lock);
 947
 948         /* After resume the irq<->evtchn mappings are all cleared out */
 949         BUG_ON(evtchn_to_irq[evtchn] != -1);
 950         /* Expect irq to have been bound before,
 951            so there should be a proper type */
 952         BUG_ON(info->type == IRQT_UNBOUND);
 953
 954         evtchn_to_irq[evtchn] = irq;
 955         irq_info[irq] = mk_evtchn_info(evtchn);
 956
 957         spin_unlock(&irq_mapping_update_lock);
 958
 959         /* new event channels are always bound to cpu 0 */
 960         irq_set_affinity(irq, cpumask_of(0));
 961
 962         /* Unmask the event channel. */
 963         enable_irq(irq);
 964 }
 965
 966 /* Rebind an evtchn so that it gets delivered to a specific cpu */
 967 static int rebind_irq_to_cpu(unsigned irq, unsigned tcpu)
 968 {
 969         struct evtchn_bind_vcpu bind_vcpu;
 970         int evtchn = evtchn_from_irq(irq);
 971
 972         /* events delivered via platform PCI interrupts are always
 973          * routed to vcpu 0 */
 974         if (!VALID_EVTCHN(evtchn) ||
 975                 (xen_hvm_domain() && !xen_have_vector_callback))
 976                 return -1;
 977
 978         /* Send future instances of this interrupt to other vcpu. */
 979         bind_vcpu.port = evtchn;
 980         bind_vcpu.vcpu = tcpu;
 981
 982         /*
 983          * If this fails, it usually just indicates that we're dealing with a
 984          * virq or IPI channel, which don't actually need to be rebound. Ignore
 985          * it, but don't do the xenlinux-level rebind in that case.
 986          */
 987         if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu) >= 0)
 988                 bind_evtchn_to_cpu(evtchn, tcpu);
 989
 990         return 0;
 991 }
 992
 993 static int set_affinity_irq(unsigned irq, const struct cpumask *dest)
 994 {
 995         unsigned tcpu = cpumask_first(dest);
 996
 997         return rebind_irq_to_cpu(irq, tcpu);
 998 }
 999
1000 int resend_irq_on_evtchn(unsigned int irq)
1001 {
1002         int masked, evtchn = evtchn_from_irq(irq);
1003         struct shared_info *s = HYPERVISOR_shared_info;
1004
1005         if (!VALID_EVTCHN(evtchn))
1006                 return 1;
1007
1008         masked = sync_test_and_set_bit(evtchn, s->evtchn_mask);
1009         sync_set_bit(evtchn, s->evtchn_pending);
1010         if (!masked)
1011                 unmask_evtchn(evtchn);
1012
1013         return 1;
1014 }
1015
1016 static void enable_dynirq(unsigned int irq)
1017 {
1018         int evtchn = evtchn_from_irq(irq);
1019
1020         if (VALID_EVTCHN(evtchn))
1021                 unmask_evtchn(evtchn);
1022 }
1023
1024 static void disable_dynirq(unsigned int irq)
1025 {
1026         int evtchn = evtchn_from_irq(irq);
1027
1028         if (VALID_EVTCHN(evtchn))
1029                 mask_evtchn(evtchn);
1030 }
1031
1032 static void ack_dynirq(unsigned int irq)
1033 {
1034         int evtchn = evtchn_from_irq(irq);
1035
1036         move_native_irq(irq);
1037
1038         if (VALID_EVTCHN(evtchn))
1039                 clear_evtchn(evtchn);
1040 }
1041
1042 static int retrigger_dynirq(unsigned int irq)
1043 {
1044         int evtchn = evtchn_from_irq(irq);
1045         struct shared_info *sh = HYPERVISOR_shared_info;
1046         int ret = 0;
1047
1048         if (VALID_EVTCHN(evtchn)) {
1049                 int masked;
1050
1051                 masked = sync_test_and_set_bit(evtchn, sh->evtchn_mask);
1052                 sync_set_bit(evtchn, sh->evtchn_pending);
1053                 if (!masked)
1054                         unmask_evtchn(evtchn);
1055                 ret = 1;
1056         }
1057
1058         return ret;
1059 }
1060
1061 static void restore_cpu_virqs(unsigned int cpu)
1062 {
1063         struct evtchn_bind_virq bind_virq;
1064         int virq, irq, evtchn;
1065
1066         for (virq = 0; virq < NR_VIRQS; virq++) {
1067                 if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1)
1068                         continue;
1069
1070                 BUG_ON(virq_from_irq(irq) != virq);
1071
1072                 /* Get a new binding from Xen. */
1073                 bind_virq.virq = virq;
1074                 bind_virq.vcpu = cpu;
1075                 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
1076                                                 &bind_virq) != 0)
1077                         BUG();
1078                 evtchn = bind_virq.port;
1079
1080                 /* Record the new mapping. */
1081                 evtchn_to_irq[evtchn] = irq;
1082                 irq_info[irq] = mk_virq_info(evtchn, virq);
1083                 bind_evtchn_to_cpu(evtchn, cpu);
1084
1085                 /* Ready for use. */
1086                 unmask_evtchn(evtchn);
1087         }
1088 }
1089
1090 static void restore_cpu_ipis(unsigned int cpu)
1091 {
1092         struct evtchn_bind_ipi bind_ipi;
1093         int ipi, irq, evtchn;
1094
1095         for (ipi = 0; ipi < XEN_NR_IPIS; ipi++) {
1096                 if ((irq = per_cpu(ipi_to_irq, cpu)[ipi]) == -1)
1097                         continue;
1098
1099                 BUG_ON(ipi_from_irq(irq) != ipi);
1100
1101                 /* Get a new binding from Xen. */
1102                 bind_ipi.vcpu = cpu;
1103                 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
1104                                                 &bind_ipi) != 0)
1105                         BUG();
1106                 evtchn = bind_ipi.port;
1107
1108                 /* Record the new mapping. */
1109                 evtchn_to_irq[evtchn] = irq;
1110                 irq_info[irq] = mk_ipi_info(evtchn, ipi);
1111                 bind_evtchn_to_cpu(evtchn, cpu);
1112
1113                 /* Ready for use. */
1114                 unmask_evtchn(evtchn);
1115
1116         }
1117 }
1118
1119 /* Clear an irq's pending state, in preparation for polling on it */
1120 void xen_clear_irq_pending(int irq)
1121 {
1122         int evtchn = evtchn_from_irq(irq);
1123
1124         if (VALID_EVTCHN(evtchn))
1125                 clear_evtchn(evtchn);
1126 }
1127
1128 void xen_set_irq_pending(int irq)
1129 {
1130         int evtchn = evtchn_from_irq(irq);
1131
1132         if (VALID_EVTCHN(evtchn))
1133                 set_evtchn(evtchn);
1134 }
1135
1136 bool xen_test_irq_pending(int irq)
1137 {
1138         int evtchn = evtchn_from_irq(irq);
1139         bool ret = false;
1140
1141         if (VALID_EVTCHN(evtchn))
1142                 ret = test_evtchn(evtchn);
1143
1144         return ret;
1145 }
1146
1147 /* Poll waiting for an irq to become pending.  In the usual case, the
1148    irq will be disabled so it won't deliver an interrupt. */
1149 void xen_poll_irq(int irq)
1150 {
1151         evtchn_port_t evtchn = evtchn_from_irq(irq);
1152
1153         if (VALID_EVTCHN(evtchn)) {
1154                 struct sched_poll poll;
1155
1156                 poll.nr_ports = 1;
1157                 poll.timeout = 0;
1158                 set_xen_guest_handle(poll.ports, &evtchn);
1159
1160                 if (HYPERVISOR_sched_op(SCHEDOP_poll, &poll) != 0)
1161                         BUG();
1162         }
1163 }
1164
1165 void xen_irq_resume(void)
1166 {
1167         unsigned int cpu, irq, evtchn;
1168
1169         init_evtchn_cpu_bindings();
1170
1171         /* New event-channel space is not 'live' yet. */
1172         for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++)
1173                 mask_evtchn(evtchn);
1174
1175         /* No IRQ <-> event-channel mappings. */
1176         for (irq = 0; irq < nr_irqs; irq++)
1177                 irq_info[irq].evtchn = 0; /* zap event-channel binding */
1178
1179         for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++)
1180                 evtchn_to_irq[evtchn] = -1;
1181
1182         for_each_possible_cpu(cpu) {
1183                 restore_cpu_virqs(cpu);
1184                 restore_cpu_ipis(cpu);
1185         }
1186 }
1187
1188 static struct irq_chip xen_dynamic_chip __read_mostly = {
1189         .name           = "xen-dyn",
1190
1191         .disable        = disable_dynirq,
1192         .mask           = disable_dynirq,
1193         .unmask         = enable_dynirq,
1194
1195         .ack            = ack_dynirq,
1196         .set_affinity   = set_affinity_irq,
1197         .retrigger      = retrigger_dynirq,
1198 };
1199
1200 static struct irq_chip xen_pirq_chip __read_mostly = {
1201         .name           = "xen-pirq",
1202
1203         .startup        = startup_pirq,
1204         .shutdown       = shutdown_pirq,
1205
1206         .enable         = enable_pirq,
1207         .unmask         = enable_pirq,
1208
1209         .disable        = disable_pirq,
1210         .mask           = disable_pirq,
1211
1212         .ack            = ack_pirq,
1213         .end            = end_pirq,
1214
1215         .set_affinity   = set_affinity_irq,
1216
1217         .retrigger      = retrigger_dynirq,
1218 };
1219
1220 static struct irq_chip xen_percpu_chip __read_mostly = {
1221         .name           = "xen-percpu",
1222
1223         .disable        = disable_dynirq,
1224         .mask           = disable_dynirq,
1225         .unmask         = enable_dynirq,
1226
1227         .ack            = ack_dynirq,
1228 };
1229
1230 int xen_set_callback_via(uint64_t via)
1231 {
1232         struct xen_hvm_param a;
1233         a.domid = DOMID_SELF;
1234         a.index = HVM_PARAM_CALLBACK_IRQ;
1235         a.value = via;
1236         return HYPERVISOR_hvm_op(HVMOP_set_param, &a);
1237 }
1238 EXPORT_SYMBOL_GPL(xen_set_callback_via);
1239
1240 #ifdef CONFIG_XEN_PVHVM
1241 /* Vector callbacks are better than PCI interrupts to receive event
1242  * channel notifications because we can receive vector callbacks on any
1243  * vcpu and we don't need PCI support or APIC interactions. */
1244 void xen_callback_vector(void)
1245 {
1246         int rc;
1247         uint64_t callback_via;
1248         if (xen_have_vector_callback) {
1249                 callback_via = HVM_CALLBACK_VECTOR(XEN_HVM_EVTCHN_CALLBACK);
1250                 rc = xen_set_callback_via(callback_via);
1251                 if (rc) {
1252                         printk(KERN_ERR "Request for Xen HVM callback vector"
1253                                         " failed.\n");
1254                         xen_have_vector_callback = 0;
1255                         return;
1256                 }
1257                 printk(KERN_INFO "Xen HVM callback vector for event delivery is "
1258                                 "enabled\n");
1259                 /* in the restore case the vector has already been allocated */
1260                 if (!test_bit(XEN_HVM_EVTCHN_CALLBACK, used_vectors))
1261                         alloc_intr_gate(XEN_HVM_EVTCHN_CALLBACK, xen_hvm_callback_vector);
1262         }
1263 }
1264 #else
1265 void xen_callback_vector(void) {}
1266 #endif
1267
1268 void __init xen_init_IRQ(void)
1269 {
1270         int i;
1271
1272         cpu_evtchn_mask_p = kcalloc(nr_cpu_ids, sizeof(struct cpu_evtchn_s),
1273                                     GFP_KERNEL);
1274         irq_info = kcalloc(nr_irqs, sizeof(*irq_info), GFP_KERNEL);
1275
1276         evtchn_to_irq = kcalloc(NR_EVENT_CHANNELS, sizeof(*evtchn_to_irq),
1277                                     GFP_KERNEL);
1278         for (i = 0; i < NR_EVENT_CHANNELS; i++)
1279                 evtchn_to_irq[i] = -1;
1280
1281         init_evtchn_cpu_bindings();
1282
1283         /* No event channels are 'live' right now. */
1284         for (i = 0; i < NR_EVENT_CHANNELS; i++)
1285                 mask_evtchn(i);
1286
1287         if (xen_hvm_domain()) {
1288                 xen_callback_vector();
1289                 native_init_IRQ();
1290         } else {
1291                 irq_ctx_init(smp_processor_id());
1292         }
1293 }