arch/sparc/kernel/irq_64.c

   1 /* irq.c: UltraSparc IRQ handling/init/registry.
   2  *
   3  * Copyright (C) 1997, 2007, 2008 David S. Miller (davem@davemloft.net)
   4  * Copyright (C) 1998  Eddie C. Dost    (ecd@skynet.be)
   5  * Copyright (C) 1998  Jakub Jelinek    (jj@ultra.linux.cz)
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/sched.h>
  10 #include <linux/linkage.h>
  11 #include <linux/ptrace.h>
  12 #include <linux/errno.h>
  13 #include <linux/kernel_stat.h>
  14 #include <linux/signal.h>
  15 #include <linux/mm.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/slab.h>
  18 #include <linux/random.h>
  19 #include <linux/init.h>
  20 #include <linux/delay.h>
  21 #include <linux/proc_fs.h>
  22 #include <linux/seq_file.h>
  23 #include <linux/ftrace.h>
  24 #include <linux/irq.h>
  25 #include <linux/kmemleak.h>
  26
  27 #include <asm/ptrace.h>
  28 #include <asm/processor.h>
  29 #include <asm/atomic.h>
  30 #include <asm/system.h>
  31 #include <asm/irq.h>
  32 #include <asm/io.h>
  33 #include <asm/iommu.h>
  34 #include <asm/upa.h>
  35 #include <asm/oplib.h>
  36 #include <asm/prom.h>
  37 #include <asm/timer.h>
  38 #include <asm/smp.h>
  39 #include <asm/starfire.h>
  40 #include <asm/uaccess.h>
  41 #include <asm/cache.h>
  42 #include <asm/cpudata.h>
  43 #include <asm/auxio.h>
  44 #include <asm/head.h>
  45 #include <asm/hypervisor.h>
  46 #include <asm/cacheflush.h>
  47
  48 #include "entry.h"
  49 #include "cpumap.h"
  50 #include "kstack.h"
  51
  52 #define NUM_IVECS       (IMAP_INR + 1)
  53
  54 struct ino_bucket *ivector_table;
  55 unsigned long ivector_table_pa;
  56
  57 /* On several sun4u processors, it is illegal to mix bypass and
  58  * non-bypass accesses.  Therefore we access all INO buckets
  59  * using bypass accesses only.
  60  */
  61 static unsigned long bucket_get_chain_pa(unsigned long bucket_pa)
  62 {
  63         unsigned long ret;
  64
  65         __asm__ __volatile__("ldxa      [%1] %2, %0"
  66                              : "=&r" (ret)
  67                              : "r" (bucket_pa +
  68                                     offsetof(struct ino_bucket,
  69                                              __irq_chain_pa)),
  70                                "i" (ASI_PHYS_USE_EC));
  71
  72         return ret;
  73 }
  74
  75 static void bucket_clear_chain_pa(unsigned long bucket_pa)
  76 {
  77         __asm__ __volatile__("stxa      %%g0, [%0] %1"
  78                              : /* no outputs */
  79                              : "r" (bucket_pa +
  80                                     offsetof(struct ino_bucket,
  81                                              __irq_chain_pa)),
  82                                "i" (ASI_PHYS_USE_EC));
  83 }
  84
  85 static unsigned int bucket_get_virt_irq(unsigned long bucket_pa)
  86 {
  87         unsigned int ret;
  88
  89         __asm__ __volatile__("lduwa     [%1] %2, %0"
  90                              : "=&r" (ret)
  91                              : "r" (bucket_pa +
  92                                     offsetof(struct ino_bucket,
  93                                              __virt_irq)),
  94                                "i" (ASI_PHYS_USE_EC));
  95
  96         return ret;
  97 }
  98
  99 static void bucket_set_virt_irq(unsigned long bucket_pa,
 100                                 unsigned int virt_irq)
 101 {
 102         __asm__ __volatile__("stwa      %0, [%1] %2"
 103                              : /* no outputs */
 104                              : "r" (virt_irq),
 105                                "r" (bucket_pa +
 106                                     offsetof(struct ino_bucket,
 107                                              __virt_irq)),
 108                                "i" (ASI_PHYS_USE_EC));
 109 }
 110
 111 #define irq_work_pa(__cpu)      &(trap_block[(__cpu)].irq_worklist_pa)
 112
 113 static struct {
 114         unsigned int dev_handle;
 115         unsigned int dev_ino;
 116         unsigned int in_use;
 117 } virt_irq_table[NR_IRQS];
 118 static DEFINE_SPINLOCK(virt_irq_alloc_lock);
 119
 120 unsigned char virt_irq_alloc(unsigned int dev_handle,
 121                              unsigned int dev_ino)
 122 {
 123         unsigned long flags;
 124         unsigned char ent;
 125
 126         BUILD_BUG_ON(NR_IRQS >= 256);
 127
 128         spin_lock_irqsave(&virt_irq_alloc_lock, flags);
 129
 130         for (ent = 1; ent < NR_IRQS; ent++) {
 131                 if (!virt_irq_table[ent].in_use)
 132                         break;
 133         }
 134         if (ent >= NR_IRQS) {
 135                 printk(KERN_ERR "IRQ: Out of virtual IRQs.\n");
 136                 ent = 0;
 137         } else {
 138                 virt_irq_table[ent].dev_handle = dev_handle;
 139                 virt_irq_table[ent].dev_ino = dev_ino;
 140                 virt_irq_table[ent].in_use = 1;
 141         }
 142
 143         spin_unlock_irqrestore(&virt_irq_alloc_lock, flags);
 144
 145         return ent;
 146 }
 147
 148 #ifdef CONFIG_PCI_MSI
 149 void virt_irq_free(unsigned int virt_irq)
 150 {
 151         unsigned long flags;
 152
 153         if (virt_irq >= NR_IRQS)
 154                 return;
 155
 156         spin_lock_irqsave(&virt_irq_alloc_lock, flags);
 157
 158         virt_irq_table[virt_irq].in_use = 0;
 159
 160         spin_unlock_irqrestore(&virt_irq_alloc_lock, flags);
 161 }
 162 #endif
 163
 164 /*
 165  * /proc/interrupts printing:
 166  */
 167
 168 int show_interrupts(struct seq_file *p, void *v)
 169 {
 170         int i = *(loff_t *) v, j;
 171         struct irqaction * action;
 172         unsigned long flags;
 173
 174         if (i == 0) {
 175                 seq_printf(p, "           ");
 176                 for_each_online_cpu(j)
 177                         seq_printf(p, "CPU%d       ",j);
 178                 seq_putc(p, '\n');
 179         }
 180
 181         if (i < NR_IRQS) {
 182                 raw_spin_lock_irqsave(&irq_desc[i].lock, flags);
 183                 action = irq_desc[i].action;
 184                 if (!action)
 185                         goto skip;
 186                 seq_printf(p, "%3d: ",i);
 187 #ifndef CONFIG_SMP
 188                 seq_printf(p, "%10u ", kstat_irqs(i));
 189 #else
 190                 for_each_online_cpu(j)
 191                         seq_printf(p, "%10u ", kstat_irqs_cpu(i, j));
 192 #endif
 193                 seq_printf(p, " %9s", irq_desc[i].chip->name);
 194                 seq_printf(p, "  %s", action->name);
 195
 196                 for (action=action->next; action; action = action->next)
 197                         seq_printf(p, ", %s", action->name);
 198
 199                 seq_putc(p, '\n');
 200 skip:
 201                 raw_spin_unlock_irqrestore(&irq_desc[i].lock, flags);
 202         } else if (i == NR_IRQS) {
 203                 seq_printf(p, "NMI: ");
 204                 for_each_online_cpu(j)
 205                         seq_printf(p, "%10u ", cpu_data(j).__nmi_count);
 206                 seq_printf(p, "     Non-maskable interrupts\n");
 207         }
 208         return 0;
 209 }
 210
 211 static unsigned int sun4u_compute_tid(unsigned long imap, unsigned long cpuid)
 212 {
 213         unsigned int tid;
 214
 215         if (this_is_starfire) {
 216                 tid = starfire_translate(imap, cpuid);
 217                 tid <<= IMAP_TID_SHIFT;
 218                 tid &= IMAP_TID_UPA;
 219         } else {
 220                 if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 221                         unsigned long ver;
 222
 223                         __asm__ ("rdpr %%ver, %0" : "=r" (ver));
 224                         if ((ver >> 32UL) == __JALAPENO_ID ||
 225                             (ver >> 32UL) == __SERRANO_ID) {
 226                                 tid = cpuid << IMAP_TID_SHIFT;
 227                                 tid &= IMAP_TID_JBUS;
 228                         } else {
 229                                 unsigned int a = cpuid & 0x1f;
 230                                 unsigned int n = (cpuid >> 5) & 0x1f;
 231
 232                                 tid = ((a << IMAP_AID_SHIFT) |
 233                                        (n << IMAP_NID_SHIFT));
 234                                 tid &= (IMAP_AID_SAFARI |
 235                                         IMAP_NID_SAFARI);
 236                         }
 237                 } else {
 238                         tid = cpuid << IMAP_TID_SHIFT;
 239                         tid &= IMAP_TID_UPA;
 240                 }
 241         }
 242
 243         return tid;
 244 }
 245
 246 struct irq_handler_data {
 247         unsigned long   iclr;
 248         unsigned long   imap;
 249
 250         void            (*pre_handler)(unsigned int, void *, void *);
 251         void            *arg1;
 252         void            *arg2;
 253 };
 254
 255 #ifdef CONFIG_SMP
 256 static int irq_choose_cpu(unsigned int virt_irq, const struct cpumask *affinity)
 257 {
 258         cpumask_t mask;
 259         int cpuid;
 260
 261         cpumask_copy(&mask, affinity);
 262         if (cpus_equal(mask, cpu_online_map)) {
 263                 cpuid = map_to_cpu(virt_irq);
 264         } else {
 265                 cpumask_t tmp;
 266
 267                 cpus_and(tmp, cpu_online_map, mask);
 268                 cpuid = cpus_empty(tmp) ? map_to_cpu(virt_irq) : first_cpu(tmp);
 269         }
 270
 271         return cpuid;
 272 }
 273 #else
 274 #define irq_choose_cpu(virt_irq, affinity)      \
 275         real_hard_smp_processor_id()
 276 #endif
 277
 278 static void sun4u_irq_enable(unsigned int virt_irq)
 279 {
 280         struct irq_handler_data *data = get_irq_chip_data(virt_irq);
 281
 282         if (likely(data)) {
 283                 unsigned long cpuid, imap, val;
 284                 unsigned int tid;
 285
 286                 cpuid = irq_choose_cpu(virt_irq,
 287                                        irq_desc[virt_irq].affinity);
 288                 imap = data->imap;
 289
 290                 tid = sun4u_compute_tid(imap, cpuid);
 291
 292                 val = upa_readq(imap);
 293                 val &= ~(IMAP_TID_UPA | IMAP_TID_JBUS |
 294                          IMAP_AID_SAFARI | IMAP_NID_SAFARI);
 295                 val |= tid | IMAP_VALID;
 296                 upa_writeq(val, imap);
 297                 upa_writeq(ICLR_IDLE, data->iclr);
 298         }
 299 }
 300
 301 static int sun4u_set_affinity(unsigned int virt_irq,
 302                                const struct cpumask *mask)
 303 {
 304         struct irq_handler_data *data = get_irq_chip_data(virt_irq);
 305
 306         if (likely(data)) {
 307                 unsigned long cpuid, imap, val;
 308                 unsigned int tid;
 309
 310                 cpuid = irq_choose_cpu(virt_irq, mask);
 311                 imap = data->imap;
 312
 313                 tid = sun4u_compute_tid(imap, cpuid);
 314
 315                 val = upa_readq(imap);
 316                 val &= ~(IMAP_TID_UPA | IMAP_TID_JBUS |
 317                          IMAP_AID_SAFARI | IMAP_NID_SAFARI);
 318                 val |= tid | IMAP_VALID;
 319                 upa_writeq(val, imap);
 320                 upa_writeq(ICLR_IDLE, data->iclr);
 321         }
 322
 323         return 0;
 324 }
 325
 326 /* Don't do anything.  The desc->status check for IRQ_DISABLED in
 327  * handler_irq() will skip the handler call and that will leave the
 328  * interrupt in the sent state.  The next ->enable() call will hit the
 329  * ICLR register to reset the state machine.
 330  *
 331  * This scheme is necessary, instead of clearing the Valid bit in the
 332  * IMAP register, to handle the case of IMAP registers being shared by
 333  * multiple INOs (and thus ICLR registers).  Since we use a different
 334  * virtual IRQ for each shared IMAP instance, the generic code thinks
 335  * there is only one user so it prematurely calls ->disable() on
 336  * free_irq().
 337  *
 338  * We have to provide an explicit ->disable() method instead of using
 339  * NULL to get the default.  The reason is that if the generic code
 340  * sees that, it also hooks up a default ->shutdown method which
 341  * invokes ->mask() which we do not want.  See irq_chip_set_defaults().
 342  */
 343 static void sun4u_irq_disable(unsigned int virt_irq)
 344 {
 345 }
 346
 347 static void sun4u_irq_eoi(unsigned int virt_irq)
 348 {
 349         struct irq_handler_data *data = get_irq_chip_data(virt_irq);
 350         struct irq_desc *desc = irq_desc + virt_irq;
 351
 352         if (unlikely(desc->status & (IRQ_DISABLED|IRQ_INPROGRESS)))
 353                 return;
 354
 355         if (likely(data))
 356                 upa_writeq(ICLR_IDLE, data->iclr);
 357 }
 358
 359 static void sun4v_irq_enable(unsigned int virt_irq)
 360 {
 361         unsigned int ino = virt_irq_table[virt_irq].dev_ino;
 362         unsigned long cpuid = irq_choose_cpu(virt_irq,
 363                                              irq_desc[virt_irq].affinity);
 364         int err;
 365
 366         err = sun4v_intr_settarget(ino, cpuid);
 367         if (err != HV_EOK)
 368                 printk(KERN_ERR "sun4v_intr_settarget(%x,%lu): "
 369                        "err(%d)\n", ino, cpuid, err);
 370         err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE);
 371         if (err != HV_EOK)
 372                 printk(KERN_ERR "sun4v_intr_setstate(%x): "
 373                        "err(%d)\n", ino, err);
 374         err = sun4v_intr_setenabled(ino, HV_INTR_ENABLED);
 375         if (err != HV_EOK)
 376                 printk(KERN_ERR "sun4v_intr_setenabled(%x): err(%d)\n",
 377                        ino, err);
 378 }
 379
 380 static int sun4v_set_affinity(unsigned int virt_irq,
 381                                const struct cpumask *mask)
 382 {
 383         unsigned int ino = virt_irq_table[virt_irq].dev_ino;
 384         unsigned long cpuid = irq_choose_cpu(virt_irq, mask);
 385         int err;
 386
 387         err = sun4v_intr_settarget(ino, cpuid);
 388         if (err != HV_EOK)
 389                 printk(KERN_ERR "sun4v_intr_settarget(%x,%lu): "
 390                        "err(%d)\n", ino, cpuid, err);
 391
 392         return 0;
 393 }
 394
 395 static void sun4v_irq_disable(unsigned int virt_irq)
 396 {
 397         unsigned int ino = virt_irq_table[virt_irq].dev_ino;
 398         int err;
 399
 400         err = sun4v_intr_setenabled(ino, HV_INTR_DISABLED);
 401         if (err != HV_EOK)
 402                 printk(KERN_ERR "sun4v_intr_setenabled(%x): "
 403                        "err(%d)\n", ino, err);
 404 }
 405
 406 static void sun4v_irq_eoi(unsigned int virt_irq)
 407 {
 408         unsigned int ino = virt_irq_table[virt_irq].dev_ino;
 409         struct irq_desc *desc = irq_desc + virt_irq;
 410         int err;
 411
 412         if (unlikely(desc->status & (IRQ_DISABLED|IRQ_INPROGRESS)))
 413                 return;
 414
 415         err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE);
 416         if (err != HV_EOK)
 417                 printk(KERN_ERR "sun4v_intr_setstate(%x): "
 418                        "err(%d)\n", ino, err);
 419 }
 420
 421 static void sun4v_virq_enable(unsigned int virt_irq)
 422 {
 423         unsigned long cpuid, dev_handle, dev_ino;
 424         int err;
 425
 426         cpuid = irq_choose_cpu(virt_irq, irq_desc[virt_irq].affinity);
 427
 428         dev_handle = virt_irq_table[virt_irq].dev_handle;
 429         dev_ino = virt_irq_table[virt_irq].dev_ino;
 430
 431         err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid);
 432         if (err != HV_EOK)
 433                 printk(KERN_ERR "sun4v_vintr_set_target(%lx,%lx,%lu): "
 434                        "err(%d)\n",
 435                        dev_handle, dev_ino, cpuid, err);
 436         err = sun4v_vintr_set_state(dev_handle, dev_ino,
 437                                     HV_INTR_STATE_IDLE);
 438         if (err != HV_EOK)
 439                 printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx,"
 440                        "HV_INTR_STATE_IDLE): err(%d)\n",
 441                        dev_handle, dev_ino, err);
 442         err = sun4v_vintr_set_valid(dev_handle, dev_ino,
 443                                     HV_INTR_ENABLED);
 444         if (err != HV_EOK)
 445                 printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx,"
 446                        "HV_INTR_ENABLED): err(%d)\n",
 447                        dev_handle, dev_ino, err);
 448 }
 449
 450 static int sun4v_virt_set_affinity(unsigned int virt_irq,
 451                                     const struct cpumask *mask)
 452 {
 453         unsigned long cpuid, dev_handle, dev_ino;
 454         int err;
 455
 456         cpuid = irq_choose_cpu(virt_irq, mask);
 457
 458         dev_handle = virt_irq_table[virt_irq].dev_handle;
 459         dev_ino = virt_irq_table[virt_irq].dev_ino;
 460
 461         err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid);
 462         if (err != HV_EOK)
 463                 printk(KERN_ERR "sun4v_vintr_set_target(%lx,%lx,%lu): "
 464                        "err(%d)\n",
 465                        dev_handle, dev_ino, cpuid, err);
 466
 467         return 0;
 468 }
 469
 470 static void sun4v_virq_disable(unsigned int virt_irq)
 471 {
 472         unsigned long dev_handle, dev_ino;
 473         int err;
 474
 475         dev_handle = virt_irq_table[virt_irq].dev_handle;
 476         dev_ino = virt_irq_table[virt_irq].dev_ino;
 477
 478         err = sun4v_vintr_set_valid(dev_handle, dev_ino,
 479                                     HV_INTR_DISABLED);
 480         if (err != HV_EOK)
 481                 printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx,"
 482                        "HV_INTR_DISABLED): err(%d)\n",
 483                        dev_handle, dev_ino, err);
 484 }
 485
 486 static void sun4v_virq_eoi(unsigned int virt_irq)
 487 {
 488         struct irq_desc *desc = irq_desc + virt_irq;
 489         unsigned long dev_handle, dev_ino;
 490         int err;
 491
 492         if (unlikely(desc->status & (IRQ_DISABLED|IRQ_INPROGRESS)))
 493                 return;
 494
 495         dev_handle = virt_irq_table[virt_irq].dev_handle;
 496         dev_ino = virt_irq_table[virt_irq].dev_ino;
 497
 498         err = sun4v_vintr_set_state(dev_handle, dev_ino,
 499                                     HV_INTR_STATE_IDLE);
 500         if (err != HV_EOK)
 501                 printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx,"
 502                        "HV_INTR_STATE_IDLE): err(%d)\n",
 503                        dev_handle, dev_ino, err);
 504 }
 505
 506 static struct irq_chip sun4u_irq = {
 507         .name           = "sun4u",
 508         .enable         = sun4u_irq_enable,
 509         .disable        = sun4u_irq_disable,
 510         .eoi            = sun4u_irq_eoi,
 511         .set_affinity   = sun4u_set_affinity,
 512 };
 513
 514 static struct irq_chip sun4v_irq = {
 515         .name           = "sun4v",
 516         .enable         = sun4v_irq_enable,
 517         .disable        = sun4v_irq_disable,
 518         .eoi            = sun4v_irq_eoi,
 519         .set_affinity   = sun4v_set_affinity,
 520 };
 521
 522 static struct irq_chip sun4v_virq = {
 523         .name           = "vsun4v",
 524         .enable         = sun4v_virq_enable,
 525         .disable        = sun4v_virq_disable,
 526         .eoi            = sun4v_virq_eoi,
 527         .set_affinity   = sun4v_virt_set_affinity,
 528 };
 529
 530 static void pre_flow_handler(unsigned int virt_irq,
 531                                       struct irq_desc *desc)
 532 {
 533         struct irq_handler_data *data = get_irq_chip_data(virt_irq);
 534         unsigned int ino = virt_irq_table[virt_irq].dev_ino;
 535
 536         data->pre_handler(ino, data->arg1, data->arg2);
 537
 538         handle_fasteoi_irq(virt_irq, desc);
 539 }
 540
 541 void irq_install_pre_handler(int virt_irq,
 542                              void (*func)(unsigned int, void *, void *),
 543                              void *arg1, void *arg2)
 544 {
 545         struct irq_handler_data *data = get_irq_chip_data(virt_irq);
 546         struct irq_desc *desc = irq_desc + virt_irq;
 547
 548         data->pre_handler = func;
 549         data->arg1 = arg1;
 550         data->arg2 = arg2;
 551
 552         desc->handle_irq = pre_flow_handler;
 553 }
 554
 555 unsigned int build_irq(int inofixup, unsigned long iclr, unsigned long imap)
 556 {
 557         struct ino_bucket *bucket;
 558         struct irq_handler_data *data;
 559         unsigned int virt_irq;
 560         int ino;
 561
 562         BUG_ON(tlb_type == hypervisor);
 563
 564         ino = (upa_readq(imap) & (IMAP_IGN | IMAP_INO)) + inofixup;
 565         bucket = &ivector_table[ino];
 566         virt_irq = bucket_get_virt_irq(__pa(bucket));
 567         if (!virt_irq) {
 568                 virt_irq = virt_irq_alloc(0, ino);
 569                 bucket_set_virt_irq(__pa(bucket), virt_irq);
 570                 set_irq_chip_and_handler_name(virt_irq,
 571                                               &sun4u_irq,
 572                                               handle_fasteoi_irq,
 573                                               "IVEC");
 574         }
 575
 576         data = get_irq_chip_data(virt_irq);
 577         if (unlikely(data))
 578                 goto out;
 579
 580         data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC);
 581         if (unlikely(!data)) {
 582                 prom_printf("IRQ: kzalloc(irq_handler_data) failed.\n");
 583                 prom_halt();
 584         }
 585         set_irq_chip_data(virt_irq, data);
 586
 587         data->imap  = imap;
 588         data->iclr  = iclr;
 589
 590 out:
 591         return virt_irq;
 592 }
 593
 594 static unsigned int sun4v_build_common(unsigned long sysino,
 595                                        struct irq_chip *chip)
 596 {
 597         struct ino_bucket *bucket;
 598         struct irq_handler_data *data;
 599         unsigned int virt_irq;
 600
 601         BUG_ON(tlb_type != hypervisor);
 602
 603         bucket = &ivector_table[sysino];
 604         virt_irq = bucket_get_virt_irq(__pa(bucket));
 605         if (!virt_irq) {
 606                 virt_irq = virt_irq_alloc(0, sysino);
 607                 bucket_set_virt_irq(__pa(bucket), virt_irq);
 608                 set_irq_chip_and_handler_name(virt_irq, chip,
 609                                               handle_fasteoi_irq,
 610                                               "IVEC");
 611         }
 612
 613         data = get_irq_chip_data(virt_irq);
 614         if (unlikely(data))
 615                 goto out;
 616
 617         data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC);
 618         if (unlikely(!data)) {
 619                 prom_printf("IRQ: kzalloc(irq_handler_data) failed.\n");
 620                 prom_halt();
 621         }
 622         set_irq_chip_data(virt_irq, data);
 623
 624         /* Catch accidental accesses to these things.  IMAP/ICLR handling
 625          * is done by hypervisor calls on sun4v platforms, not by direct
 626          * register accesses.
 627          */
 628         data->imap = ~0UL;
 629         data->iclr = ~0UL;
 630
 631 out:
 632         return virt_irq;
 633 }
 634
 635 unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino)
 636 {
 637         unsigned long sysino = sun4v_devino_to_sysino(devhandle, devino);
 638
 639         return sun4v_build_common(sysino, &sun4v_irq);
 640 }
 641
 642 unsigned int sun4v_build_virq(u32 devhandle, unsigned int devino)
 643 {
 644         struct irq_handler_data *data;
 645         unsigned long hv_err, cookie;
 646         struct ino_bucket *bucket;
 647         struct irq_desc *desc;
 648         unsigned int virt_irq;
 649
 650         bucket = kzalloc(sizeof(struct ino_bucket), GFP_ATOMIC);
 651         if (unlikely(!bucket))
 652                 return 0;
 653
 654         /* The only reference we store to the IRQ bucket is
 655          * by physical address which kmemleak can't see, tell
 656          * it that this object explicitly is not a leak and
 657          * should be scanned.
 658          */
 659         kmemleak_not_leak(bucket);
 660
 661         __flush_dcache_range((unsigned long) bucket,
 662                              ((unsigned long) bucket +
 663                               sizeof(struct ino_bucket)));
 664
 665         virt_irq = virt_irq_alloc(devhandle, devino);
 666         bucket_set_virt_irq(__pa(bucket), virt_irq);
 667
 668         set_irq_chip_and_handler_name(virt_irq, &sun4v_virq,
 669                                       handle_fasteoi_irq,
 670                                       "IVEC");
 671
 672         data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC);
 673         if (unlikely(!data))
 674                 return 0;
 675
 676         /* In order to make the LDC channel startup sequence easier,
 677          * especially wrt. locking, we do not let request_irq() enable
 678          * the interrupt.
 679          */
 680         desc = irq_desc + virt_irq;
 681         desc->status |= IRQ_NOAUTOEN;
 682
 683         set_irq_chip_data(virt_irq, data);
 684
 685         /* Catch accidental accesses to these things.  IMAP/ICLR handling
 686          * is done by hypervisor calls on sun4v platforms, not by direct
 687          * register accesses.
 688          */
 689         data->imap = ~0UL;
 690         data->iclr = ~0UL;
 691
 692         cookie = ~__pa(bucket);
 693         hv_err = sun4v_vintr_set_cookie(devhandle, devino, cookie);
 694         if (hv_err) {
 695                 prom_printf("IRQ: Fatal, cannot set cookie for [%x:%x] "
 696                             "err=%lu\n", devhandle, devino, hv_err);
 697                 prom_halt();
 698         }
 699
 700         return virt_irq;
 701 }
 702
 703 void ack_bad_irq(unsigned int virt_irq)
 704 {
 705         unsigned int ino = virt_irq_table[virt_irq].dev_ino;
 706
 707         if (!ino)
 708                 ino = 0xdeadbeef;
 709
 710         printk(KERN_CRIT "Unexpected IRQ from ino[%x] virt_irq[%u]\n",
 711                ino, virt_irq);
 712 }
 713
 714 void *hardirq_stack[NR_CPUS];
 715 void *softirq_stack[NR_CPUS];
 716
 717 void __irq_entry handler_irq(int irq, struct pt_regs *regs)
 718 {
 719         unsigned long pstate, bucket_pa;
 720         struct pt_regs *old_regs;
 721         void *orig_sp;
 722
 723         clear_softint(1 << irq);
 724
 725         old_regs = set_irq_regs(regs);
 726         irq_enter();
 727
 728         /* Grab an atomic snapshot of the pending IVECs.  */
 729         __asm__ __volatile__("rdpr      %%pstate, %0\n\t"
 730                              "wrpr      %0, %3, %%pstate\n\t"
 731                              "ldx       [%2], %1\n\t"
 732                              "stx       %%g0, [%2]\n\t"
 733                              "wrpr      %0, 0x0, %%pstate\n\t"
 734                              : "=&r" (pstate), "=&r" (bucket_pa)
 735                              : "r" (irq_work_pa(smp_processor_id())),
 736                                "i" (PSTATE_IE)
 737                              : "memory");
 738
 739         orig_sp = set_hardirq_stack();
 740
 741         while (bucket_pa) {
 742                 struct irq_desc *desc;
 743                 unsigned long next_pa;
 744                 unsigned int virt_irq;
 745
 746                 next_pa = bucket_get_chain_pa(bucket_pa);
 747                 virt_irq = bucket_get_virt_irq(bucket_pa);
 748                 bucket_clear_chain_pa(bucket_pa);
 749
 750                 desc = irq_desc + virt_irq;
 751
 752                 if (!(desc->status & IRQ_DISABLED))
 753                         desc->handle_irq(virt_irq, desc);
 754
 755                 bucket_pa = next_pa;
 756         }
 757
 758         restore_hardirq_stack(orig_sp);
 759
 760         irq_exit();
 761         set_irq_regs(old_regs);
 762 }
 763
 764 void do_softirq(void)
 765 {
 766         unsigned long flags;
 767
 768         if (in_interrupt())
 769                 return;
 770
 771         local_irq_save(flags);
 772
 773         if (local_softirq_pending()) {
 774                 void *orig_sp, *sp = softirq_stack[smp_processor_id()];
 775
 776                 sp += THREAD_SIZE - 192 - STACK_BIAS;
 777
 778                 __asm__ __volatile__("mov %%sp, %0\n\t"
 779                                      "mov %1, %%sp"
 780                                      : "=&r" (orig_sp)
 781                                      : "r" (sp));
 782                 __do_softirq();
 783                 __asm__ __volatile__("mov %0, %%sp"
 784                                      : : "r" (orig_sp));
 785         }
 786
 787         local_irq_restore(flags);
 788 }
 789
 790 #ifdef CONFIG_HOTPLUG_CPU
 791 void fixup_irqs(void)
 792 {
 793         unsigned int irq;
 794
 795         for (irq = 0; irq < NR_IRQS; irq++) {
 796                 unsigned long flags;
 797
 798                 raw_spin_lock_irqsave(&irq_desc[irq].lock, flags);
 799                 if (irq_desc[irq].action &&
 800                     !(irq_desc[irq].status & IRQ_PER_CPU)) {
 801                         if (irq_desc[irq].chip->set_affinity)
 802                                 irq_desc[irq].chip->set_affinity(irq,
 803                                         irq_desc[irq].affinity);
 804                 }
 805                 raw_spin_unlock_irqrestore(&irq_desc[irq].lock, flags);
 806         }
 807
 808         tick_ops->disable_irq();
 809 }
 810 #endif
 811
 812 struct sun5_timer {
 813         u64     count0;
 814         u64     limit0;
 815         u64     count1;
 816         u64     limit1;
 817 };
 818
 819 static struct sun5_timer *prom_timers;
 820 static u64 prom_limit0, prom_limit1;
 821
 822 static void map_prom_timers(void)
 823 {
 824         struct device_node *dp;
 825         const unsigned int *addr;
 826
 827         /* PROM timer node hangs out in the top level of device siblings... */
 828         dp = of_find_node_by_path("/");
 829         dp = dp->child;
 830         while (dp) {
 831                 if (!strcmp(dp->name, "counter-timer"))
 832                         break;
 833                 dp = dp->sibling;
 834         }
 835
 836         /* Assume if node is not present, PROM uses different tick mechanism
 837          * which we should not care about.
 838          */
 839         if (!dp) {
 840                 prom_timers = (struct sun5_timer *) 0;
 841                 return;
 842         }
 843
 844         /* If PROM is really using this, it must be mapped by him. */
 845         addr = of_get_property(dp, "address", NULL);
 846         if (!addr) {
 847                 prom_printf("PROM does not have timer mapped, trying to continue.\n");
 848                 prom_timers = (struct sun5_timer *) 0;
 849                 return;
 850         }
 851         prom_timers = (struct sun5_timer *) ((unsigned long)addr[0]);
 852 }
 853
 854 static void kill_prom_timer(void)
 855 {
 856         if (!prom_timers)
 857                 return;
 858
 859         /* Save them away for later. */
 860         prom_limit0 = prom_timers->limit0;
 861         prom_limit1 = prom_timers->limit1;
 862
 863         /* Just as in sun4c/sun4m PROM uses timer which ticks at IRQ 14.
 864          * We turn both off here just to be paranoid.
 865          */
 866         prom_timers->limit0 = 0;
 867         prom_timers->limit1 = 0;
 868
 869         /* Wheee, eat the interrupt packet too... */
 870         __asm__ __volatile__(
 871 "       mov     0x40, %%g2\n"
 872 "       ldxa    [%%g0] %0, %%g1\n"
 873 "       ldxa    [%%g2] %1, %%g1\n"
 874 "       stxa    %%g0, [%%g0] %0\n"
 875 "       membar  #Sync\n"
 876         : /* no outputs */
 877         : "i" (ASI_INTR_RECEIVE), "i" (ASI_INTR_R)
 878         : "g1", "g2");
 879 }
 880
 881 void notrace init_irqwork_curcpu(void)
 882 {
 883         int cpu = hard_smp_processor_id();
 884
 885         trap_block[cpu].irq_worklist_pa = 0UL;
 886 }
 887
 888 /* Please be very careful with register_one_mondo() and
 889  * sun4v_register_mondo_queues().
 890  *
 891  * On SMP this gets invoked from the CPU trampoline before
 892  * the cpu has fully taken over the trap table from OBP,
 893  * and it's kernel stack + %g6 thread register state is
 894  * not fully cooked yet.
 895  *
 896  * Therefore you cannot make any OBP calls, not even prom_printf,
 897  * from these two routines.
 898  */
 899 static void __cpuinit notrace register_one_mondo(unsigned long paddr, unsigned long type, unsigned long qmask)
 900 {
 901         unsigned long num_entries = (qmask + 1) / 64;
 902         unsigned long status;
 903
 904         status = sun4v_cpu_qconf(type, paddr, num_entries);
 905         if (status != HV_EOK) {
 906                 prom_printf("SUN4V: sun4v_cpu_qconf(%lu:%lx:%lu) failed, "
 907                             "err %lu\n", type, paddr, num_entries, status);
 908                 prom_halt();
 909         }
 910 }
 911
 912 void __cpuinit notrace sun4v_register_mondo_queues(int this_cpu)
 913 {
 914         struct trap_per_cpu *tb = &trap_block[this_cpu];
 915
 916         register_one_mondo(tb->cpu_mondo_pa, HV_CPU_QUEUE_CPU_MONDO,
 917                            tb->cpu_mondo_qmask);
 918         register_one_mondo(tb->dev_mondo_pa, HV_CPU_QUEUE_DEVICE_MONDO,
 919                            tb->dev_mondo_qmask);
 920         register_one_mondo(tb->resum_mondo_pa, HV_CPU_QUEUE_RES_ERROR,
 921                            tb->resum_qmask);
 922         register_one_mondo(tb->nonresum_mondo_pa, HV_CPU_QUEUE_NONRES_ERROR,
 923                            tb->nonresum_qmask);
 924 }
 925
 926 /* Each queue region must be a power of 2 multiple of 64 bytes in
 927  * size.  The base real address must be aligned to the size of the
 928  * region.  Thus, an 8KB queue must be 8KB aligned, for example.
 929  */
 930 static void __init alloc_one_queue(unsigned long *pa_ptr, unsigned long qmask)
 931 {
 932         unsigned long size = PAGE_ALIGN(qmask + 1);
 933         unsigned long order = get_order(size);
 934         unsigned long p;
 935
 936         p = __get_free_pages(GFP_KERNEL, order);
 937         if (!p) {
 938                 prom_printf("SUN4V: Error, cannot allocate queue.\n");
 939                 prom_halt();
 940         }
 941
 942         *pa_ptr = __pa(p);
 943 }
 944
 945 static void __init init_cpu_send_mondo_info(struct trap_per_cpu *tb)
 946 {
 947 #ifdef CONFIG_SMP
 948         unsigned long page;
 949
 950         BUILD_BUG_ON((NR_CPUS * sizeof(u16)) > (PAGE_SIZE - 64));
 951
 952         page = get_zeroed_page(GFP_KERNEL);
 953         if (!page) {
 954                 prom_printf("SUN4V: Error, cannot allocate cpu mondo page.\n");
 955                 prom_halt();
 956         }
 957
 958         tb->cpu_mondo_block_pa = __pa(page);
 959         tb->cpu_list_pa = __pa(page + 64);
 960 #endif
 961 }
 962
 963 /* Allocate mondo and error queues for all possible cpus.  */
 964 static void __init sun4v_init_mondo_queues(void)
 965 {
 966         int cpu;
 967
 968         for_each_possible_cpu(cpu) {
 969                 struct trap_per_cpu *tb = &trap_block[cpu];
 970
 971                 alloc_one_queue(&tb->cpu_mondo_pa, tb->cpu_mondo_qmask);
 972                 alloc_one_queue(&tb->dev_mondo_pa, tb->dev_mondo_qmask);
 973                 alloc_one_queue(&tb->resum_mondo_pa, tb->resum_qmask);
 974                 alloc_one_queue(&tb->resum_kernel_buf_pa, tb->resum_qmask);
 975                 alloc_one_queue(&tb->nonresum_mondo_pa, tb->nonresum_qmask);
 976                 alloc_one_queue(&tb->nonresum_kernel_buf_pa,
 977                                 tb->nonresum_qmask);
 978         }
 979 }
 980
 981 static void __init init_send_mondo_info(void)
 982 {
 983         int cpu;
 984
 985         for_each_possible_cpu(cpu) {
 986                 struct trap_per_cpu *tb = &trap_block[cpu];
 987
 988                 init_cpu_send_mondo_info(tb);
 989         }
 990 }
 991
 992 static struct irqaction timer_irq_action = {
 993         .name = "timer",
 994 };
 995
 996 /* Only invoked on boot processor. */
 997 void __init init_IRQ(void)
 998 {
 999         unsigned long size;
1000
1001         map_prom_timers();
1002         kill_prom_timer();
1003
1004         size = sizeof(struct ino_bucket) * NUM_IVECS;
1005         ivector_table = kzalloc(size, GFP_KERNEL);
1006         if (!ivector_table) {
1007                 prom_printf("Fatal error, cannot allocate ivector_table\n");
1008                 prom_halt();
1009         }
1010         __flush_dcache_range((unsigned long) ivector_table,
1011                              ((unsigned long) ivector_table) + size);
1012
1013         ivector_table_pa = __pa(ivector_table);
1014
1015         if (tlb_type == hypervisor)
1016                 sun4v_init_mondo_queues();
1017
1018         init_send_mondo_info();
1019
1020         if (tlb_type == hypervisor) {
1021                 /* Load up the boot cpu's entries.  */
1022                 sun4v_register_mondo_queues(hard_smp_processor_id());
1023         }
1024
1025         /* We need to clear any IRQ's pending in the soft interrupt
1026          * registers, a spurious one could be left around from the
1027          * PROM timer which we just disabled.
1028          */
1029         clear_softint(get_softint());
1030
1031         /* Now that ivector table is initialized, it is safe
1032          * to receive IRQ vector traps.  We will normally take
1033          * one or two right now, in case some device PROM used
1034          * to boot us wants to speak to us.  We just ignore them.
1035          */
1036         __asm__ __volatile__("rdpr      %%pstate, %%g1\n\t"
1037                              "or        %%g1, %0, %%g1\n\t"
1038                              "wrpr      %%g1, 0x0, %%pstate"
1039                              : /* No outputs */
1040                              : "i" (PSTATE_IE)
1041                              : "g1");
1042
1043         irq_desc[0].action = &timer_irq_action;
1044 }