arch/sparc64/kernel/pci_sun4v.c

   1 /* pci_sun4v.c: SUN4V specific PCI controller support.
   2  *
   3  * Copyright (C) 2006, 2007, 2008 David S. Miller (davem@davemloft.net)
   4  */
   5
   6 #include <linux/kernel.h>
   7 #include <linux/types.h>
   8 #include <linux/pci.h>
   9 #include <linux/init.h>
  10 #include <linux/slab.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/percpu.h>
  13 #include <linux/irq.h>
  14 #include <linux/msi.h>
  15 #include <linux/log2.h>
  16
  17 #include <asm/iommu.h>
  18 #include <asm/irq.h>
  19 #include <asm/upa.h>
  20 #include <asm/pstate.h>
  21 #include <asm/oplib.h>
  22 #include <asm/hypervisor.h>
  23 #include <asm/prom.h>
  24
  25 #include "pci_impl.h"
  26 #include "iommu_common.h"
  27
  28 #include "pci_sun4v.h"
  29
  30 static unsigned long vpci_major = 1;
  31 static unsigned long vpci_minor = 1;
  32
  33 #define PGLIST_NENTS    (PAGE_SIZE / sizeof(u64))
  34
  35 struct iommu_batch {
  36         struct device   *dev;           /* Device mapping is for.       */
  37         unsigned long   prot;           /* IOMMU page protections       */
  38         unsigned long   entry;          /* Index into IOTSB.            */
  39         u64             *pglist;        /* List of physical pages       */
  40         unsigned long   npages;         /* Number of pages in list.     */
  41 };
  42
  43 static DEFINE_PER_CPU(struct iommu_batch, iommu_batch);
  44
  45 /* Interrupts must be disabled.  */
  46 static inline void iommu_batch_start(struct device *dev, unsigned long prot, unsigned long entry)
  47 {
  48         struct iommu_batch *p = &__get_cpu_var(iommu_batch);
  49
  50         p->dev          = dev;
  51         p->prot         = prot;
  52         p->entry        = entry;
  53         p->npages       = 0;
  54 }
  55
  56 /* Interrupts must be disabled.  */
  57 static long iommu_batch_flush(struct iommu_batch *p)
  58 {
  59         struct pci_pbm_info *pbm = p->dev->archdata.host_controller;
  60         unsigned long devhandle = pbm->devhandle;
  61         unsigned long prot = p->prot;
  62         unsigned long entry = p->entry;
  63         u64 *pglist = p->pglist;
  64         unsigned long npages = p->npages;
  65
  66         while (npages != 0) {
  67                 long num;
  68
  69                 num = pci_sun4v_iommu_map(devhandle, HV_PCI_TSBID(0, entry),
  70                                           npages, prot, __pa(pglist));
  71                 if (unlikely(num < 0)) {
  72                         if (printk_ratelimit())
  73                                 printk("iommu_batch_flush: IOMMU map of "
  74                                        "[%08lx:%08lx:%lx:%lx:%lx] failed with "
  75                                        "status %ld\n",
  76                                        devhandle, HV_PCI_TSBID(0, entry),
  77                                        npages, prot, __pa(pglist), num);
  78                         return -1;
  79                 }
  80
  81                 entry += num;
  82                 npages -= num;
  83                 pglist += num;
  84         }
  85
  86         p->entry = entry;
  87         p->npages = 0;
  88
  89         return 0;
  90 }
  91
  92 static inline void iommu_batch_new_entry(unsigned long entry)
  93 {
  94         struct iommu_batch *p = &__get_cpu_var(iommu_batch);
  95
  96         if (p->entry + p->npages == entry)
  97                 return;
  98         if (p->entry != ~0UL)
  99                 iommu_batch_flush(p);
 100         p->entry = entry;
 101 }
 102
 103 /* Interrupts must be disabled.  */
 104 static inline long iommu_batch_add(u64 phys_page)
 105 {
 106         struct iommu_batch *p = &__get_cpu_var(iommu_batch);
 107
 108         BUG_ON(p->npages >= PGLIST_NENTS);
 109
 110         p->pglist[p->npages++] = phys_page;
 111         if (p->npages == PGLIST_NENTS)
 112                 return iommu_batch_flush(p);
 113
 114         return 0;
 115 }
 116
 117 /* Interrupts must be disabled.  */
 118 static inline long iommu_batch_end(void)
 119 {
 120         struct iommu_batch *p = &__get_cpu_var(iommu_batch);
 121
 122         BUG_ON(p->npages >= PGLIST_NENTS);
 123
 124         return iommu_batch_flush(p);
 125 }
 126
 127 static void *dma_4v_alloc_coherent(struct device *dev, size_t size,
 128                                    dma_addr_t *dma_addrp, gfp_t gfp)
 129 {
 130         struct iommu *iommu;
 131         unsigned long flags, order, first_page, npages, n;
 132         void *ret;
 133         long entry;
 134
 135         size = IO_PAGE_ALIGN(size);
 136         order = get_order(size);
 137         if (unlikely(order >= MAX_ORDER))
 138                 return NULL;
 139
 140         npages = size >> IO_PAGE_SHIFT;
 141
 142         first_page = __get_free_pages(gfp, order);
 143         if (unlikely(first_page == 0UL))
 144                 return NULL;
 145
 146         memset((char *)first_page, 0, PAGE_SIZE << order);
 147
 148         iommu = dev->archdata.iommu;
 149
 150         spin_lock_irqsave(&iommu->lock, flags);
 151         entry = iommu_range_alloc(dev, iommu, npages, NULL);
 152         spin_unlock_irqrestore(&iommu->lock, flags);
 153
 154         if (unlikely(entry == DMA_ERROR_CODE))
 155                 goto range_alloc_fail;
 156
 157         *dma_addrp = (iommu->page_table_map_base +
 158                       (entry << IO_PAGE_SHIFT));
 159         ret = (void *) first_page;
 160         first_page = __pa(first_page);
 161
 162         local_irq_save(flags);
 163
 164         iommu_batch_start(dev,
 165                           (HV_PCI_MAP_ATTR_READ |
 166                            HV_PCI_MAP_ATTR_WRITE),
 167                           entry);
 168
 169         for (n = 0; n < npages; n++) {
 170                 long err = iommu_batch_add(first_page + (n * PAGE_SIZE));
 171                 if (unlikely(err < 0L))
 172                         goto iommu_map_fail;
 173         }
 174
 175         if (unlikely(iommu_batch_end() < 0L))
 176                 goto iommu_map_fail;
 177
 178         local_irq_restore(flags);
 179
 180         return ret;
 181
 182 iommu_map_fail:
 183         /* Interrupts are disabled.  */
 184         spin_lock(&iommu->lock);
 185         iommu_range_free(iommu, *dma_addrp, npages);
 186         spin_unlock_irqrestore(&iommu->lock, flags);
 187
 188 range_alloc_fail:
 189         free_pages(first_page, order);
 190         return NULL;
 191 }
 192
 193 static void dma_4v_free_coherent(struct device *dev, size_t size, void *cpu,
 194                                  dma_addr_t dvma)
 195 {
 196         struct pci_pbm_info *pbm;
 197         struct iommu *iommu;
 198         unsigned long flags, order, npages, entry;
 199         u32 devhandle;
 200
 201         npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT;
 202         iommu = dev->archdata.iommu;
 203         pbm = dev->archdata.host_controller;
 204         devhandle = pbm->devhandle;
 205         entry = ((dvma - iommu->page_table_map_base) >> IO_PAGE_SHIFT);
 206
 207         spin_lock_irqsave(&iommu->lock, flags);
 208
 209         iommu_range_free(iommu, dvma, npages);
 210
 211         do {
 212                 unsigned long num;
 213
 214                 num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry),
 215                                             npages);
 216                 entry += num;
 217                 npages -= num;
 218         } while (npages != 0);
 219
 220         spin_unlock_irqrestore(&iommu->lock, flags);
 221
 222         order = get_order(size);
 223         if (order < 10)
 224                 free_pages((unsigned long)cpu, order);
 225 }
 226
 227 static dma_addr_t dma_4v_map_single(struct device *dev, void *ptr, size_t sz,
 228                                     enum dma_data_direction direction)
 229 {
 230         struct iommu *iommu;
 231         unsigned long flags, npages, oaddr;
 232         unsigned long i, base_paddr;
 233         u32 bus_addr, ret;
 234         unsigned long prot;
 235         long entry;
 236
 237         iommu = dev->archdata.iommu;
 238
 239         if (unlikely(direction == DMA_NONE))
 240                 goto bad;
 241
 242         oaddr = (unsigned long)ptr;
 243         npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK);
 244         npages >>= IO_PAGE_SHIFT;
 245
 246         spin_lock_irqsave(&iommu->lock, flags);
 247         entry = iommu_range_alloc(dev, iommu, npages, NULL);
 248         spin_unlock_irqrestore(&iommu->lock, flags);
 249
 250         if (unlikely(entry == DMA_ERROR_CODE))
 251                 goto bad;
 252
 253         bus_addr = (iommu->page_table_map_base +
 254                     (entry << IO_PAGE_SHIFT));
 255         ret = bus_addr | (oaddr & ~IO_PAGE_MASK);
 256         base_paddr = __pa(oaddr & IO_PAGE_MASK);
 257         prot = HV_PCI_MAP_ATTR_READ;
 258         if (direction != DMA_TO_DEVICE)
 259                 prot |= HV_PCI_MAP_ATTR_WRITE;
 260
 261         local_irq_save(flags);
 262
 263         iommu_batch_start(dev, prot, entry);
 264
 265         for (i = 0; i < npages; i++, base_paddr += IO_PAGE_SIZE) {
 266                 long err = iommu_batch_add(base_paddr);
 267                 if (unlikely(err < 0L))
 268                         goto iommu_map_fail;
 269         }
 270         if (unlikely(iommu_batch_end() < 0L))
 271                 goto iommu_map_fail;
 272
 273         local_irq_restore(flags);
 274
 275         return ret;
 276
 277 bad:
 278         if (printk_ratelimit())
 279                 WARN_ON(1);
 280         return DMA_ERROR_CODE;
 281
 282 iommu_map_fail:
 283         /* Interrupts are disabled.  */
 284         spin_lock(&iommu->lock);
 285         iommu_range_free(iommu, bus_addr, npages);
 286         spin_unlock_irqrestore(&iommu->lock, flags);
 287
 288         return DMA_ERROR_CODE;
 289 }
 290
 291 static void dma_4v_unmap_single(struct device *dev, dma_addr_t bus_addr,
 292                                 size_t sz, enum dma_data_direction direction)
 293 {
 294         struct pci_pbm_info *pbm;
 295         struct iommu *iommu;
 296         unsigned long flags, npages;
 297         long entry;
 298         u32 devhandle;
 299
 300         if (unlikely(direction == DMA_NONE)) {
 301                 if (printk_ratelimit())
 302                         WARN_ON(1);
 303                 return;
 304         }
 305
 306         iommu = dev->archdata.iommu;
 307         pbm = dev->archdata.host_controller;
 308         devhandle = pbm->devhandle;
 309
 310         npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK);
 311         npages >>= IO_PAGE_SHIFT;
 312         bus_addr &= IO_PAGE_MASK;
 313
 314         spin_lock_irqsave(&iommu->lock, flags);
 315
 316         iommu_range_free(iommu, bus_addr, npages);
 317
 318         entry = (bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT;
 319         do {
 320                 unsigned long num;
 321
 322                 num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry),
 323                                             npages);
 324                 entry += num;
 325                 npages -= num;
 326         } while (npages != 0);
 327
 328         spin_unlock_irqrestore(&iommu->lock, flags);
 329 }
 330
 331 static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist,
 332                          int nelems, enum dma_data_direction direction)
 333 {
 334         struct scatterlist *s, *outs, *segstart;
 335         unsigned long flags, handle, prot;
 336         dma_addr_t dma_next = 0, dma_addr;
 337         unsigned int max_seg_size;
 338         int outcount, incount, i;
 339         struct iommu *iommu;
 340         long err;
 341
 342         BUG_ON(direction == DMA_NONE);
 343
 344         iommu = dev->archdata.iommu;
 345         if (nelems == 0 || !iommu)
 346                 return 0;
 347
 348         prot = HV_PCI_MAP_ATTR_READ;
 349         if (direction != DMA_TO_DEVICE)
 350                 prot |= HV_PCI_MAP_ATTR_WRITE;
 351
 352         outs = s = segstart = &sglist[0];
 353         outcount = 1;
 354         incount = nelems;
 355         handle = 0;
 356
 357         /* Init first segment length for backout at failure */
 358         outs->dma_length = 0;
 359
 360         spin_lock_irqsave(&iommu->lock, flags);
 361
 362         iommu_batch_start(dev, prot, ~0UL);
 363
 364         max_seg_size = dma_get_max_seg_size(dev);
 365         for_each_sg(sglist, s, nelems, i) {
 366                 unsigned long paddr, npages, entry, slen;
 367
 368                 slen = s->length;
 369                 /* Sanity check */
 370                 if (slen == 0) {
 371                         dma_next = 0;
 372                         continue;
 373                 }
 374                 /* Allocate iommu entries for that segment */
 375                 paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s);
 376                 npages = iommu_num_pages(paddr, slen);
 377                 entry = iommu_range_alloc(dev, iommu, npages, &handle);
 378
 379                 /* Handle failure */
 380                 if (unlikely(entry == DMA_ERROR_CODE)) {
 381                         if (printk_ratelimit())
 382                                 printk(KERN_INFO "iommu_alloc failed, iommu %p paddr %lx"
 383                                        " npages %lx\n", iommu, paddr, npages);
 384                         goto iommu_map_failed;
 385                 }
 386
 387                 iommu_batch_new_entry(entry);
 388
 389                 /* Convert entry to a dma_addr_t */
 390                 dma_addr = iommu->page_table_map_base +
 391                         (entry << IO_PAGE_SHIFT);
 392                 dma_addr |= (s->offset & ~IO_PAGE_MASK);
 393
 394                 /* Insert into HW table */
 395                 paddr &= IO_PAGE_MASK;
 396                 while (npages--) {
 397                         err = iommu_batch_add(paddr);
 398                         if (unlikely(err < 0L))
 399                                 goto iommu_map_failed;
 400                         paddr += IO_PAGE_SIZE;
 401                 }
 402
 403                 /* If we are in an open segment, try merging */
 404                 if (segstart != s) {
 405                         /* We cannot merge if:
 406                          * - allocated dma_addr isn't contiguous to previous allocation
 407                          */
 408                         if ((dma_addr != dma_next) ||
 409                             (outs->dma_length + s->length > max_seg_size)) {
 410                                 /* Can't merge: create a new segment */
 411                                 segstart = s;
 412                                 outcount++;
 413                                 outs = sg_next(outs);
 414                         } else {
 415                                 outs->dma_length += s->length;
 416                         }
 417                 }
 418
 419                 if (segstart == s) {
 420                         /* This is a new segment, fill entries */
 421                         outs->dma_address = dma_addr;
 422                         outs->dma_length = slen;
 423                 }
 424
 425                 /* Calculate next page pointer for contiguous check */
 426                 dma_next = dma_addr + slen;
 427         }
 428
 429         err = iommu_batch_end();
 430
 431         if (unlikely(err < 0L))
 432                 goto iommu_map_failed;
 433
 434         spin_unlock_irqrestore(&iommu->lock, flags);
 435
 436         if (outcount < incount) {
 437                 outs = sg_next(outs);
 438                 outs->dma_address = DMA_ERROR_CODE;
 439                 outs->dma_length = 0;
 440         }
 441
 442         return outcount;
 443
 444 iommu_map_failed:
 445         for_each_sg(sglist, s, nelems, i) {
 446                 if (s->dma_length != 0) {
 447                         unsigned long vaddr, npages;
 448
 449                         vaddr = s->dma_address & IO_PAGE_MASK;
 450                         npages = iommu_num_pages(s->dma_address, s->dma_length);
 451                         iommu_range_free(iommu, vaddr, npages);
 452                         /* XXX demap? XXX */
 453                         s->dma_address = DMA_ERROR_CODE;
 454                         s->dma_length = 0;
 455                 }
 456                 if (s == outs)
 457                         break;
 458         }
 459         spin_unlock_irqrestore(&iommu->lock, flags);
 460
 461         return 0;
 462 }
 463
 464 static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist,
 465                             int nelems, enum dma_data_direction direction)
 466 {
 467         struct pci_pbm_info *pbm;
 468         struct scatterlist *sg;
 469         struct iommu *iommu;
 470         unsigned long flags;
 471         u32 devhandle;
 472
 473         BUG_ON(direction == DMA_NONE);
 474
 475         iommu = dev->archdata.iommu;
 476         pbm = dev->archdata.host_controller;
 477         devhandle = pbm->devhandle;
 478
 479         spin_lock_irqsave(&iommu->lock, flags);
 480
 481         sg = sglist;
 482         while (nelems--) {
 483                 dma_addr_t dma_handle = sg->dma_address;
 484                 unsigned int len = sg->dma_length;
 485                 unsigned long npages, entry;
 486
 487                 if (!len)
 488                         break;
 489                 npages = iommu_num_pages(dma_handle, len);
 490                 iommu_range_free(iommu, dma_handle, npages);
 491
 492                 entry = ((dma_handle - iommu->page_table_map_base) >> IO_PAGE_SHIFT);
 493                 while (npages) {
 494                         unsigned long num;
 495
 496                         num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry),
 497                                                     npages);
 498                         entry += num;
 499                         npages -= num;
 500                 }
 501
 502                 sg = sg_next(sg);
 503         }
 504
 505         spin_unlock_irqrestore(&iommu->lock, flags);
 506 }
 507
 508 static void dma_4v_sync_single_for_cpu(struct device *dev,
 509                                        dma_addr_t bus_addr, size_t sz,
 510                                        enum dma_data_direction direction)
 511 {
 512         /* Nothing to do... */
 513 }
 514
 515 static void dma_4v_sync_sg_for_cpu(struct device *dev,
 516                                    struct scatterlist *sglist, int nelems,
 517                                    enum dma_data_direction direction)
 518 {
 519         /* Nothing to do... */
 520 }
 521
 522 const struct dma_ops sun4v_dma_ops = {
 523         .alloc_coherent                 = dma_4v_alloc_coherent,
 524         .free_coherent                  = dma_4v_free_coherent,
 525         .map_single                     = dma_4v_map_single,
 526         .unmap_single                   = dma_4v_unmap_single,
 527         .map_sg                         = dma_4v_map_sg,
 528         .unmap_sg                       = dma_4v_unmap_sg,
 529         .sync_single_for_cpu            = dma_4v_sync_single_for_cpu,
 530         .sync_sg_for_cpu                = dma_4v_sync_sg_for_cpu,
 531 };
 532
 533 static void __init pci_sun4v_scan_bus(struct pci_pbm_info *pbm)
 534 {
 535         struct property *prop;
 536         struct device_node *dp;
 537
 538         dp = pbm->prom_node;
 539         prop = of_find_property(dp, "66mhz-capable", NULL);
 540         pbm->is_66mhz_capable = (prop != NULL);
 541         pbm->pci_bus = pci_scan_one_pbm(pbm);
 542
 543         /* XXX register error interrupt handlers XXX */
 544 }
 545
 546 static unsigned long __init probe_existing_entries(struct pci_pbm_info *pbm,
 547                                                    struct iommu *iommu)
 548 {
 549         struct iommu_arena *arena = &iommu->arena;
 550         unsigned long i, cnt = 0;
 551         u32 devhandle;
 552
 553         devhandle = pbm->devhandle;
 554         for (i = 0; i < arena->limit; i++) {
 555                 unsigned long ret, io_attrs, ra;
 556
 557                 ret = pci_sun4v_iommu_getmap(devhandle,
 558                                              HV_PCI_TSBID(0, i),
 559                                              &io_attrs, &ra);
 560                 if (ret == HV_EOK) {
 561                         if (page_in_phys_avail(ra)) {
 562                                 pci_sun4v_iommu_demap(devhandle,
 563                                                       HV_PCI_TSBID(0, i), 1);
 564                         } else {
 565                                 cnt++;
 566                                 __set_bit(i, arena->map);
 567                         }
 568                 }
 569         }
 570
 571         return cnt;
 572 }
 573
 574 static void __init pci_sun4v_iommu_init(struct pci_pbm_info *pbm)
 575 {
 576         struct iommu *iommu = pbm->iommu;
 577         struct property *prop;
 578         unsigned long num_tsb_entries, sz, tsbsize;
 579         u32 vdma[2], dma_mask, dma_offset;
 580
 581         prop = of_find_property(pbm->prom_node, "virtual-dma", NULL);
 582         if (prop) {
 583                 u32 *val = prop->value;
 584
 585                 vdma[0] = val[0];
 586                 vdma[1] = val[1];
 587         } else {
 588                 /* No property, use default values. */
 589                 vdma[0] = 0x80000000;
 590                 vdma[1] = 0x80000000;
 591         }
 592
 593         if ((vdma[0] | vdma[1]) & ~IO_PAGE_MASK) {
 594                 prom_printf("PCI-SUN4V: strange virtual-dma[%08x:%08x].\n",
 595                             vdma[0], vdma[1]);
 596                 prom_halt();
 597         };
 598
 599         dma_mask = (roundup_pow_of_two(vdma[1]) - 1UL);
 600         num_tsb_entries = vdma[1] / IO_PAGE_SIZE;
 601         tsbsize = num_tsb_entries * sizeof(iopte_t);
 602
 603         dma_offset = vdma[0];
 604
 605         /* Setup initial software IOMMU state. */
 606         spin_lock_init(&iommu->lock);
 607         iommu->ctx_lowest_free = 1;
 608         iommu->page_table_map_base = dma_offset;
 609         iommu->dma_addr_mask = dma_mask;
 610
 611         /* Allocate and initialize the free area map.  */
 612         sz = (num_tsb_entries + 7) / 8;
 613         sz = (sz + 7UL) & ~7UL;
 614         iommu->arena.map = kzalloc(sz, GFP_KERNEL);
 615         if (!iommu->arena.map) {
 616                 prom_printf("PCI_IOMMU: Error, kmalloc(arena.map) failed.\n");
 617                 prom_halt();
 618         }
 619         iommu->arena.limit = num_tsb_entries;
 620
 621         sz = probe_existing_entries(pbm, iommu);
 622         if (sz)
 623                 printk("%s: Imported %lu TSB entries from OBP\n",
 624                        pbm->name, sz);
 625 }
 626
 627 #ifdef CONFIG_PCI_MSI
 628 struct pci_sun4v_msiq_entry {
 629         u64             version_type;
 630 #define MSIQ_VERSION_MASK               0xffffffff00000000UL
 631 #define MSIQ_VERSION_SHIFT              32
 632 #define MSIQ_TYPE_MASK                  0x00000000000000ffUL
 633 #define MSIQ_TYPE_SHIFT                 0
 634 #define MSIQ_TYPE_NONE                  0x00
 635 #define MSIQ_TYPE_MSG                   0x01
 636 #define MSIQ_TYPE_MSI32                 0x02
 637 #define MSIQ_TYPE_MSI64                 0x03
 638 #define MSIQ_TYPE_INTX                  0x08
 639 #define MSIQ_TYPE_NONE2                 0xff
 640
 641         u64             intx_sysino;
 642         u64             reserved1;
 643         u64             stick;
 644         u64             req_id;  /* bus/device/func */
 645 #define MSIQ_REQID_BUS_MASK             0xff00UL
 646 #define MSIQ_REQID_BUS_SHIFT            8
 647 #define MSIQ_REQID_DEVICE_MASK          0x00f8UL
 648 #define MSIQ_REQID_DEVICE_SHIFT         3
 649 #define MSIQ_REQID_FUNC_MASK            0x0007UL
 650 #define MSIQ_REQID_FUNC_SHIFT           0
 651
 652         u64             msi_address;
 653
 654         /* The format of this value is message type dependent.
 655          * For MSI bits 15:0 are the data from the MSI packet.
 656          * For MSI-X bits 31:0 are the data from the MSI packet.
 657          * For MSG, the message code and message routing code where:
 658          *      bits 39:32 is the bus/device/fn of the msg target-id
 659          *      bits 18:16 is the message routing code
 660          *      bits 7:0 is the message code
 661          * For INTx the low order 2-bits are:
 662          *      00 - INTA
 663          *      01 - INTB
 664          *      10 - INTC
 665          *      11 - INTD
 666          */
 667         u64             msi_data;
 668
 669         u64             reserved2;
 670 };
 671
 672 static int pci_sun4v_get_head(struct pci_pbm_info *pbm, unsigned long msiqid,
 673                               unsigned long *head)
 674 {
 675         unsigned long err, limit;
 676
 677         err = pci_sun4v_msiq_gethead(pbm->devhandle, msiqid, head);
 678         if (unlikely(err))
 679                 return -ENXIO;
 680
 681         limit = pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry);
 682         if (unlikely(*head >= limit))
 683                 return -EFBIG;
 684
 685         return 0;
 686 }
 687
 688 static int pci_sun4v_dequeue_msi(struct pci_pbm_info *pbm,
 689                                  unsigned long msiqid, unsigned long *head,
 690                                  unsigned long *msi)
 691 {
 692         struct pci_sun4v_msiq_entry *ep;
 693         unsigned long err, type;
 694
 695         /* Note: void pointer arithmetic, 'head' is a byte offset  */
 696         ep = (pbm->msi_queues + ((msiqid - pbm->msiq_first) *
 697                                  (pbm->msiq_ent_count *
 698                                   sizeof(struct pci_sun4v_msiq_entry))) +
 699               *head);
 700
 701         if ((ep->version_type & MSIQ_TYPE_MASK) == 0)
 702                 return 0;
 703
 704         type = (ep->version_type & MSIQ_TYPE_MASK) >> MSIQ_TYPE_SHIFT;
 705         if (unlikely(type != MSIQ_TYPE_MSI32 &&
 706                      type != MSIQ_TYPE_MSI64))
 707                 return -EINVAL;
 708
 709         *msi = ep->msi_data;
 710
 711         err = pci_sun4v_msi_setstate(pbm->devhandle,
 712                                      ep->msi_data /* msi_num */,
 713                                      HV_MSISTATE_IDLE);
 714         if (unlikely(err))
 715                 return -ENXIO;
 716
 717         /* Clear the entry.  */
 718         ep->version_type &= ~MSIQ_TYPE_MASK;
 719
 720         (*head) += sizeof(struct pci_sun4v_msiq_entry);
 721         if (*head >=
 722             (pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry)))
 723                 *head = 0;
 724
 725         return 1;
 726 }
 727
 728 static int pci_sun4v_set_head(struct pci_pbm_info *pbm, unsigned long msiqid,
 729                               unsigned long head)
 730 {
 731         unsigned long err;
 732
 733         err = pci_sun4v_msiq_sethead(pbm->devhandle, msiqid, head);
 734         if (unlikely(err))
 735                 return -EINVAL;
 736
 737         return 0;
 738 }
 739
 740 static int pci_sun4v_msi_setup(struct pci_pbm_info *pbm, unsigned long msiqid,
 741                                unsigned long msi, int is_msi64)
 742 {
 743         if (pci_sun4v_msi_setmsiq(pbm->devhandle, msi, msiqid,
 744                                   (is_msi64 ?
 745                                    HV_MSITYPE_MSI64 : HV_MSITYPE_MSI32)))
 746                 return -ENXIO;
 747         if (pci_sun4v_msi_setstate(pbm->devhandle, msi, HV_MSISTATE_IDLE))
 748                 return -ENXIO;
 749         if (pci_sun4v_msi_setvalid(pbm->devhandle, msi, HV_MSIVALID_VALID))
 750                 return -ENXIO;
 751         return 0;
 752 }
 753
 754 static int pci_sun4v_msi_teardown(struct pci_pbm_info *pbm, unsigned long msi)
 755 {
 756         unsigned long err, msiqid;
 757
 758         err = pci_sun4v_msi_getmsiq(pbm->devhandle, msi, &msiqid);
 759         if (err)
 760                 return -ENXIO;
 761
 762         pci_sun4v_msi_setvalid(pbm->devhandle, msi, HV_MSIVALID_INVALID);
 763
 764         return 0;
 765 }
 766
 767 static int pci_sun4v_msiq_alloc(struct pci_pbm_info *pbm)
 768 {
 769         unsigned long q_size, alloc_size, pages, order;
 770         int i;
 771
 772         q_size = pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry);
 773         alloc_size = (pbm->msiq_num * q_size);
 774         order = get_order(alloc_size);
 775         pages = __get_free_pages(GFP_KERNEL | __GFP_COMP, order);
 776         if (pages == 0UL) {
 777                 printk(KERN_ERR "MSI: Cannot allocate MSI queues (o=%lu).\n",
 778                        order);
 779                 return -ENOMEM;
 780         }
 781         memset((char *)pages, 0, PAGE_SIZE << order);
 782         pbm->msi_queues = (void *) pages;
 783
 784         for (i = 0; i < pbm->msiq_num; i++) {
 785                 unsigned long err, base = __pa(pages + (i * q_size));
 786                 unsigned long ret1, ret2;
 787
 788                 err = pci_sun4v_msiq_conf(pbm->devhandle,
 789                                           pbm->msiq_first + i,
 790                                           base, pbm->msiq_ent_count);
 791                 if (err) {
 792                         printk(KERN_ERR "MSI: msiq register fails (err=%lu)\n",
 793                                err);
 794                         goto h_error;
 795                 }
 796
 797                 err = pci_sun4v_msiq_info(pbm->devhandle,
 798                                           pbm->msiq_first + i,
 799                                           &ret1, &ret2);
 800                 if (err) {
 801                         printk(KERN_ERR "MSI: Cannot read msiq (err=%lu)\n",
 802                                err);
 803                         goto h_error;
 804                 }
 805                 if (ret1 != base || ret2 != pbm->msiq_ent_count) {
 806                         printk(KERN_ERR "MSI: Bogus qconf "
 807                                "expected[%lx:%x] got[%lx:%lx]\n",
 808                                base, pbm->msiq_ent_count,
 809                                ret1, ret2);
 810                         goto h_error;
 811                 }
 812         }
 813
 814         return 0;
 815
 816 h_error:
 817         free_pages(pages, order);
 818         return -EINVAL;
 819 }
 820
 821 static void pci_sun4v_msiq_free(struct pci_pbm_info *pbm)
 822 {
 823         unsigned long q_size, alloc_size, pages, order;
 824         int i;
 825
 826         for (i = 0; i < pbm->msiq_num; i++) {
 827                 unsigned long msiqid = pbm->msiq_first + i;
 828
 829                 (void) pci_sun4v_msiq_conf(pbm->devhandle, msiqid, 0UL, 0);
 830         }
 831
 832         q_size = pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry);
 833         alloc_size = (pbm->msiq_num * q_size);
 834         order = get_order(alloc_size);
 835
 836         pages = (unsigned long) pbm->msi_queues;
 837
 838         free_pages(pages, order);
 839
 840         pbm->msi_queues = NULL;
 841 }
 842
 843 static int pci_sun4v_msiq_build_irq(struct pci_pbm_info *pbm,
 844                                     unsigned long msiqid,
 845                                     unsigned long devino)
 846 {
 847         unsigned int virt_irq = sun4v_build_irq(pbm->devhandle, devino);
 848
 849         if (!virt_irq)
 850                 return -ENOMEM;
 851
 852         if (pci_sun4v_msiq_setstate(pbm->devhandle, msiqid, HV_MSIQSTATE_IDLE))
 853                 return -EINVAL;
 854         if (pci_sun4v_msiq_setvalid(pbm->devhandle, msiqid, HV_MSIQ_VALID))
 855                 return -EINVAL;
 856
 857         return virt_irq;
 858 }
 859
 860 static const struct sparc64_msiq_ops pci_sun4v_msiq_ops = {
 861         .get_head       =       pci_sun4v_get_head,
 862         .dequeue_msi    =       pci_sun4v_dequeue_msi,
 863         .set_head       =       pci_sun4v_set_head,
 864         .msi_setup      =       pci_sun4v_msi_setup,
 865         .msi_teardown   =       pci_sun4v_msi_teardown,
 866         .msiq_alloc     =       pci_sun4v_msiq_alloc,
 867         .msiq_free      =       pci_sun4v_msiq_free,
 868         .msiq_build_irq =       pci_sun4v_msiq_build_irq,
 869 };
 870
 871 static void pci_sun4v_msi_init(struct pci_pbm_info *pbm)
 872 {
 873         sparc64_pbm_msi_init(pbm, &pci_sun4v_msiq_ops);
 874 }
 875 #else /* CONFIG_PCI_MSI */
 876 static void pci_sun4v_msi_init(struct pci_pbm_info *pbm)
 877 {
 878 }
 879 #endif /* !(CONFIG_PCI_MSI) */
 880
 881 static void __init pci_sun4v_pbm_init(struct pci_controller_info *p,
 882                                       struct device_node *dp, u32 devhandle)
 883 {
 884         struct pci_pbm_info *pbm;
 885
 886         if (devhandle & 0x40)
 887                 pbm = &p->pbm_B;
 888         else
 889                 pbm = &p->pbm_A;
 890
 891         pbm->next = pci_pbm_root;
 892         pci_pbm_root = pbm;
 893
 894         pbm->scan_bus = pci_sun4v_scan_bus;
 895         pbm->pci_ops = &sun4v_pci_ops;
 896         pbm->config_space_reg_bits = 12;
 897
 898         pbm->index = pci_num_pbms++;
 899
 900         pbm->parent = p;
 901         pbm->prom_node = dp;
 902
 903         pbm->devhandle = devhandle;
 904
 905         pbm->name = dp->full_name;
 906
 907         printk("%s: SUN4V PCI Bus Module\n", pbm->name);
 908
 909         pci_determine_mem_io_space(pbm);
 910
 911         pci_get_pbm_props(pbm);
 912         pci_sun4v_iommu_init(pbm);
 913         pci_sun4v_msi_init(pbm);
 914 }
 915
 916 void __init sun4v_pci_init(struct device_node *dp, char *model_name)
 917 {
 918         static int hvapi_negotiated = 0;
 919         struct pci_controller_info *p;
 920         struct pci_pbm_info *pbm;
 921         struct iommu *iommu;
 922         struct property *prop;
 923         struct linux_prom64_registers *regs;
 924         u32 devhandle;
 925         int i;
 926
 927         if (!hvapi_negotiated++) {
 928                 int err = sun4v_hvapi_register(HV_GRP_PCI,
 929                                                vpci_major,
 930                                                &vpci_minor);
 931
 932                 if (err) {
 933                         prom_printf("SUN4V_PCI: Could not register hvapi, "
 934                                     "err=%d\n", err);
 935                         prom_halt();
 936                 }
 937                 printk("SUN4V_PCI: Registered hvapi major[%lu] minor[%lu]\n",
 938                        vpci_major, vpci_minor);
 939
 940                 dma_ops = &sun4v_dma_ops;
 941         }
 942
 943         prop = of_find_property(dp, "reg", NULL);
 944         if (!prop) {
 945                 prom_printf("SUN4V_PCI: Could not find config registers\n");
 946                 prom_halt();
 947         }
 948         regs = prop->value;
 949
 950         devhandle = (regs->phys_addr >> 32UL) & 0x0fffffff;
 951
 952         for (pbm = pci_pbm_root; pbm; pbm = pbm->next) {
 953                 if (pbm->devhandle == (devhandle ^ 0x40)) {
 954                         pci_sun4v_pbm_init(pbm->parent, dp, devhandle);
 955                         return;
 956                 }
 957         }
 958
 959         for_each_possible_cpu(i) {
 960                 unsigned long page = get_zeroed_page(GFP_ATOMIC);
 961
 962                 if (!page)
 963                         goto fatal_memory_error;
 964
 965                 per_cpu(iommu_batch, i).pglist = (u64 *) page;
 966         }
 967
 968         p = kzalloc(sizeof(struct pci_controller_info), GFP_ATOMIC);
 969         if (!p)
 970                 goto fatal_memory_error;
 971
 972         iommu = kzalloc(sizeof(struct iommu), GFP_ATOMIC);
 973         if (!iommu)
 974                 goto fatal_memory_error;
 975
 976         p->pbm_A.iommu = iommu;
 977
 978         iommu = kzalloc(sizeof(struct iommu), GFP_ATOMIC);
 979         if (!iommu)
 980                 goto fatal_memory_error;
 981
 982         p->pbm_B.iommu = iommu;
 983
 984         pci_sun4v_pbm_init(p, dp, devhandle);
 985         return;
 986
 987 fatal_memory_error:
 988         prom_printf("SUN4V_PCI: Fatal memory allocation error.\n");
 989         prom_halt();
 990 }