arch/sparc64/kernel/pci_sun4v.c

   1 /* pci_sun4v.c: SUN4V specific PCI controller support.
   2  *
   3  * Copyright (C) 2006 David S. Miller (davem@davemloft.net)
   4  */
   5
   6 #include <linux/kernel.h>
   7 #include <linux/types.h>
   8 #include <linux/pci.h>
   9 #include <linux/init.h>
  10 #include <linux/slab.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/percpu.h>
  13 #include <linux/irq.h>
  14 #include <linux/msi.h>
  15
  16 #include <asm/pbm.h>
  17 #include <asm/iommu.h>
  18 #include <asm/irq.h>
  19 #include <asm/upa.h>
  20 #include <asm/pstate.h>
  21 #include <asm/oplib.h>
  22 #include <asm/hypervisor.h>
  23 #include <asm/prom.h>
  24
  25 #include "pci_impl.h"
  26 #include "iommu_common.h"
  27
  28 #include "pci_sun4v.h"
  29
  30 #define PGLIST_NENTS    (PAGE_SIZE / sizeof(u64))
  31
  32 struct pci_iommu_batch {
  33         struct pci_dev  *pdev;          /* Device mapping is for.       */
  34         unsigned long   prot;           /* IOMMU page protections       */
  35         unsigned long   entry;          /* Index into IOTSB.            */
  36         u64             *pglist;        /* List of physical pages       */
  37         unsigned long   npages;         /* Number of pages in list.     */
  38 };
  39
  40 static DEFINE_PER_CPU(struct pci_iommu_batch, pci_iommu_batch);
  41
  42 /* Interrupts must be disabled.  */
  43 static inline void pci_iommu_batch_start(struct pci_dev *pdev, unsigned long prot, unsigned long entry)
  44 {
  45         struct pci_iommu_batch *p = &__get_cpu_var(pci_iommu_batch);
  46
  47         p->pdev         = pdev;
  48         p->prot         = prot;
  49         p->entry        = entry;
  50         p->npages       = 0;
  51 }
  52
  53 /* Interrupts must be disabled.  */
  54 static long pci_iommu_batch_flush(struct pci_iommu_batch *p)
  55 {
  56         struct pcidev_cookie *pcp = p->pdev->sysdata;
  57         unsigned long devhandle = pcp->pbm->devhandle;
  58         unsigned long prot = p->prot;
  59         unsigned long entry = p->entry;
  60         u64 *pglist = p->pglist;
  61         unsigned long npages = p->npages;
  62
  63         while (npages != 0) {
  64                 long num;
  65
  66                 num = pci_sun4v_iommu_map(devhandle, HV_PCI_TSBID(0, entry),
  67                                           npages, prot, __pa(pglist));
  68                 if (unlikely(num < 0)) {
  69                         if (printk_ratelimit())
  70                                 printk("pci_iommu_batch_flush: IOMMU map of "
  71                                        "[%08lx:%08lx:%lx:%lx:%lx] failed with "
  72                                        "status %ld\n",
  73                                        devhandle, HV_PCI_TSBID(0, entry),
  74                                        npages, prot, __pa(pglist), num);
  75                         return -1;
  76                 }
  77
  78                 entry += num;
  79                 npages -= num;
  80                 pglist += num;
  81         }
  82
  83         p->entry = entry;
  84         p->npages = 0;
  85
  86         return 0;
  87 }
  88
  89 /* Interrupts must be disabled.  */
  90 static inline long pci_iommu_batch_add(u64 phys_page)
  91 {
  92         struct pci_iommu_batch *p = &__get_cpu_var(pci_iommu_batch);
  93
  94         BUG_ON(p->npages >= PGLIST_NENTS);
  95
  96         p->pglist[p->npages++] = phys_page;
  97         if (p->npages == PGLIST_NENTS)
  98                 return pci_iommu_batch_flush(p);
  99
 100         return 0;
 101 }
 102
 103 /* Interrupts must be disabled.  */
 104 static inline long pci_iommu_batch_end(void)
 105 {
 106         struct pci_iommu_batch *p = &__get_cpu_var(pci_iommu_batch);
 107
 108         BUG_ON(p->npages >= PGLIST_NENTS);
 109
 110         return pci_iommu_batch_flush(p);
 111 }
 112
 113 static long pci_arena_alloc(struct pci_iommu_arena *arena, unsigned long npages)
 114 {
 115         unsigned long n, i, start, end, limit;
 116         int pass;
 117
 118         limit = arena->limit;
 119         start = arena->hint;
 120         pass = 0;
 121
 122 again:
 123         n = find_next_zero_bit(arena->map, limit, start);
 124         end = n + npages;
 125         if (unlikely(end >= limit)) {
 126                 if (likely(pass < 1)) {
 127                         limit = start;
 128                         start = 0;
 129                         pass++;
 130                         goto again;
 131                 } else {
 132                         /* Scanned the whole thing, give up. */
 133                         return -1;
 134                 }
 135         }
 136
 137         for (i = n; i < end; i++) {
 138                 if (test_bit(i, arena->map)) {
 139                         start = i + 1;
 140                         goto again;
 141                 }
 142         }
 143
 144         for (i = n; i < end; i++)
 145                 __set_bit(i, arena->map);
 146
 147         arena->hint = end;
 148
 149         return n;
 150 }
 151
 152 static void pci_arena_free(struct pci_iommu_arena *arena, unsigned long base, unsigned long npages)
 153 {
 154         unsigned long i;
 155
 156         for (i = base; i < (base + npages); i++)
 157                 __clear_bit(i, arena->map);
 158 }
 159
 160 static void *pci_4v_alloc_consistent(struct pci_dev *pdev, size_t size, dma_addr_t *dma_addrp, gfp_t gfp)
 161 {
 162         struct pcidev_cookie *pcp;
 163         struct pci_iommu *iommu;
 164         unsigned long flags, order, first_page, npages, n;
 165         void *ret;
 166         long entry;
 167
 168         size = IO_PAGE_ALIGN(size);
 169         order = get_order(size);
 170         if (unlikely(order >= MAX_ORDER))
 171                 return NULL;
 172
 173         npages = size >> IO_PAGE_SHIFT;
 174
 175         first_page = __get_free_pages(gfp, order);
 176         if (unlikely(first_page == 0UL))
 177                 return NULL;
 178
 179         memset((char *)first_page, 0, PAGE_SIZE << order);
 180
 181         pcp = pdev->sysdata;
 182         iommu = pcp->pbm->iommu;
 183
 184         spin_lock_irqsave(&iommu->lock, flags);
 185         entry = pci_arena_alloc(&iommu->arena, npages);
 186         spin_unlock_irqrestore(&iommu->lock, flags);
 187
 188         if (unlikely(entry < 0L))
 189                 goto arena_alloc_fail;
 190
 191         *dma_addrp = (iommu->page_table_map_base +
 192                       (entry << IO_PAGE_SHIFT));
 193         ret = (void *) first_page;
 194         first_page = __pa(first_page);
 195
 196         local_irq_save(flags);
 197
 198         pci_iommu_batch_start(pdev,
 199                               (HV_PCI_MAP_ATTR_READ |
 200                                HV_PCI_MAP_ATTR_WRITE),
 201                               entry);
 202
 203         for (n = 0; n < npages; n++) {
 204                 long err = pci_iommu_batch_add(first_page + (n * PAGE_SIZE));
 205                 if (unlikely(err < 0L))
 206                         goto iommu_map_fail;
 207         }
 208
 209         if (unlikely(pci_iommu_batch_end() < 0L))
 210                 goto iommu_map_fail;
 211
 212         local_irq_restore(flags);
 213
 214         return ret;
 215
 216 iommu_map_fail:
 217         /* Interrupts are disabled.  */
 218         spin_lock(&iommu->lock);
 219         pci_arena_free(&iommu->arena, entry, npages);
 220         spin_unlock_irqrestore(&iommu->lock, flags);
 221
 222 arena_alloc_fail:
 223         free_pages(first_page, order);
 224         return NULL;
 225 }
 226
 227 static void pci_4v_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_t dvma)
 228 {
 229         struct pcidev_cookie *pcp;
 230         struct pci_iommu *iommu;
 231         unsigned long flags, order, npages, entry;
 232         u32 devhandle;
 233
 234         npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT;
 235         pcp = pdev->sysdata;
 236         iommu = pcp->pbm->iommu;
 237         devhandle = pcp->pbm->devhandle;
 238         entry = ((dvma - iommu->page_table_map_base) >> IO_PAGE_SHIFT);
 239
 240         spin_lock_irqsave(&iommu->lock, flags);
 241
 242         pci_arena_free(&iommu->arena, entry, npages);
 243
 244         do {
 245                 unsigned long num;
 246
 247                 num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry),
 248                                             npages);
 249                 entry += num;
 250                 npages -= num;
 251         } while (npages != 0);
 252
 253         spin_unlock_irqrestore(&iommu->lock, flags);
 254
 255         order = get_order(size);
 256         if (order < 10)
 257                 free_pages((unsigned long)cpu, order);
 258 }
 259
 260 static dma_addr_t pci_4v_map_single(struct pci_dev *pdev, void *ptr, size_t sz, int direction)
 261 {
 262         struct pcidev_cookie *pcp;
 263         struct pci_iommu *iommu;
 264         unsigned long flags, npages, oaddr;
 265         unsigned long i, base_paddr;
 266         u32 bus_addr, ret;
 267         unsigned long prot;
 268         long entry;
 269
 270         pcp = pdev->sysdata;
 271         iommu = pcp->pbm->iommu;
 272
 273         if (unlikely(direction == PCI_DMA_NONE))
 274                 goto bad;
 275
 276         oaddr = (unsigned long)ptr;
 277         npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK);
 278         npages >>= IO_PAGE_SHIFT;
 279
 280         spin_lock_irqsave(&iommu->lock, flags);
 281         entry = pci_arena_alloc(&iommu->arena, npages);
 282         spin_unlock_irqrestore(&iommu->lock, flags);
 283
 284         if (unlikely(entry < 0L))
 285                 goto bad;
 286
 287         bus_addr = (iommu->page_table_map_base +
 288                     (entry << IO_PAGE_SHIFT));
 289         ret = bus_addr | (oaddr & ~IO_PAGE_MASK);
 290         base_paddr = __pa(oaddr & IO_PAGE_MASK);
 291         prot = HV_PCI_MAP_ATTR_READ;
 292         if (direction != PCI_DMA_TODEVICE)
 293                 prot |= HV_PCI_MAP_ATTR_WRITE;
 294
 295         local_irq_save(flags);
 296
 297         pci_iommu_batch_start(pdev, prot, entry);
 298
 299         for (i = 0; i < npages; i++, base_paddr += IO_PAGE_SIZE) {
 300                 long err = pci_iommu_batch_add(base_paddr);
 301                 if (unlikely(err < 0L))
 302                         goto iommu_map_fail;
 303         }
 304         if (unlikely(pci_iommu_batch_end() < 0L))
 305                 goto iommu_map_fail;
 306
 307         local_irq_restore(flags);
 308
 309         return ret;
 310
 311 bad:
 312         if (printk_ratelimit())
 313                 WARN_ON(1);
 314         return PCI_DMA_ERROR_CODE;
 315
 316 iommu_map_fail:
 317         /* Interrupts are disabled.  */
 318         spin_lock(&iommu->lock);
 319         pci_arena_free(&iommu->arena, entry, npages);
 320         spin_unlock_irqrestore(&iommu->lock, flags);
 321
 322         return PCI_DMA_ERROR_CODE;
 323 }
 324
 325 static void pci_4v_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int direction)
 326 {
 327         struct pcidev_cookie *pcp;
 328         struct pci_iommu *iommu;
 329         unsigned long flags, npages;
 330         long entry;
 331         u32 devhandle;
 332
 333         if (unlikely(direction == PCI_DMA_NONE)) {
 334                 if (printk_ratelimit())
 335                         WARN_ON(1);
 336                 return;
 337         }
 338
 339         pcp = pdev->sysdata;
 340         iommu = pcp->pbm->iommu;
 341         devhandle = pcp->pbm->devhandle;
 342
 343         npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK);
 344         npages >>= IO_PAGE_SHIFT;
 345         bus_addr &= IO_PAGE_MASK;
 346
 347         spin_lock_irqsave(&iommu->lock, flags);
 348
 349         entry = (bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT;
 350         pci_arena_free(&iommu->arena, entry, npages);
 351
 352         do {
 353                 unsigned long num;
 354
 355                 num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry),
 356                                             npages);
 357                 entry += num;
 358                 npages -= num;
 359         } while (npages != 0);
 360
 361         spin_unlock_irqrestore(&iommu->lock, flags);
 362 }
 363
 364 #define SG_ENT_PHYS_ADDRESS(SG) \
 365         (__pa(page_address((SG)->page)) + (SG)->offset)
 366
 367 static inline long fill_sg(long entry, struct pci_dev *pdev,
 368                            struct scatterlist *sg,
 369                            int nused, int nelems, unsigned long prot)
 370 {
 371         struct scatterlist *dma_sg = sg;
 372         struct scatterlist *sg_end = sg + nelems;
 373         unsigned long flags;
 374         int i;
 375
 376         local_irq_save(flags);
 377
 378         pci_iommu_batch_start(pdev, prot, entry);
 379
 380         for (i = 0; i < nused; i++) {
 381                 unsigned long pteval = ~0UL;
 382                 u32 dma_npages;
 383
 384                 dma_npages = ((dma_sg->dma_address & (IO_PAGE_SIZE - 1UL)) +
 385                               dma_sg->dma_length +
 386                               ((IO_PAGE_SIZE - 1UL))) >> IO_PAGE_SHIFT;
 387                 do {
 388                         unsigned long offset;
 389                         signed int len;
 390
 391                         /* If we are here, we know we have at least one
 392                          * more page to map.  So walk forward until we
 393                          * hit a page crossing, and begin creating new
 394                          * mappings from that spot.
 395                          */
 396                         for (;;) {
 397                                 unsigned long tmp;
 398
 399                                 tmp = SG_ENT_PHYS_ADDRESS(sg);
 400                                 len = sg->length;
 401                                 if (((tmp ^ pteval) >> IO_PAGE_SHIFT) != 0UL) {
 402                                         pteval = tmp & IO_PAGE_MASK;
 403                                         offset = tmp & (IO_PAGE_SIZE - 1UL);
 404                                         break;
 405                                 }
 406                                 if (((tmp ^ (tmp + len - 1UL)) >> IO_PAGE_SHIFT) != 0UL) {
 407                                         pteval = (tmp + IO_PAGE_SIZE) & IO_PAGE_MASK;
 408                                         offset = 0UL;
 409                                         len -= (IO_PAGE_SIZE - (tmp & (IO_PAGE_SIZE - 1UL)));
 410                                         break;
 411                                 }
 412                                 sg++;
 413                         }
 414
 415                         pteval = (pteval & IOPTE_PAGE);
 416                         while (len > 0) {
 417                                 long err;
 418
 419                                 err = pci_iommu_batch_add(pteval);
 420                                 if (unlikely(err < 0L))
 421                                         goto iommu_map_failed;
 422
 423                                 pteval += IO_PAGE_SIZE;
 424                                 len -= (IO_PAGE_SIZE - offset);
 425                                 offset = 0;
 426                                 dma_npages--;
 427                         }
 428
 429                         pteval = (pteval & IOPTE_PAGE) + len;
 430                         sg++;
 431
 432                         /* Skip over any tail mappings we've fully mapped,
 433                          * adjusting pteval along the way.  Stop when we
 434                          * detect a page crossing event.
 435                          */
 436                         while (sg < sg_end &&
 437                                (pteval << (64 - IO_PAGE_SHIFT)) != 0UL &&
 438                                (pteval == SG_ENT_PHYS_ADDRESS(sg)) &&
 439                                ((pteval ^
 440                                  (SG_ENT_PHYS_ADDRESS(sg) + sg->length - 1UL)) >> IO_PAGE_SHIFT) == 0UL) {
 441                                 pteval += sg->length;
 442                                 sg++;
 443                         }
 444                         if ((pteval << (64 - IO_PAGE_SHIFT)) == 0UL)
 445                                 pteval = ~0UL;
 446                 } while (dma_npages != 0);
 447                 dma_sg++;
 448         }
 449
 450         if (unlikely(pci_iommu_batch_end() < 0L))
 451                 goto iommu_map_failed;
 452
 453         local_irq_restore(flags);
 454         return 0;
 455
 456 iommu_map_failed:
 457         local_irq_restore(flags);
 458         return -1L;
 459 }
 460
 461 static int pci_4v_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int direction)
 462 {
 463         struct pcidev_cookie *pcp;
 464         struct pci_iommu *iommu;
 465         unsigned long flags, npages, prot;
 466         u32 dma_base;
 467         struct scatterlist *sgtmp;
 468         long entry, err;
 469         int used;
 470
 471         /* Fast path single entry scatterlists. */
 472         if (nelems == 1) {
 473                 sglist->dma_address =
 474                         pci_4v_map_single(pdev,
 475                                           (page_address(sglist->page) + sglist->offset),
 476                                           sglist->length, direction);
 477                 if (unlikely(sglist->dma_address == PCI_DMA_ERROR_CODE))
 478                         return 0;
 479                 sglist->dma_length = sglist->length;
 480                 return 1;
 481         }
 482
 483         pcp = pdev->sysdata;
 484         iommu = pcp->pbm->iommu;
 485
 486         if (unlikely(direction == PCI_DMA_NONE))
 487                 goto bad;
 488
 489         /* Step 1: Prepare scatter list. */
 490         npages = prepare_sg(sglist, nelems);
 491
 492         /* Step 2: Allocate a cluster and context, if necessary. */
 493         spin_lock_irqsave(&iommu->lock, flags);
 494         entry = pci_arena_alloc(&iommu->arena, npages);
 495         spin_unlock_irqrestore(&iommu->lock, flags);
 496
 497         if (unlikely(entry < 0L))
 498                 goto bad;
 499
 500         dma_base = iommu->page_table_map_base +
 501                 (entry << IO_PAGE_SHIFT);
 502
 503         /* Step 3: Normalize DMA addresses. */
 504         used = nelems;
 505
 506         sgtmp = sglist;
 507         while (used && sgtmp->dma_length) {
 508                 sgtmp->dma_address += dma_base;
 509                 sgtmp++;
 510                 used--;
 511         }
 512         used = nelems - used;
 513
 514         /* Step 4: Create the mappings. */
 515         prot = HV_PCI_MAP_ATTR_READ;
 516         if (direction != PCI_DMA_TODEVICE)
 517                 prot |= HV_PCI_MAP_ATTR_WRITE;
 518
 519         err = fill_sg(entry, pdev, sglist, used, nelems, prot);
 520         if (unlikely(err < 0L))
 521                 goto iommu_map_failed;
 522
 523         return used;
 524
 525 bad:
 526         if (printk_ratelimit())
 527                 WARN_ON(1);
 528         return 0;
 529
 530 iommu_map_failed:
 531         spin_lock_irqsave(&iommu->lock, flags);
 532         pci_arena_free(&iommu->arena, entry, npages);
 533         spin_unlock_irqrestore(&iommu->lock, flags);
 534
 535         return 0;
 536 }
 537
 538 static void pci_4v_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int direction)
 539 {
 540         struct pcidev_cookie *pcp;
 541         struct pci_iommu *iommu;
 542         unsigned long flags, i, npages;
 543         long entry;
 544         u32 devhandle, bus_addr;
 545
 546         if (unlikely(direction == PCI_DMA_NONE)) {
 547                 if (printk_ratelimit())
 548                         WARN_ON(1);
 549         }
 550
 551         pcp = pdev->sysdata;
 552         iommu = pcp->pbm->iommu;
 553         devhandle = pcp->pbm->devhandle;
 554
 555         bus_addr = sglist->dma_address & IO_PAGE_MASK;
 556
 557         for (i = 1; i < nelems; i++)
 558                 if (sglist[i].dma_length == 0)
 559                         break;
 560         i--;
 561         npages = (IO_PAGE_ALIGN(sglist[i].dma_address + sglist[i].dma_length) -
 562                   bus_addr) >> IO_PAGE_SHIFT;
 563
 564         entry = ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT);
 565
 566         spin_lock_irqsave(&iommu->lock, flags);
 567
 568         pci_arena_free(&iommu->arena, entry, npages);
 569
 570         do {
 571                 unsigned long num;
 572
 573                 num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry),
 574                                             npages);
 575                 entry += num;
 576                 npages -= num;
 577         } while (npages != 0);
 578
 579         spin_unlock_irqrestore(&iommu->lock, flags);
 580 }
 581
 582 static void pci_4v_dma_sync_single_for_cpu(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int direction)
 583 {
 584         /* Nothing to do... */
 585 }
 586
 587 static void pci_4v_dma_sync_sg_for_cpu(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int direction)
 588 {
 589         /* Nothing to do... */
 590 }
 591
 592 struct pci_iommu_ops pci_sun4v_iommu_ops = {
 593         .alloc_consistent               = pci_4v_alloc_consistent,
 594         .free_consistent                = pci_4v_free_consistent,
 595         .map_single                     = pci_4v_map_single,
 596         .unmap_single                   = pci_4v_unmap_single,
 597         .map_sg                         = pci_4v_map_sg,
 598         .unmap_sg                       = pci_4v_unmap_sg,
 599         .dma_sync_single_for_cpu        = pci_4v_dma_sync_single_for_cpu,
 600         .dma_sync_sg_for_cpu            = pci_4v_dma_sync_sg_for_cpu,
 601 };
 602
 603 /* SUN4V PCI configuration space accessors. */
 604
 605 struct pdev_entry {
 606         struct pdev_entry       *next;
 607         u32                     devhandle;
 608         unsigned int            bus;
 609         unsigned int            device;
 610         unsigned int            func;
 611 };
 612
 613 #define PDEV_HTAB_SIZE  16
 614 #define PDEV_HTAB_MASK  (PDEV_HTAB_SIZE - 1)
 615 static struct pdev_entry *pdev_htab[PDEV_HTAB_SIZE];
 616
 617 static inline unsigned int pdev_hashfn(u32 devhandle, unsigned int bus, unsigned int device, unsigned int func)
 618 {
 619         unsigned int val;
 620
 621         val = (devhandle ^ (devhandle >> 4));
 622         val ^= bus;
 623         val ^= device;
 624         val ^= func;
 625
 626         return val & PDEV_HTAB_MASK;
 627 }
 628
 629 static int pdev_htab_add(u32 devhandle, unsigned int bus, unsigned int device, unsigned int func)
 630 {
 631         struct pdev_entry *p = kmalloc(sizeof(*p), GFP_KERNEL);
 632         struct pdev_entry **slot;
 633
 634         if (!p)
 635                 return -ENOMEM;
 636
 637         slot = &pdev_htab[pdev_hashfn(devhandle, bus, device, func)];
 638         p->next = *slot;
 639         *slot = p;
 640
 641         p->devhandle = devhandle;
 642         p->bus = bus;
 643         p->device = device;
 644         p->func = func;
 645
 646         return 0;
 647 }
 648
 649 /* Recursively descend into the OBP device tree, rooted at toplevel_node,
 650  * looking for a PCI device matching bus and devfn.
 651  */
 652 static int obp_find(struct device_node *toplevel_node, unsigned int bus, unsigned int devfn)
 653 {
 654         toplevel_node = toplevel_node->child;
 655
 656         while (toplevel_node != NULL) {
 657                 struct linux_prom_pci_registers *regs;
 658                 struct property *prop;
 659                 int ret;
 660
 661                 ret = obp_find(toplevel_node, bus, devfn);
 662                 if (ret != 0)
 663                         return ret;
 664
 665                 prop = of_find_property(toplevel_node, "reg", NULL);
 666                 if (!prop)
 667                         goto next_sibling;
 668
 669                 regs = prop->value;
 670                 if (((regs->phys_hi >> 16) & 0xff) == bus &&
 671                     ((regs->phys_hi >> 8) & 0xff) == devfn)
 672                         break;
 673
 674         next_sibling:
 675                 toplevel_node = toplevel_node->sibling;
 676         }
 677
 678         return toplevel_node != NULL;
 679 }
 680
 681 static int pdev_htab_populate(struct pci_pbm_info *pbm)
 682 {
 683         u32 devhandle = pbm->devhandle;
 684         unsigned int bus;
 685
 686         for (bus = pbm->pci_first_busno; bus <= pbm->pci_last_busno; bus++) {
 687                 unsigned int devfn;
 688
 689                 for (devfn = 0; devfn < 256; devfn++) {
 690                         unsigned int device = PCI_SLOT(devfn);
 691                         unsigned int func = PCI_FUNC(devfn);
 692
 693                         if (obp_find(pbm->prom_node, bus, devfn)) {
 694                                 int err = pdev_htab_add(devhandle, bus,
 695                                                         device, func);
 696                                 if (err)
 697                                         return err;
 698                         }
 699                 }
 700         }
 701
 702         return 0;
 703 }
 704
 705 static struct pdev_entry *pdev_find(u32 devhandle, unsigned int bus, unsigned int device, unsigned int func)
 706 {
 707         struct pdev_entry *p;
 708
 709         p = pdev_htab[pdev_hashfn(devhandle, bus, device, func)];
 710         while (p) {
 711                 if (p->devhandle == devhandle &&
 712                     p->bus == bus &&
 713                     p->device == device &&
 714                     p->func == func)
 715                         break;
 716
 717                 p = p->next;
 718         }
 719
 720         return p;
 721 }
 722
 723 static inline int pci_sun4v_out_of_range(struct pci_pbm_info *pbm, unsigned int bus, unsigned int device, unsigned int func)
 724 {
 725         if (bus < pbm->pci_first_busno ||
 726             bus > pbm->pci_last_busno)
 727                 return 1;
 728         return pdev_find(pbm->devhandle, bus, device, func) == NULL;
 729 }
 730
 731 static int pci_sun4v_read_pci_cfg(struct pci_bus *bus_dev, unsigned int devfn,
 732                                   int where, int size, u32 *value)
 733 {
 734         struct pci_pbm_info *pbm = bus_dev->sysdata;
 735         u32 devhandle = pbm->devhandle;
 736         unsigned int bus = bus_dev->number;
 737         unsigned int device = PCI_SLOT(devfn);
 738         unsigned int func = PCI_FUNC(devfn);
 739         unsigned long ret;
 740
 741         if (pci_sun4v_out_of_range(pbm, bus, device, func)) {
 742                 ret = ~0UL;
 743         } else {
 744                 ret = pci_sun4v_config_get(devhandle,
 745                                 HV_PCI_DEVICE_BUILD(bus, device, func),
 746                                 where, size);
 747 #if 0
 748                 printk("rcfg: [%x:%x:%x:%d]=[%lx]\n",
 749                        devhandle, HV_PCI_DEVICE_BUILD(bus, device, func),
 750                        where, size, ret);
 751 #endif
 752         }
 753         switch (size) {
 754         case 1:
 755                 *value = ret & 0xff;
 756                 break;
 757         case 2:
 758                 *value = ret & 0xffff;
 759                 break;
 760         case 4:
 761                 *value = ret & 0xffffffff;
 762                 break;
 763         };
 764
 765
 766         return PCIBIOS_SUCCESSFUL;
 767 }
 768
 769 static int pci_sun4v_write_pci_cfg(struct pci_bus *bus_dev, unsigned int devfn,
 770                                    int where, int size, u32 value)
 771 {
 772         struct pci_pbm_info *pbm = bus_dev->sysdata;
 773         u32 devhandle = pbm->devhandle;
 774         unsigned int bus = bus_dev->number;
 775         unsigned int device = PCI_SLOT(devfn);
 776         unsigned int func = PCI_FUNC(devfn);
 777         unsigned long ret;
 778
 779         if (pci_sun4v_out_of_range(pbm, bus, device, func)) {
 780                 /* Do nothing. */
 781         } else {
 782                 ret = pci_sun4v_config_put(devhandle,
 783                                 HV_PCI_DEVICE_BUILD(bus, device, func),
 784                                 where, size, value);
 785 #if 0
 786                 printk("wcfg: [%x:%x:%x:%d] v[%x] == [%lx]\n",
 787                        devhandle, HV_PCI_DEVICE_BUILD(bus, device, func),
 788                        where, size, value, ret);
 789 #endif
 790         }
 791         return PCIBIOS_SUCCESSFUL;
 792 }
 793
 794 static struct pci_ops pci_sun4v_ops = {
 795         .read =         pci_sun4v_read_pci_cfg,
 796         .write =        pci_sun4v_write_pci_cfg,
 797 };
 798
 799
 800 static void pbm_scan_bus(struct pci_controller_info *p,
 801                          struct pci_pbm_info *pbm)
 802 {
 803         struct pcidev_cookie *cookie = kzalloc(sizeof(*cookie), GFP_KERNEL);
 804
 805         if (!cookie) {
 806                 prom_printf("%s: Critical allocation failure.\n", pbm->name);
 807                 prom_halt();
 808         }
 809
 810         /* All we care about is the PBM. */
 811         cookie->pbm = pbm;
 812
 813         pbm->pci_bus = pci_scan_bus(pbm->pci_first_busno, p->pci_ops, pbm);
 814 #if 0
 815         pci_fixup_host_bridge_self(pbm->pci_bus);
 816         pbm->pci_bus->self->sysdata = cookie;
 817 #endif
 818         pci_fill_in_pbm_cookies(pbm->pci_bus, pbm, pbm->prom_node);
 819         pci_record_assignments(pbm, pbm->pci_bus);
 820         pci_assign_unassigned(pbm, pbm->pci_bus);
 821         pci_fixup_irq(pbm, pbm->pci_bus);
 822         pci_determine_66mhz_disposition(pbm, pbm->pci_bus);
 823         pci_setup_busmastering(pbm, pbm->pci_bus);
 824 }
 825
 826 static void pci_sun4v_scan_bus(struct pci_controller_info *p)
 827 {
 828         struct property *prop;
 829         struct device_node *dp;
 830
 831         if ((dp = p->pbm_A.prom_node) != NULL) {
 832                 prop = of_find_property(dp, "66mhz-capable", NULL);
 833                 p->pbm_A.is_66mhz_capable = (prop != NULL);
 834
 835                 pbm_scan_bus(p, &p->pbm_A);
 836         }
 837         if ((dp = p->pbm_B.prom_node) != NULL) {
 838                 prop = of_find_property(dp, "66mhz-capable", NULL);
 839                 p->pbm_B.is_66mhz_capable = (prop != NULL);
 840
 841                 pbm_scan_bus(p, &p->pbm_B);
 842         }
 843
 844         /* XXX register error interrupt handlers XXX */
 845 }
 846
 847 static void pci_sun4v_base_address_update(struct pci_dev *pdev, int resource)
 848 {
 849         struct pcidev_cookie *pcp = pdev->sysdata;
 850         struct pci_pbm_info *pbm = pcp->pbm;
 851         struct resource *res, *root;
 852         u32 reg;
 853         int where, size, is_64bit;
 854
 855         res = &pdev->resource[resource];
 856         if (resource < 6) {
 857                 where = PCI_BASE_ADDRESS_0 + (resource * 4);
 858         } else if (resource == PCI_ROM_RESOURCE) {
 859                 where = pdev->rom_base_reg;
 860         } else {
 861                 /* Somebody might have asked allocation of a non-standard resource */
 862                 return;
 863         }
 864
 865         /* XXX 64-bit MEM handling is not %100 correct... XXX */
 866         is_64bit = 0;
 867         if (res->flags & IORESOURCE_IO)
 868                 root = &pbm->io_space;
 869         else {
 870                 root = &pbm->mem_space;
 871                 if ((res->flags & PCI_BASE_ADDRESS_MEM_TYPE_MASK)
 872                     == PCI_BASE_ADDRESS_MEM_TYPE_64)
 873                         is_64bit = 1;
 874         }
 875
 876         size = res->end - res->start;
 877         pci_read_config_dword(pdev, where, &reg);
 878         reg = ((reg & size) |
 879                (((u32)(res->start - root->start)) & ~size));
 880         if (resource == PCI_ROM_RESOURCE) {
 881                 reg |= PCI_ROM_ADDRESS_ENABLE;
 882                 res->flags |= IORESOURCE_ROM_ENABLE;
 883         }
 884         pci_write_config_dword(pdev, where, reg);
 885
 886         /* This knows that the upper 32-bits of the address
 887          * must be zero.  Our PCI common layer enforces this.
 888          */
 889         if (is_64bit)
 890                 pci_write_config_dword(pdev, where + 4, 0);
 891 }
 892
 893 static void pci_sun4v_resource_adjust(struct pci_dev *pdev,
 894                                       struct resource *res,
 895                                       struct resource *root)
 896 {
 897         res->start += root->start;
 898         res->end += root->start;
 899 }
 900
 901 /* Use ranges property to determine where PCI MEM, I/O, and Config
 902  * space are for this PCI bus module.
 903  */
 904 static void pci_sun4v_determine_mem_io_space(struct pci_pbm_info *pbm)
 905 {
 906         int i, saw_mem, saw_io;
 907
 908         saw_mem = saw_io = 0;
 909         for (i = 0; i < pbm->num_pbm_ranges; i++) {
 910                 struct linux_prom_pci_ranges *pr = &pbm->pbm_ranges[i];
 911                 unsigned long a;
 912                 int type;
 913
 914                 type = (pr->child_phys_hi >> 24) & 0x3;
 915                 a = (((unsigned long)pr->parent_phys_hi << 32UL) |
 916                      ((unsigned long)pr->parent_phys_lo  <<  0UL));
 917
 918                 switch (type) {
 919                 case 1:
 920                         /* 16-bit IO space, 16MB */
 921                         pbm->io_space.start = a;
 922                         pbm->io_space.end = a + ((16UL*1024UL*1024UL) - 1UL);
 923                         pbm->io_space.flags = IORESOURCE_IO;
 924                         saw_io = 1;
 925                         break;
 926
 927                 case 2:
 928                         /* 32-bit MEM space, 2GB */
 929                         pbm->mem_space.start = a;
 930                         pbm->mem_space.end = a + (0x80000000UL - 1UL);
 931                         pbm->mem_space.flags = IORESOURCE_MEM;
 932                         saw_mem = 1;
 933                         break;
 934
 935                 case 3:
 936                         /* XXX 64-bit MEM handling XXX */
 937
 938                 default:
 939                         break;
 940                 };
 941         }
 942
 943         if (!saw_io || !saw_mem) {
 944                 prom_printf("%s: Fatal error, missing %s PBM range.\n",
 945                             pbm->name,
 946                             (!saw_io ? "IO" : "MEM"));
 947                 prom_halt();
 948         }
 949
 950         printk("%s: PCI IO[%lx] MEM[%lx]\n",
 951                pbm->name,
 952                pbm->io_space.start,
 953                pbm->mem_space.start);
 954 }
 955
 956 static void pbm_register_toplevel_resources(struct pci_controller_info *p,
 957                                             struct pci_pbm_info *pbm)
 958 {
 959         pbm->io_space.name = pbm->mem_space.name = pbm->name;
 960
 961         request_resource(&ioport_resource, &pbm->io_space);
 962         request_resource(&iomem_resource, &pbm->mem_space);
 963         pci_register_legacy_regions(&pbm->io_space,
 964                                     &pbm->mem_space);
 965 }
 966
 967 static unsigned long probe_existing_entries(struct pci_pbm_info *pbm,
 968                                             struct pci_iommu *iommu)
 969 {
 970         struct pci_iommu_arena *arena = &iommu->arena;
 971         unsigned long i, cnt = 0;
 972         u32 devhandle;
 973
 974         devhandle = pbm->devhandle;
 975         for (i = 0; i < arena->limit; i++) {
 976                 unsigned long ret, io_attrs, ra;
 977
 978                 ret = pci_sun4v_iommu_getmap(devhandle,
 979                                              HV_PCI_TSBID(0, i),
 980                                              &io_attrs, &ra);
 981                 if (ret == HV_EOK) {
 982                         if (page_in_phys_avail(ra)) {
 983                                 pci_sun4v_iommu_demap(devhandle,
 984                                                       HV_PCI_TSBID(0, i), 1);
 985                         } else {
 986                                 cnt++;
 987                                 __set_bit(i, arena->map);
 988                         }
 989                 }
 990         }
 991
 992         return cnt;
 993 }
 994
 995 static void pci_sun4v_iommu_init(struct pci_pbm_info *pbm)
 996 {
 997         struct pci_iommu *iommu = pbm->iommu;
 998         struct property *prop;
 999         unsigned long num_tsb_entries, sz;
1000         u32 vdma[2], dma_mask, dma_offset;
1001         int tsbsize;
1002
1003         prop = of_find_property(pbm->prom_node, "virtual-dma", NULL);
1004         if (prop) {
1005                 u32 *val = prop->value;
1006
1007                 vdma[0] = val[0];
1008                 vdma[1] = val[1];
1009         } else {
1010                 /* No property, use default values. */
1011                 vdma[0] = 0x80000000;
1012                 vdma[1] = 0x80000000;
1013         }
1014
1015         dma_mask = vdma[0];
1016         switch (vdma[1]) {
1017                 case 0x20000000:
1018                         dma_mask |= 0x1fffffff;
1019                         tsbsize = 64;
1020                         break;
1021
1022                 case 0x40000000:
1023                         dma_mask |= 0x3fffffff;
1024                         tsbsize = 128;
1025                         break;
1026
1027                 case 0x80000000:
1028                         dma_mask |= 0x7fffffff;
1029                         tsbsize = 256;
1030                         break;
1031
1032                 default:
1033                         prom_printf("PCI-SUN4V: strange virtual-dma size.\n");
1034                         prom_halt();
1035         };
1036
1037         tsbsize *= (8 * 1024);
1038
1039         num_tsb_entries = tsbsize / sizeof(iopte_t);
1040
1041         dma_offset = vdma[0];
1042
1043         /* Setup initial software IOMMU state. */
1044         spin_lock_init(&iommu->lock);
1045         iommu->ctx_lowest_free = 1;
1046         iommu->page_table_map_base = dma_offset;
1047         iommu->dma_addr_mask = dma_mask;
1048
1049         /* Allocate and initialize the free area map.  */
1050         sz = num_tsb_entries / 8;
1051         sz = (sz + 7UL) & ~7UL;
1052         iommu->arena.map = kzalloc(sz, GFP_KERNEL);
1053         if (!iommu->arena.map) {
1054                 prom_printf("PCI_IOMMU: Error, kmalloc(arena.map) failed.\n");
1055                 prom_halt();
1056         }
1057         iommu->arena.limit = num_tsb_entries;
1058
1059         sz = probe_existing_entries(pbm, iommu);
1060         if (sz)
1061                 printk("%s: Imported %lu TSB entries from OBP\n",
1062                        pbm->name, sz);
1063 }
1064
1065 static void pci_sun4v_get_bus_range(struct pci_pbm_info *pbm)
1066 {
1067         struct property *prop;
1068         unsigned int *busrange;
1069
1070         prop = of_find_property(pbm->prom_node, "bus-range", NULL);
1071
1072         busrange = prop->value;
1073
1074         pbm->pci_first_busno = busrange[0];
1075         pbm->pci_last_busno = busrange[1];
1076
1077 }
1078
1079 #ifdef CONFIG_PCI_MSI
1080 struct pci_sun4v_msiq_entry {
1081         u64             version_type;
1082 #define MSIQ_VERSION_MASK               0xffffffff00000000UL
1083 #define MSIQ_VERSION_SHIFT              32
1084 #define MSIQ_TYPE_MASK                  0x00000000000000ffUL
1085 #define MSIQ_TYPE_SHIFT                 0
1086 #define MSIQ_TYPE_NONE                  0x00
1087 #define MSIQ_TYPE_MSG                   0x01
1088 #define MSIQ_TYPE_MSI32                 0x02
1089 #define MSIQ_TYPE_MSI64                 0x03
1090 #define MSIQ_TYPE_INTX                  0x08
1091 #define MSIQ_TYPE_NONE2                 0xff
1092
1093         u64             intx_sysino;
1094         u64             reserved1;
1095         u64             stick;
1096         u64             req_id;  /* bus/device/func */
1097 #define MSIQ_REQID_BUS_MASK             0xff00UL
1098 #define MSIQ_REQID_BUS_SHIFT            8
1099 #define MSIQ_REQID_DEVICE_MASK          0x00f8UL
1100 #define MSIQ_REQID_DEVICE_SHIFT         3
1101 #define MSIQ_REQID_FUNC_MASK            0x0007UL
1102 #define MSIQ_REQID_FUNC_SHIFT           0
1103
1104         u64             msi_address;
1105
1106         /* The format of this value is message type dependant.
1107          * For MSI bits 15:0 are the data from the MSI packet.
1108          * For MSI-X bits 31:0 are the data from the MSI packet.
1109          * For MSG, the message code and message routing code where:
1110          *      bits 39:32 is the bus/device/fn of the msg target-id
1111          *      bits 18:16 is the message routing code
1112          *      bits 7:0 is the message code
1113          * For INTx the low order 2-bits are:
1114          *      00 - INTA
1115          *      01 - INTB
1116          *      10 - INTC
1117          *      11 - INTD
1118          */
1119         u64             msi_data;
1120
1121         u64             reserved2;
1122 };
1123
1124 /* For now this just runs as a pre-handler for the real interrupt handler.
1125  * So we just walk through the queue and ACK all the entries, update the
1126  * head pointer, and return.
1127  *
1128  * In the longer term it would be nice to do something more integrated
1129  * wherein we can pass in some of this MSI info to the drivers.  This
1130  * would be most useful for PCIe fabric error messages, although we could
1131  * invoke those directly from the loop here in order to pass the info around.
1132  */
1133 static void pci_sun4v_msi_prehandler(unsigned int ino, void *data1, void *data2)
1134 {
1135         struct pci_pbm_info *pbm = data1;
1136         struct pci_sun4v_msiq_entry *base, *ep;
1137         unsigned long msiqid, orig_head, head, type, err;
1138
1139         msiqid = (unsigned long) data2;
1140
1141         head = 0xdeadbeef;
1142         err = pci_sun4v_msiq_gethead(pbm->devhandle, msiqid, &head);
1143         if (unlikely(err))
1144                 goto hv_error_get;
1145
1146         if (unlikely(head >= (pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry))))
1147                 goto bad_offset;
1148
1149         head /= sizeof(struct pci_sun4v_msiq_entry);
1150         orig_head = head;
1151         base = (pbm->msi_queues + ((msiqid - pbm->msiq_first) *
1152                                    (pbm->msiq_ent_count *
1153                                     sizeof(struct pci_sun4v_msiq_entry))));
1154         ep = &base[head];
1155         while ((ep->version_type & MSIQ_TYPE_MASK) != 0) {
1156                 type = (ep->version_type & MSIQ_TYPE_MASK) >> MSIQ_TYPE_SHIFT;
1157                 if (unlikely(type != MSIQ_TYPE_MSI32 &&
1158                              type != MSIQ_TYPE_MSI64))
1159                         goto bad_type;
1160
1161                 pci_sun4v_msi_setstate(pbm->devhandle,
1162                                        ep->msi_data /* msi_num */,
1163                                        HV_MSISTATE_IDLE);
1164
1165                 /* Clear the entry.  */
1166                 ep->version_type &= ~MSIQ_TYPE_MASK;
1167
1168                 /* Go to next entry in ring.  */
1169                 head++;
1170                 if (head >= pbm->msiq_ent_count)
1171                         head = 0;
1172                 ep = &base[head];
1173         }
1174
1175         if (likely(head != orig_head)) {
1176                 /* ACK entries by updating head pointer.  */
1177                 head *= sizeof(struct pci_sun4v_msiq_entry);
1178                 err = pci_sun4v_msiq_sethead(pbm->devhandle, msiqid, head);
1179                 if (unlikely(err))
1180                         goto hv_error_set;
1181         }
1182         return;
1183
1184 hv_error_set:
1185         printk(KERN_EMERG "MSI: Hypervisor set head gives error %lu\n", err);
1186         goto hv_error_cont;
1187
1188 hv_error_get:
1189         printk(KERN_EMERG "MSI: Hypervisor get head gives error %lu\n", err);
1190
1191 hv_error_cont:
1192         printk(KERN_EMERG "MSI: devhandle[%x] msiqid[%lx] head[%lu]\n",
1193                pbm->devhandle, msiqid, head);
1194         return;
1195
1196 bad_offset:
1197         printk(KERN_EMERG "MSI: Hypervisor gives bad offset %lx max(%lx)\n",
1198                head, pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry));
1199         return;
1200
1201 bad_type:
1202         printk(KERN_EMERG "MSI: Entry has bad type %lx\n", type);
1203         return;
1204 }
1205
1206 static int msi_bitmap_alloc(struct pci_pbm_info *pbm)
1207 {
1208         unsigned long size, bits_per_ulong;
1209
1210         bits_per_ulong = sizeof(unsigned long) * 8;
1211         size = (pbm->msi_num + (bits_per_ulong - 1)) & ~(bits_per_ulong - 1);
1212         size /= 8;
1213         BUG_ON(size % sizeof(unsigned long));
1214
1215         pbm->msi_bitmap = kzalloc(size, GFP_KERNEL);
1216         if (!pbm->msi_bitmap)
1217                 return -ENOMEM;
1218
1219         return 0;
1220 }
1221
1222 static void msi_bitmap_free(struct pci_pbm_info *pbm)
1223 {
1224         kfree(pbm->msi_bitmap);
1225         pbm->msi_bitmap = NULL;
1226 }
1227
1228 static int msi_queue_alloc(struct pci_pbm_info *pbm)
1229 {
1230         unsigned long q_size, alloc_size, pages, order;
1231         int i;
1232
1233         q_size = pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry);
1234         alloc_size = (pbm->msiq_num * q_size);
1235         order = get_order(alloc_size);
1236         pages = __get_free_pages(GFP_KERNEL | __GFP_COMP, order);
1237         if (pages == 0UL) {
1238                 printk(KERN_ERR "MSI: Cannot allocate MSI queues (o=%lu).\n",
1239                        order);
1240                 return -ENOMEM;
1241         }
1242         memset((char *)pages, 0, PAGE_SIZE << order);
1243         pbm->msi_queues = (void *) pages;
1244
1245         for (i = 0; i < pbm->msiq_num; i++) {
1246                 unsigned long err, base = __pa(pages + (i * q_size));
1247                 unsigned long ret1, ret2;
1248
1249                 err = pci_sun4v_msiq_conf(pbm->devhandle,
1250                                           pbm->msiq_first + i,
1251                                           base, pbm->msiq_ent_count);
1252                 if (err) {
1253                         printk(KERN_ERR "MSI: msiq register fails (err=%lu)\n",
1254                                err);
1255                         goto h_error;
1256                 }
1257
1258                 err = pci_sun4v_msiq_info(pbm->devhandle,
1259                                           pbm->msiq_first + i,
1260                                           &ret1, &ret2);
1261                 if (err) {
1262                         printk(KERN_ERR "MSI: Cannot read msiq (err=%lu)\n",
1263                                err);
1264                         goto h_error;
1265                 }
1266                 if (ret1 != base || ret2 != pbm->msiq_ent_count) {
1267                         printk(KERN_ERR "MSI: Bogus qconf "
1268                                "expected[%lx:%x] got[%lx:%lx]\n",
1269                                base, pbm->msiq_ent_count,
1270                                ret1, ret2);
1271                         goto h_error;
1272                 }
1273         }
1274
1275         return 0;
1276
1277 h_error:
1278         free_pages(pages, order);
1279         return -EINVAL;
1280 }
1281
1282 static void pci_sun4v_msi_init(struct pci_pbm_info *pbm)
1283 {
1284         u32 *val;
1285         int len;
1286
1287         val = of_get_property(pbm->prom_node, "#msi-eqs", &len);
1288         if (!val || len != 4)
1289                 goto no_msi;
1290         pbm->msiq_num = *val;
1291         if (pbm->msiq_num) {
1292                 struct msiq_prop {
1293                         u32 first_msiq;
1294                         u32 num_msiq;
1295                         u32 first_devino;
1296                 } *mqp;
1297                 struct msi_range_prop {
1298                         u32 first_msi;
1299                         u32 num_msi;
1300                 } *mrng;
1301                 struct addr_range_prop {
1302                         u32 msi32_high;
1303                         u32 msi32_low;
1304                         u32 msi32_len;
1305                         u32 msi64_high;
1306                         u32 msi64_low;
1307                         u32 msi64_len;
1308                 } *arng;
1309
1310                 val = of_get_property(pbm->prom_node, "msi-eq-size", &len);
1311                 if (!val || len != 4)
1312                         goto no_msi;
1313
1314                 pbm->msiq_ent_count = *val;
1315
1316                 mqp = of_get_property(pbm->prom_node,
1317                                       "msi-eq-to-devino", &len);
1318                 if (!mqp || len != sizeof(struct msiq_prop))
1319                         goto no_msi;
1320
1321                 pbm->msiq_first = mqp->first_msiq;
1322                 pbm->msiq_first_devino = mqp->first_devino;
1323
1324                 val = of_get_property(pbm->prom_node, "#msi", &len);
1325                 if (!val || len != 4)
1326                         goto no_msi;
1327                 pbm->msi_num = *val;
1328
1329                 mrng = of_get_property(pbm->prom_node, "msi-ranges", &len);
1330                 if (!mrng || len != sizeof(struct msi_range_prop))
1331                         goto no_msi;
1332                 pbm->msi_first = mrng->first_msi;
1333
1334                 val = of_get_property(pbm->prom_node, "msi-data-mask", &len);
1335                 if (!val || len != 4)
1336                         goto no_msi;
1337                 pbm->msi_data_mask = *val;
1338
1339                 val = of_get_property(pbm->prom_node, "msix-data-width", &len);
1340                 if (!val || len != 4)
1341                         goto no_msi;
1342                 pbm->msix_data_width = *val;
1343
1344                 arng = of_get_property(pbm->prom_node, "msi-address-ranges",
1345                                        &len);
1346                 if (!arng || len != sizeof(struct addr_range_prop))
1347                         goto no_msi;
1348                 pbm->msi32_start = ((u64)arng->msi32_high << 32) |
1349                         (u64) arng->msi32_low;
1350                 pbm->msi64_start = ((u64)arng->msi64_high << 32) |
1351                         (u64) arng->msi64_low;
1352                 pbm->msi32_len = arng->msi32_len;
1353                 pbm->msi64_len = arng->msi64_len;
1354
1355                 if (msi_bitmap_alloc(pbm))
1356                         goto no_msi;
1357
1358                 if (msi_queue_alloc(pbm)) {
1359                         msi_bitmap_free(pbm);
1360                         goto no_msi;
1361                 }
1362
1363                 printk(KERN_INFO "%s: MSI Queue first[%u] num[%u] count[%u] "
1364                        "devino[0x%x]\n",
1365                        pbm->name,
1366                        pbm->msiq_first, pbm->msiq_num,
1367                        pbm->msiq_ent_count,
1368                        pbm->msiq_first_devino);
1369                 printk(KERN_INFO "%s: MSI first[%u] num[%u] mask[0x%x] "
1370                        "width[%u]\n",
1371                        pbm->name,
1372                        pbm->msi_first, pbm->msi_num, pbm->msi_data_mask,
1373                        pbm->msix_data_width);
1374                 printk(KERN_INFO "%s: MSI addr32[0x%lx:0x%x] "
1375                        "addr64[0x%lx:0x%x]\n",
1376                        pbm->name,
1377                        pbm->msi32_start, pbm->msi32_len,
1378                        pbm->msi64_start, pbm->msi64_len);
1379                 printk(KERN_INFO "%s: MSI queues at RA [%p]\n",
1380                        pbm->name,
1381                        pbm->msi_queues);
1382         }
1383
1384         return;
1385
1386 no_msi:
1387         pbm->msiq_num = 0;
1388         printk(KERN_INFO "%s: No MSI support.\n", pbm->name);
1389 }
1390
1391 static int alloc_msi(struct pci_pbm_info *pbm)
1392 {
1393         int i;
1394
1395         for (i = 0; i < pbm->msi_num; i++) {
1396                 if (!test_and_set_bit(i, pbm->msi_bitmap))
1397                         return i + pbm->msi_first;
1398         }
1399
1400         return -ENOENT;
1401 }
1402
1403 static void free_msi(struct pci_pbm_info *pbm, int msi_num)
1404 {
1405         msi_num -= pbm->msi_first;
1406         clear_bit(msi_num, pbm->msi_bitmap);
1407 }
1408
1409 static int pci_sun4v_setup_msi_irq(unsigned int *virt_irq_p,
1410                                    struct pci_dev *pdev,
1411                                    struct msi_desc *entry)
1412 {
1413         struct pcidev_cookie *pcp = pdev->sysdata;
1414         struct pci_pbm_info *pbm = pcp->pbm;
1415         unsigned long devino, msiqid;
1416         struct msi_msg msg;
1417         int msi_num, err;
1418
1419         *virt_irq_p = 0;
1420
1421         msi_num = alloc_msi(pbm);
1422         if (msi_num < 0)
1423                 return msi_num;
1424
1425         devino = sun4v_build_msi(pbm->devhandle, virt_irq_p,
1426                                  pbm->msiq_first_devino,
1427                                  (pbm->msiq_first_devino +
1428                                   pbm->msiq_num));
1429         err = -ENOMEM;
1430         if (!devino)
1431                 goto out_err;
1432
1433         set_irq_msi(*virt_irq_p, entry);
1434
1435         msiqid = ((devino - pbm->msiq_first_devino) +
1436                   pbm->msiq_first);
1437
1438         err = -EINVAL;
1439         if (pci_sun4v_msiq_setstate(pbm->devhandle, msiqid, HV_MSIQSTATE_IDLE))
1440         if (err)
1441                 goto out_err;
1442
1443         if (pci_sun4v_msiq_setvalid(pbm->devhandle, msiqid, HV_MSIQ_VALID))
1444                 goto out_err;
1445
1446         if (pci_sun4v_msi_setmsiq(pbm->devhandle,
1447                                   msi_num, msiqid,
1448                                   (entry->msi_attrib.is_64 ?
1449                                    HV_MSITYPE_MSI64 : HV_MSITYPE_MSI32)))
1450                 goto out_err;
1451
1452         if (pci_sun4v_msi_setstate(pbm->devhandle, msi_num, HV_MSISTATE_IDLE))
1453                 goto out_err;
1454
1455         if (pci_sun4v_msi_setvalid(pbm->devhandle, msi_num, HV_MSIVALID_VALID))
1456                 goto out_err;
1457
1458         pcp->msi_num = msi_num;
1459
1460         if (entry->msi_attrib.is_64) {
1461                 msg.address_hi = pbm->msi64_start >> 32;
1462                 msg.address_lo = pbm->msi64_start & 0xffffffff;
1463         } else {
1464                 msg.address_hi = 0;
1465                 msg.address_lo = pbm->msi32_start;
1466         }
1467         msg.data = msi_num;
1468         write_msi_msg(*virt_irq_p, &msg);
1469
1470         irq_install_pre_handler(*virt_irq_p,
1471                                 pci_sun4v_msi_prehandler,
1472                                 pbm, (void *) msiqid);
1473
1474         return 0;
1475
1476 out_err:
1477         free_msi(pbm, msi_num);
1478         sun4v_destroy_msi(*virt_irq_p);
1479         *virt_irq_p = 0;
1480         return err;
1481
1482 }
1483
1484 static void pci_sun4v_teardown_msi_irq(unsigned int virt_irq,
1485                                        struct pci_dev *pdev)
1486 {
1487         struct pcidev_cookie *pcp = pdev->sysdata;
1488         struct pci_pbm_info *pbm = pcp->pbm;
1489         unsigned long msiqid, err;
1490         unsigned int msi_num;
1491
1492         msi_num = pcp->msi_num;
1493         err = pci_sun4v_msi_getmsiq(pbm->devhandle, msi_num, &msiqid);
1494         if (err) {
1495                 printk(KERN_ERR "%s: getmsiq gives error %lu\n",
1496                        pbm->name, err);
1497                 return;
1498         }
1499
1500         pci_sun4v_msi_setvalid(pbm->devhandle, msi_num, HV_MSIVALID_INVALID);
1501         pci_sun4v_msiq_setvalid(pbm->devhandle, msiqid, HV_MSIQ_INVALID);
1502
1503         free_msi(pbm, msi_num);
1504
1505         /* The sun4v_destroy_msi() will liberate the devino and thus the MSIQ
1506          * allocation.
1507          */
1508         sun4v_destroy_msi(virt_irq);
1509 }
1510 #else /* CONFIG_PCI_MSI */
1511 static void pci_sun4v_msi_init(struct pci_pbm_info *pbm)
1512 {
1513 }
1514 #endif /* !(CONFIG_PCI_MSI) */
1515
1516 static void pci_sun4v_pbm_init(struct pci_controller_info *p, struct device_node *dp, u32 devhandle)
1517 {
1518         struct pci_pbm_info *pbm;
1519         struct property *prop;
1520         int len, i;
1521
1522         if (devhandle & 0x40)
1523                 pbm = &p->pbm_B;
1524         else
1525                 pbm = &p->pbm_A;
1526
1527         pbm->parent = p;
1528         pbm->prom_node = dp;
1529         pbm->pci_first_slot = 1;
1530
1531         pbm->devhandle = devhandle;
1532
1533         pbm->name = dp->full_name;
1534
1535         printk("%s: SUN4V PCI Bus Module\n", pbm->name);
1536
1537         prop = of_find_property(dp, "ranges", &len);
1538         pbm->pbm_ranges = prop->value;
1539         pbm->num_pbm_ranges =
1540                 (len / sizeof(struct linux_prom_pci_ranges));
1541
1542         /* Mask out the top 8 bits of the ranges, leaving the real
1543          * physical address.
1544          */
1545         for (i = 0; i < pbm->num_pbm_ranges; i++)
1546                 pbm->pbm_ranges[i].parent_phys_hi &= 0x0fffffff;
1547
1548         pci_sun4v_determine_mem_io_space(pbm);
1549         pbm_register_toplevel_resources(p, pbm);
1550
1551         prop = of_find_property(dp, "interrupt-map", &len);
1552         pbm->pbm_intmap = prop->value;
1553         pbm->num_pbm_intmap =
1554                 (len / sizeof(struct linux_prom_pci_intmap));
1555
1556         prop = of_find_property(dp, "interrupt-map-mask", NULL);
1557         pbm->pbm_intmask = prop->value;
1558
1559         pci_sun4v_get_bus_range(pbm);
1560         pci_sun4v_iommu_init(pbm);
1561         pci_sun4v_msi_init(pbm);
1562
1563         pdev_htab_populate(pbm);
1564 }
1565
1566 void sun4v_pci_init(struct device_node *dp, char *model_name)
1567 {
1568         struct pci_controller_info *p;
1569         struct pci_iommu *iommu;
1570         struct property *prop;
1571         struct linux_prom64_registers *regs;
1572         u32 devhandle;
1573         int i;
1574
1575         prop = of_find_property(dp, "reg", NULL);
1576         regs = prop->value;
1577
1578         devhandle = (regs->phys_addr >> 32UL) & 0x0fffffff;
1579
1580         for (p = pci_controller_root; p; p = p->next) {
1581                 struct pci_pbm_info *pbm;
1582
1583                 if (p->pbm_A.prom_node && p->pbm_B.prom_node)
1584                         continue;
1585
1586                 pbm = (p->pbm_A.prom_node ?
1587                        &p->pbm_A :
1588                        &p->pbm_B);
1589
1590                 if (pbm->devhandle == (devhandle ^ 0x40)) {
1591                         pci_sun4v_pbm_init(p, dp, devhandle);
1592                         return;
1593                 }
1594         }
1595
1596         for_each_possible_cpu(i) {
1597                 unsigned long page = get_zeroed_page(GFP_ATOMIC);
1598
1599                 if (!page)
1600                         goto fatal_memory_error;
1601
1602                 per_cpu(pci_iommu_batch, i).pglist = (u64 *) page;
1603         }
1604
1605         p = kzalloc(sizeof(struct pci_controller_info), GFP_ATOMIC);
1606         if (!p)
1607                 goto fatal_memory_error;
1608
1609         iommu = kzalloc(sizeof(struct pci_iommu), GFP_ATOMIC);
1610         if (!iommu)
1611                 goto fatal_memory_error;
1612
1613         p->pbm_A.iommu = iommu;
1614
1615         iommu = kzalloc(sizeof(struct pci_iommu), GFP_ATOMIC);
1616         if (!iommu)
1617                 goto fatal_memory_error;
1618
1619         p->pbm_B.iommu = iommu;
1620
1621         p->next = pci_controller_root;
1622         pci_controller_root = p;
1623
1624         p->index = pci_num_controllers++;
1625         p->pbms_same_domain = 0;
1626
1627         p->scan_bus = pci_sun4v_scan_bus;
1628         p->base_address_update = pci_sun4v_base_address_update;
1629         p->resource_adjust = pci_sun4v_resource_adjust;
1630 #ifdef CONFIG_PCI_MSI
1631         p->setup_msi_irq = pci_sun4v_setup_msi_irq;
1632         p->teardown_msi_irq = pci_sun4v_teardown_msi_irq;
1633 #endif
1634         p->pci_ops = &pci_sun4v_ops;
1635
1636         /* Like PSYCHO and SCHIZO we have a 2GB aligned area
1637          * for memory space.
1638          */
1639         pci_memspace_mask = 0x7fffffffUL;
1640
1641         pci_sun4v_pbm_init(p, dp, devhandle);
1642         return;
1643
1644 fatal_memory_error:
1645         prom_printf("SUN4V_PCI: Fatal memory allocation error.\n");
1646         prom_halt();
1647 }