arch/powerpc/platforms/powernv/pci-ioda.c

   1 /*
   2  * Support PCI/PCIe on PowerNV platforms
   3  *
   4  * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version
   9  * 2 of the License, or (at your option) any later version.
  10  */
  11
  12 #undef DEBUG
  13
  14 #include <linux/kernel.h>
  15 #include <linux/pci.h>
  16 #include <linux/crash_dump.h>
  17 #include <linux/debugfs.h>
  18 #include <linux/delay.h>
  19 #include <linux/string.h>
  20 #include <linux/init.h>
  21 #include <linux/bootmem.h>
  22 #include <linux/irq.h>
  23 #include <linux/io.h>
  24 #include <linux/msi.h>
  25 #include <linux/memblock.h>
  26 #include <linux/iommu.h>
  27
  28 #include <asm/sections.h>
  29 #include <asm/io.h>
  30 #include <asm/prom.h>
  31 #include <asm/pci-bridge.h>
  32 #include <asm/machdep.h>
  33 #include <asm/msi_bitmap.h>
  34 #include <asm/ppc-pci.h>
  35 #include <asm/opal.h>
  36 #include <asm/iommu.h>
  37 #include <asm/tce.h>
  38 #include <asm/xics.h>
  39 #include <asm/debug.h>
  40 #include <asm/firmware.h>
  41 #include <asm/pnv-pci.h>
  42
  43 #include <misc/cxl-base.h>
  44
  45 #include "powernv.h"
  46 #include "pci.h"
  47
  48 /* 256M DMA window, 4K TCE pages, 8 bytes TCE */
  49 #define TCE32_TABLE_SIZE        ((0x10000000 / 0x1000) * 8)
  50
  51 static void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level,
  52                             const char *fmt, ...)
  53 {
  54         struct va_format vaf;
  55         va_list args;
  56         char pfix[32];
  57
  58         va_start(args, fmt);
  59
  60         vaf.fmt = fmt;
  61         vaf.va = &args;
  62
  63         if (pe->flags & PNV_IODA_PE_DEV)
  64                 strlcpy(pfix, dev_name(&pe->pdev->dev), sizeof(pfix));
  65         else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
  66                 sprintf(pfix, "%04x:%02x     ",
  67                         pci_domain_nr(pe->pbus), pe->pbus->number);
  68 #ifdef CONFIG_PCI_IOV
  69         else if (pe->flags & PNV_IODA_PE_VF)
  70                 sprintf(pfix, "%04x:%02x:%2x.%d",
  71                         pci_domain_nr(pe->parent_dev->bus),
  72                         (pe->rid & 0xff00) >> 8,
  73                         PCI_SLOT(pe->rid), PCI_FUNC(pe->rid));
  74 #endif /* CONFIG_PCI_IOV*/
  75
  76         printk("%spci %s: [PE# %.3d] %pV",
  77                level, pfix, pe->pe_number, &vaf);
  78
  79         va_end(args);
  80 }
  81
  82 #define pe_err(pe, fmt, ...)                                    \
  83         pe_level_printk(pe, KERN_ERR, fmt, ##__VA_ARGS__)
  84 #define pe_warn(pe, fmt, ...)                                   \
  85         pe_level_printk(pe, KERN_WARNING, fmt, ##__VA_ARGS__)
  86 #define pe_info(pe, fmt, ...)                                   \
  87         pe_level_printk(pe, KERN_INFO, fmt, ##__VA_ARGS__)
  88
  89 static bool pnv_iommu_bypass_disabled __read_mostly;
  90
  91 static int __init iommu_setup(char *str)
  92 {
  93         if (!str)
  94                 return -EINVAL;
  95
  96         while (*str) {
  97                 if (!strncmp(str, "nobypass", 8)) {
  98                         pnv_iommu_bypass_disabled = true;
  99                         pr_info("PowerNV: IOMMU bypass window disabled.\n");
 100                         break;
 101                 }
 102                 str += strcspn(str, ",");
 103                 if (*str == ',')
 104                         str++;
 105         }
 106
 107         return 0;
 108 }
 109 early_param("iommu", iommu_setup);
 110
 111 /*
 112  * stdcix is only supposed to be used in hypervisor real mode as per
 113  * the architecture spec
 114  */
 115 static inline void __raw_rm_writeq(u64 val, volatile void __iomem *paddr)
 116 {
 117         __asm__ __volatile__("stdcix %0,0,%1"
 118                 : : "r" (val), "r" (paddr) : "memory");
 119 }
 120
 121 static inline bool pnv_pci_is_mem_pref_64(unsigned long flags)
 122 {
 123         return ((flags & (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH)) ==
 124                 (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH));
 125 }
 126
 127 static void pnv_ioda_reserve_pe(struct pnv_phb *phb, int pe_no)
 128 {
 129         if (!(pe_no >= 0 && pe_no < phb->ioda.total_pe)) {
 130                 pr_warn("%s: Invalid PE %d on PHB#%x\n",
 131                         __func__, pe_no, phb->hose->global_number);
 132                 return;
 133         }
 134
 135         if (test_and_set_bit(pe_no, phb->ioda.pe_alloc)) {
 136                 pr_warn("%s: PE %d was assigned on PHB#%x\n",
 137                         __func__, pe_no, phb->hose->global_number);
 138                 return;
 139         }
 140
 141         phb->ioda.pe_array[pe_no].phb = phb;
 142         phb->ioda.pe_array[pe_no].pe_number = pe_no;
 143 }
 144
 145 static int pnv_ioda_alloc_pe(struct pnv_phb *phb)
 146 {
 147         unsigned long pe;
 148
 149         do {
 150                 pe = find_next_zero_bit(phb->ioda.pe_alloc,
 151                                         phb->ioda.total_pe, 0);
 152                 if (pe >= phb->ioda.total_pe)
 153                         return IODA_INVALID_PE;
 154         } while(test_and_set_bit(pe, phb->ioda.pe_alloc));
 155
 156         phb->ioda.pe_array[pe].phb = phb;
 157         phb->ioda.pe_array[pe].pe_number = pe;
 158         return pe;
 159 }
 160
 161 static void pnv_ioda_free_pe(struct pnv_phb *phb, int pe)
 162 {
 163         WARN_ON(phb->ioda.pe_array[pe].pdev);
 164
 165         memset(&phb->ioda.pe_array[pe], 0, sizeof(struct pnv_ioda_pe));
 166         clear_bit(pe, phb->ioda.pe_alloc);
 167 }
 168
 169 /* The default M64 BAR is shared by all PEs */
 170 static int pnv_ioda2_init_m64(struct pnv_phb *phb)
 171 {
 172         const char *desc;
 173         struct resource *r;
 174         s64 rc;
 175
 176         /* Configure the default M64 BAR */
 177         rc = opal_pci_set_phb_mem_window(phb->opal_id,
 178                                          OPAL_M64_WINDOW_TYPE,
 179                                          phb->ioda.m64_bar_idx,
 180                                          phb->ioda.m64_base,
 181                                          0, /* unused */
 182                                          phb->ioda.m64_size);
 183         if (rc != OPAL_SUCCESS) {
 184                 desc = "configuring";
 185                 goto fail;
 186         }
 187
 188         /* Enable the default M64 BAR */
 189         rc = opal_pci_phb_mmio_enable(phb->opal_id,
 190                                       OPAL_M64_WINDOW_TYPE,
 191                                       phb->ioda.m64_bar_idx,
 192                                       OPAL_ENABLE_M64_SPLIT);
 193         if (rc != OPAL_SUCCESS) {
 194                 desc = "enabling";
 195                 goto fail;
 196         }
 197
 198         /* Mark the M64 BAR assigned */
 199         set_bit(phb->ioda.m64_bar_idx, &phb->ioda.m64_bar_alloc);
 200
 201         /*
 202          * Strip off the segment used by the reserved PE, which is
 203          * expected to be 0 or last one of PE capabicity.
 204          */
 205         r = &phb->hose->mem_resources[1];
 206         if (phb->ioda.reserved_pe == 0)
 207                 r->start += phb->ioda.m64_segsize;
 208         else if (phb->ioda.reserved_pe == (phb->ioda.total_pe - 1))
 209                 r->end -= phb->ioda.m64_segsize;
 210         else
 211                 pr_warn("  Cannot strip M64 segment for reserved PE#%d\n",
 212                         phb->ioda.reserved_pe);
 213
 214         return 0;
 215
 216 fail:
 217         pr_warn("  Failure %lld %s M64 BAR#%d\n",
 218                 rc, desc, phb->ioda.m64_bar_idx);
 219         opal_pci_phb_mmio_enable(phb->opal_id,
 220                                  OPAL_M64_WINDOW_TYPE,
 221                                  phb->ioda.m64_bar_idx,
 222                                  OPAL_DISABLE_M64);
 223         return -EIO;
 224 }
 225
 226 static void pnv_ioda2_reserve_m64_pe(struct pnv_phb *phb)
 227 {
 228         resource_size_t sgsz = phb->ioda.m64_segsize;
 229         struct pci_dev *pdev;
 230         struct resource *r;
 231         int base, step, i;
 232
 233         /*
 234          * Root bus always has full M64 range and root port has
 235          * M64 range used in reality. So we're checking root port
 236          * instead of root bus.
 237          */
 238         list_for_each_entry(pdev, &phb->hose->bus->devices, bus_list) {
 239                 for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++) {
 240                         r = &pdev->resource[PCI_BRIDGE_RESOURCES + i];
 241                         if (!r->parent ||
 242                             !pnv_pci_is_mem_pref_64(r->flags))
 243                                 continue;
 244
 245                         base = (r->start - phb->ioda.m64_base) / sgsz;
 246                         for (step = 0; step < resource_size(r) / sgsz; step++)
 247                                 pnv_ioda_reserve_pe(phb, base + step);
 248                 }
 249         }
 250 }
 251
 252 static int pnv_ioda2_pick_m64_pe(struct pnv_phb *phb,
 253                                  struct pci_bus *bus, int all)
 254 {
 255         resource_size_t segsz = phb->ioda.m64_segsize;
 256         struct pci_dev *pdev;
 257         struct resource *r;
 258         struct pnv_ioda_pe *master_pe, *pe;
 259         unsigned long size, *pe_alloc;
 260         bool found;
 261         int start, i, j;
 262
 263         /* Root bus shouldn't use M64 */
 264         if (pci_is_root_bus(bus))
 265                 return IODA_INVALID_PE;
 266
 267         /* We support only one M64 window on each bus */
 268         found = false;
 269         pci_bus_for_each_resource(bus, r, i) {
 270                 if (r && r->parent &&
 271                     pnv_pci_is_mem_pref_64(r->flags)) {
 272                         found = true;
 273                         break;
 274                 }
 275         }
 276
 277         /* No M64 window found ? */
 278         if (!found)
 279                 return IODA_INVALID_PE;
 280
 281         /* Allocate bitmap */
 282         size = _ALIGN_UP(phb->ioda.total_pe / 8, sizeof(unsigned long));
 283         pe_alloc = kzalloc(size, GFP_KERNEL);
 284         if (!pe_alloc) {
 285                 pr_warn("%s: Out of memory !\n",
 286                         __func__);
 287                 return IODA_INVALID_PE;
 288         }
 289
 290         /*
 291          * Figure out reserved PE numbers by the PE
 292          * the its child PEs.
 293          */
 294         start = (r->start - phb->ioda.m64_base) / segsz;
 295         for (i = 0; i < resource_size(r) / segsz; i++)
 296                 set_bit(start + i, pe_alloc);
 297
 298         if (all)
 299                 goto done;
 300
 301         /*
 302          * If the PE doesn't cover all subordinate buses,
 303          * we need subtract from reserved PEs for children.
 304          */
 305         list_for_each_entry(pdev, &bus->devices, bus_list) {
 306                 if (!pdev->subordinate)
 307                         continue;
 308
 309                 pci_bus_for_each_resource(pdev->subordinate, r, i) {
 310                         if (!r || !r->parent ||
 311                             !pnv_pci_is_mem_pref_64(r->flags))
 312                                 continue;
 313
 314                         start = (r->start - phb->ioda.m64_base) / segsz;
 315                         for (j = 0; j < resource_size(r) / segsz ; j++)
 316                                 clear_bit(start + j, pe_alloc);
 317                 }
 318         }
 319
 320         /*
 321          * the current bus might not own M64 window and that's all
 322          * contributed by its child buses. For the case, we needn't
 323          * pick M64 dependent PE#.
 324          */
 325         if (bitmap_empty(pe_alloc, phb->ioda.total_pe)) {
 326                 kfree(pe_alloc);
 327                 return IODA_INVALID_PE;
 328         }
 329
 330         /*
 331          * Figure out the master PE and put all slave PEs to master
 332          * PE's list to form compound PE.
 333          */
 334 done:
 335         master_pe = NULL;
 336         i = -1;
 337         while ((i = find_next_bit(pe_alloc, phb->ioda.total_pe, i + 1)) <
 338                 phb->ioda.total_pe) {
 339                 pe = &phb->ioda.pe_array[i];
 340
 341                 if (!master_pe) {
 342                         pe->flags |= PNV_IODA_PE_MASTER;
 343                         INIT_LIST_HEAD(&pe->slaves);
 344                         master_pe = pe;
 345                 } else {
 346                         pe->flags |= PNV_IODA_PE_SLAVE;
 347                         pe->master = master_pe;
 348                         list_add_tail(&pe->list, &master_pe->slaves);
 349                 }
 350         }
 351
 352         kfree(pe_alloc);
 353         return master_pe->pe_number;
 354 }
 355
 356 static void __init pnv_ioda_parse_m64_window(struct pnv_phb *phb)
 357 {
 358         struct pci_controller *hose = phb->hose;
 359         struct device_node *dn = hose->dn;
 360         struct resource *res;
 361         const u32 *r;
 362         u64 pci_addr;
 363
 364         /* FIXME: Support M64 for P7IOC */
 365         if (phb->type != PNV_PHB_IODA2) {
 366                 pr_info("  Not support M64 window\n");
 367                 return;
 368         }
 369
 370         if (!firmware_has_feature(FW_FEATURE_OPALv3)) {
 371                 pr_info("  Firmware too old to support M64 window\n");
 372                 return;
 373         }
 374
 375         r = of_get_property(dn, "ibm,opal-m64-window", NULL);
 376         if (!r) {
 377                 pr_info("  No <ibm,opal-m64-window> on %s\n",
 378                         dn->full_name);
 379                 return;
 380         }
 381
 382         res = &hose->mem_resources[1];
 383         res->start = of_translate_address(dn, r + 2);
 384         res->end = res->start + of_read_number(r + 4, 2) - 1;
 385         res->flags = (IORESOURCE_MEM | IORESOURCE_MEM_64 | IORESOURCE_PREFETCH);
 386         pci_addr = of_read_number(r, 2);
 387         hose->mem_offset[1] = res->start - pci_addr;
 388
 389         phb->ioda.m64_size = resource_size(res);
 390         phb->ioda.m64_segsize = phb->ioda.m64_size / phb->ioda.total_pe;
 391         phb->ioda.m64_base = pci_addr;
 392
 393         pr_info(" MEM64 0x%016llx..0x%016llx -> 0x%016llx\n",
 394                         res->start, res->end, pci_addr);
 395
 396         /* Use last M64 BAR to cover M64 window */
 397         phb->ioda.m64_bar_idx = 15;
 398         phb->init_m64 = pnv_ioda2_init_m64;
 399         phb->reserve_m64_pe = pnv_ioda2_reserve_m64_pe;
 400         phb->pick_m64_pe = pnv_ioda2_pick_m64_pe;
 401 }
 402
 403 static void pnv_ioda_freeze_pe(struct pnv_phb *phb, int pe_no)
 404 {
 405         struct pnv_ioda_pe *pe = &phb->ioda.pe_array[pe_no];
 406         struct pnv_ioda_pe *slave;
 407         s64 rc;
 408
 409         /* Fetch master PE */
 410         if (pe->flags & PNV_IODA_PE_SLAVE) {
 411                 pe = pe->master;
 412                 if (WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER)))
 413                         return;
 414
 415                 pe_no = pe->pe_number;
 416         }
 417
 418         /* Freeze master PE */
 419         rc = opal_pci_eeh_freeze_set(phb->opal_id,
 420                                      pe_no,
 421                                      OPAL_EEH_ACTION_SET_FREEZE_ALL);
 422         if (rc != OPAL_SUCCESS) {
 423                 pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
 424                         __func__, rc, phb->hose->global_number, pe_no);
 425                 return;
 426         }
 427
 428         /* Freeze slave PEs */
 429         if (!(pe->flags & PNV_IODA_PE_MASTER))
 430                 return;
 431
 432         list_for_each_entry(slave, &pe->slaves, list) {
 433                 rc = opal_pci_eeh_freeze_set(phb->opal_id,
 434                                              slave->pe_number,
 435                                              OPAL_EEH_ACTION_SET_FREEZE_ALL);
 436                 if (rc != OPAL_SUCCESS)
 437                         pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
 438                                 __func__, rc, phb->hose->global_number,
 439                                 slave->pe_number);
 440         }
 441 }
 442
 443 static int pnv_ioda_unfreeze_pe(struct pnv_phb *phb, int pe_no, int opt)
 444 {
 445         struct pnv_ioda_pe *pe, *slave;
 446         s64 rc;
 447
 448         /* Find master PE */
 449         pe = &phb->ioda.pe_array[pe_no];
 450         if (pe->flags & PNV_IODA_PE_SLAVE) {
 451                 pe = pe->master;
 452                 WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
 453                 pe_no = pe->pe_number;
 454         }
 455
 456         /* Clear frozen state for master PE */
 457         rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no, opt);
 458         if (rc != OPAL_SUCCESS) {
 459                 pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
 460                         __func__, rc, opt, phb->hose->global_number, pe_no);
 461                 return -EIO;
 462         }
 463
 464         if (!(pe->flags & PNV_IODA_PE_MASTER))
 465                 return 0;
 466
 467         /* Clear frozen state for slave PEs */
 468         list_for_each_entry(slave, &pe->slaves, list) {
 469                 rc = opal_pci_eeh_freeze_clear(phb->opal_id,
 470                                              slave->pe_number,
 471                                              opt);
 472                 if (rc != OPAL_SUCCESS) {
 473                         pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
 474                                 __func__, rc, opt, phb->hose->global_number,
 475                                 slave->pe_number);
 476                         return -EIO;
 477                 }
 478         }
 479
 480         return 0;
 481 }
 482
 483 static int pnv_ioda_get_pe_state(struct pnv_phb *phb, int pe_no)
 484 {
 485         struct pnv_ioda_pe *slave, *pe;
 486         u8 fstate, state;
 487         __be16 pcierr;
 488         s64 rc;
 489
 490         /* Sanity check on PE number */
 491         if (pe_no < 0 || pe_no >= phb->ioda.total_pe)
 492                 return OPAL_EEH_STOPPED_PERM_UNAVAIL;
 493
 494         /*
 495          * Fetch the master PE and the PE instance might be
 496          * not initialized yet.
 497          */
 498         pe = &phb->ioda.pe_array[pe_no];
 499         if (pe->flags & PNV_IODA_PE_SLAVE) {
 500                 pe = pe->master;
 501                 WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
 502                 pe_no = pe->pe_number;
 503         }
 504
 505         /* Check the master PE */
 506         rc = opal_pci_eeh_freeze_status(phb->opal_id, pe_no,
 507                                         &state, &pcierr, NULL);
 508         if (rc != OPAL_SUCCESS) {
 509                 pr_warn("%s: Failure %lld getting "
 510                         "PHB#%x-PE#%x state\n",
 511                         __func__, rc,
 512                         phb->hose->global_number, pe_no);
 513                 return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
 514         }
 515
 516         /* Check the slave PE */
 517         if (!(pe->flags & PNV_IODA_PE_MASTER))
 518                 return state;
 519
 520         list_for_each_entry(slave, &pe->slaves, list) {
 521                 rc = opal_pci_eeh_freeze_status(phb->opal_id,
 522                                                 slave->pe_number,
 523                                                 &fstate,
 524                                                 &pcierr,
 525                                                 NULL);
 526                 if (rc != OPAL_SUCCESS) {
 527                         pr_warn("%s: Failure %lld getting "
 528                                 "PHB#%x-PE#%x state\n",
 529                                 __func__, rc,
 530                                 phb->hose->global_number, slave->pe_number);
 531                         return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
 532                 }
 533
 534                 /*
 535                  * Override the result based on the ascending
 536                  * priority.
 537                  */
 538                 if (fstate > state)
 539                         state = fstate;
 540         }
 541
 542         return state;
 543 }
 544
 545 /* Currently those 2 are only used when MSIs are enabled, this will change
 546  * but in the meantime, we need to protect them to avoid warnings
 547  */
 548 #ifdef CONFIG_PCI_MSI
 549 static struct pnv_ioda_pe *pnv_ioda_get_pe(struct pci_dev *dev)
 550 {
 551         struct pci_controller *hose = pci_bus_to_host(dev->bus);
 552         struct pnv_phb *phb = hose->private_data;
 553         struct pci_dn *pdn = pci_get_pdn(dev);
 554
 555         if (!pdn)
 556                 return NULL;
 557         if (pdn->pe_number == IODA_INVALID_PE)
 558                 return NULL;
 559         return &phb->ioda.pe_array[pdn->pe_number];
 560 }
 561 #endif /* CONFIG_PCI_MSI */
 562
 563 static int pnv_ioda_set_one_peltv(struct pnv_phb *phb,
 564                                   struct pnv_ioda_pe *parent,
 565                                   struct pnv_ioda_pe *child,
 566                                   bool is_add)
 567 {
 568         const char *desc = is_add ? "adding" : "removing";
 569         uint8_t op = is_add ? OPAL_ADD_PE_TO_DOMAIN :
 570                               OPAL_REMOVE_PE_FROM_DOMAIN;
 571         struct pnv_ioda_pe *slave;
 572         long rc;
 573
 574         /* Parent PE affects child PE */
 575         rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
 576                                 child->pe_number, op);
 577         if (rc != OPAL_SUCCESS) {
 578                 pe_warn(child, "OPAL error %ld %s to parent PELTV\n",
 579                         rc, desc);
 580                 return -ENXIO;
 581         }
 582
 583         if (!(child->flags & PNV_IODA_PE_MASTER))
 584                 return 0;
 585
 586         /* Compound case: parent PE affects slave PEs */
 587         list_for_each_entry(slave, &child->slaves, list) {
 588                 rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
 589                                         slave->pe_number, op);
 590                 if (rc != OPAL_SUCCESS) {
 591                         pe_warn(slave, "OPAL error %ld %s to parent PELTV\n",
 592                                 rc, desc);
 593                         return -ENXIO;
 594                 }
 595         }
 596
 597         return 0;
 598 }
 599
 600 static int pnv_ioda_set_peltv(struct pnv_phb *phb,
 601                               struct pnv_ioda_pe *pe,
 602                               bool is_add)
 603 {
 604         struct pnv_ioda_pe *slave;
 605         struct pci_dev *pdev = NULL;
 606         int ret;
 607
 608         /*
 609          * Clear PE frozen state. If it's master PE, we need
 610          * clear slave PE frozen state as well.
 611          */
 612         if (is_add) {
 613                 opal_pci_eeh_freeze_clear(phb->opal_id, pe->pe_number,
 614                                           OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 615                 if (pe->flags & PNV_IODA_PE_MASTER) {
 616                         list_for_each_entry(slave, &pe->slaves, list)
 617                                 opal_pci_eeh_freeze_clear(phb->opal_id,
 618                                                           slave->pe_number,
 619                                                           OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 620                 }
 621         }
 622
 623         /*
 624          * Associate PE in PELT. We need add the PE into the
 625          * corresponding PELT-V as well. Otherwise, the error
 626          * originated from the PE might contribute to other
 627          * PEs.
 628          */
 629         ret = pnv_ioda_set_one_peltv(phb, pe, pe, is_add);
 630         if (ret)
 631                 return ret;
 632
 633         /* For compound PEs, any one affects all of them */
 634         if (pe->flags & PNV_IODA_PE_MASTER) {
 635                 list_for_each_entry(slave, &pe->slaves, list) {
 636                         ret = pnv_ioda_set_one_peltv(phb, slave, pe, is_add);
 637                         if (ret)
 638                                 return ret;
 639                 }
 640         }
 641
 642         if (pe->flags & (PNV_IODA_PE_BUS_ALL | PNV_IODA_PE_BUS))
 643                 pdev = pe->pbus->self;
 644         else if (pe->flags & PNV_IODA_PE_DEV)
 645                 pdev = pe->pdev->bus->self;
 646 #ifdef CONFIG_PCI_IOV
 647         else if (pe->flags & PNV_IODA_PE_VF)
 648                 pdev = pe->parent_dev->bus->self;
 649 #endif /* CONFIG_PCI_IOV */
 650         while (pdev) {
 651                 struct pci_dn *pdn = pci_get_pdn(pdev);
 652                 struct pnv_ioda_pe *parent;
 653
 654                 if (pdn && pdn->pe_number != IODA_INVALID_PE) {
 655                         parent = &phb->ioda.pe_array[pdn->pe_number];
 656                         ret = pnv_ioda_set_one_peltv(phb, parent, pe, is_add);
 657                         if (ret)
 658                                 return ret;
 659                 }
 660
 661                 pdev = pdev->bus->self;
 662         }
 663
 664         return 0;
 665 }
 666
 667 #ifdef CONFIG_PCI_IOV
 668 static int pnv_ioda_deconfigure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
 669 {
 670         struct pci_dev *parent;
 671         uint8_t bcomp, dcomp, fcomp;
 672         int64_t rc;
 673         long rid_end, rid;
 674
 675         /* Currently, we just deconfigure VF PE. Bus PE will always there.*/
 676         if (pe->pbus) {
 677                 int count;
 678
 679                 dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
 680                 fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
 681                 parent = pe->pbus->self;
 682                 if (pe->flags & PNV_IODA_PE_BUS_ALL)
 683                         count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1;
 684                 else
 685                         count = 1;
 686
 687                 switch(count) {
 688                 case  1: bcomp = OpalPciBusAll;         break;
 689                 case  2: bcomp = OpalPciBus7Bits;       break;
 690                 case  4: bcomp = OpalPciBus6Bits;       break;
 691                 case  8: bcomp = OpalPciBus5Bits;       break;
 692                 case 16: bcomp = OpalPciBus4Bits;       break;
 693                 case 32: bcomp = OpalPciBus3Bits;       break;
 694                 default:
 695                         dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
 696                                 count);
 697                         /* Do an exact match only */
 698                         bcomp = OpalPciBusAll;
 699                 }
 700                 rid_end = pe->rid + (count << 8);
 701         } else {
 702                 if (pe->flags & PNV_IODA_PE_VF)
 703                         parent = pe->parent_dev;
 704                 else
 705                         parent = pe->pdev->bus->self;
 706                 bcomp = OpalPciBusAll;
 707                 dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
 708                 fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
 709                 rid_end = pe->rid + 1;
 710         }
 711
 712         /* Clear the reverse map */
 713         for (rid = pe->rid; rid < rid_end; rid++)
 714                 phb->ioda.pe_rmap[rid] = 0;
 715
 716         /* Release from all parents PELT-V */
 717         while (parent) {
 718                 struct pci_dn *pdn = pci_get_pdn(parent);
 719                 if (pdn && pdn->pe_number != IODA_INVALID_PE) {
 720                         rc = opal_pci_set_peltv(phb->opal_id, pdn->pe_number,
 721                                                 pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
 722                         /* XXX What to do in case of error ? */
 723                 }
 724                 parent = parent->bus->self;
 725         }
 726
 727         opal_pci_eeh_freeze_set(phb->opal_id, pe->pe_number,
 728                                   OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 729
 730         /* Disassociate PE in PELT */
 731         rc = opal_pci_set_peltv(phb->opal_id, pe->pe_number,
 732                                 pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
 733         if (rc)
 734                 pe_warn(pe, "OPAL error %ld remove self from PELTV\n", rc);
 735         rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
 736                              bcomp, dcomp, fcomp, OPAL_UNMAP_PE);
 737         if (rc)
 738                 pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
 739
 740         pe->pbus = NULL;
 741         pe->pdev = NULL;
 742         pe->parent_dev = NULL;
 743
 744         return 0;
 745 }
 746 #endif /* CONFIG_PCI_IOV */
 747
 748 static int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
 749 {
 750         struct pci_dev *parent;
 751         uint8_t bcomp, dcomp, fcomp;
 752         long rc, rid_end, rid;
 753
 754         /* Bus validation ? */
 755         if (pe->pbus) {
 756                 int count;
 757
 758                 dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
 759                 fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
 760                 parent = pe->pbus->self;
 761                 if (pe->flags & PNV_IODA_PE_BUS_ALL)
 762                         count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1;
 763                 else
 764                         count = 1;
 765
 766                 switch(count) {
 767                 case  1: bcomp = OpalPciBusAll;         break;
 768                 case  2: bcomp = OpalPciBus7Bits;       break;
 769                 case  4: bcomp = OpalPciBus6Bits;       break;
 770                 case  8: bcomp = OpalPciBus5Bits;       break;
 771                 case 16: bcomp = OpalPciBus4Bits;       break;
 772                 case 32: bcomp = OpalPciBus3Bits;       break;
 773                 default:
 774                         dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
 775                                 count);
 776                         /* Do an exact match only */
 777                         bcomp = OpalPciBusAll;
 778                 }
 779                 rid_end = pe->rid + (count << 8);
 780         } else {
 781 #ifdef CONFIG_PCI_IOV
 782                 if (pe->flags & PNV_IODA_PE_VF)
 783                         parent = pe->parent_dev;
 784                 else
 785 #endif /* CONFIG_PCI_IOV */
 786                         parent = pe->pdev->bus->self;
 787                 bcomp = OpalPciBusAll;
 788                 dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
 789                 fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
 790                 rid_end = pe->rid + 1;
 791         }
 792
 793         /*
 794          * Associate PE in PELT. We need add the PE into the
 795          * corresponding PELT-V as well. Otherwise, the error
 796          * originated from the PE might contribute to other
 797          * PEs.
 798          */
 799         rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
 800                              bcomp, dcomp, fcomp, OPAL_MAP_PE);
 801         if (rc) {
 802                 pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
 803                 return -ENXIO;
 804         }
 805
 806         /* Configure PELTV */
 807         pnv_ioda_set_peltv(phb, pe, true);
 808
 809         /* Setup reverse map */
 810         for (rid = pe->rid; rid < rid_end; rid++)
 811                 phb->ioda.pe_rmap[rid] = pe->pe_number;
 812
 813         /* Setup one MVTs on IODA1 */
 814         if (phb->type != PNV_PHB_IODA1) {
 815                 pe->mve_number = 0;
 816                 goto out;
 817         }
 818
 819         pe->mve_number = pe->pe_number;
 820         rc = opal_pci_set_mve(phb->opal_id, pe->mve_number, pe->pe_number);
 821         if (rc != OPAL_SUCCESS) {
 822                 pe_err(pe, "OPAL error %ld setting up MVE %d\n",
 823                        rc, pe->mve_number);
 824                 pe->mve_number = -1;
 825         } else {
 826                 rc = opal_pci_set_mve_enable(phb->opal_id,
 827                                              pe->mve_number, OPAL_ENABLE_MVE);
 828                 if (rc) {
 829                         pe_err(pe, "OPAL error %ld enabling MVE %d\n",
 830                                rc, pe->mve_number);
 831                         pe->mve_number = -1;
 832                 }
 833         }
 834
 835 out:
 836         return 0;
 837 }
 838
 839 static void pnv_ioda_link_pe_by_weight(struct pnv_phb *phb,
 840                                        struct pnv_ioda_pe *pe)
 841 {
 842         struct pnv_ioda_pe *lpe;
 843
 844         list_for_each_entry(lpe, &phb->ioda.pe_dma_list, dma_link) {
 845                 if (lpe->dma_weight < pe->dma_weight) {
 846                         list_add_tail(&pe->dma_link, &lpe->dma_link);
 847                         return;
 848                 }
 849         }
 850         list_add_tail(&pe->dma_link, &phb->ioda.pe_dma_list);
 851 }
 852
 853 static unsigned int pnv_ioda_dma_weight(struct pci_dev *dev)
 854 {
 855         /* This is quite simplistic. The "base" weight of a device
 856          * is 10. 0 means no DMA is to be accounted for it.
 857          */
 858
 859         /* If it's a bridge, no DMA */
 860         if (dev->hdr_type != PCI_HEADER_TYPE_NORMAL)
 861                 return 0;
 862
 863         /* Reduce the weight of slow USB controllers */
 864         if (dev->class == PCI_CLASS_SERIAL_USB_UHCI ||
 865             dev->class == PCI_CLASS_SERIAL_USB_OHCI ||
 866             dev->class == PCI_CLASS_SERIAL_USB_EHCI)
 867                 return 3;
 868
 869         /* Increase the weight of RAID (includes Obsidian) */
 870         if ((dev->class >> 8) == PCI_CLASS_STORAGE_RAID)
 871                 return 15;
 872
 873         /* Default */
 874         return 10;
 875 }
 876
 877 #ifdef CONFIG_PCI_IOV
 878 static int pnv_pci_vf_resource_shift(struct pci_dev *dev, int offset)
 879 {
 880         struct pci_dn *pdn = pci_get_pdn(dev);
 881         int i;
 882         struct resource *res, res2;
 883         resource_size_t size;
 884         u16 num_vfs;
 885
 886         if (!dev->is_physfn)
 887                 return -EINVAL;
 888
 889         /*
 890          * "offset" is in VFs.  The M64 windows are sized so that when they
 891          * are segmented, each segment is the same size as the IOV BAR.
 892          * Each segment is in a separate PE, and the high order bits of the
 893          * address are the PE number.  Therefore, each VF's BAR is in a
 894          * separate PE, and changing the IOV BAR start address changes the
 895          * range of PEs the VFs are in.
 896          */
 897         num_vfs = pdn->num_vfs;
 898         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
 899                 res = &dev->resource[i + PCI_IOV_RESOURCES];
 900                 if (!res->flags || !res->parent)
 901                         continue;
 902
 903                 if (!pnv_pci_is_mem_pref_64(res->flags))
 904                         continue;
 905
 906                 /*
 907                  * The actual IOV BAR range is determined by the start address
 908                  * and the actual size for num_vfs VFs BAR.  This check is to
 909                  * make sure that after shifting, the range will not overlap
 910                  * with another device.
 911                  */
 912                 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
 913                 res2.flags = res->flags;
 914                 res2.start = res->start + (size * offset);
 915                 res2.end = res2.start + (size * num_vfs) - 1;
 916
 917                 if (res2.end > res->end) {
 918                         dev_err(&dev->dev, "VF BAR%d: %pR would extend past %pR (trying to enable %d VFs shifted by %d)\n",
 919                                 i, &res2, res, num_vfs, offset);
 920                         return -EBUSY;
 921                 }
 922         }
 923
 924         /*
 925          * After doing so, there would be a "hole" in the /proc/iomem when
 926          * offset is a positive value. It looks like the device return some
 927          * mmio back to the system, which actually no one could use it.
 928          */
 929         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
 930                 res = &dev->resource[i + PCI_IOV_RESOURCES];
 931                 if (!res->flags || !res->parent)
 932                         continue;
 933
 934                 if (!pnv_pci_is_mem_pref_64(res->flags))
 935                         continue;
 936
 937                 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
 938                 res2 = *res;
 939                 res->start += size * offset;
 940
 941                 dev_info(&dev->dev, "VF BAR%d: %pR shifted to %pR (enabling %d VFs shifted by %d)\n",
 942                          i, &res2, res, num_vfs, offset);
 943                 pci_update_resource(dev, i + PCI_IOV_RESOURCES);
 944         }
 945         return 0;
 946 }
 947 #endif /* CONFIG_PCI_IOV */
 948
 949 #if 0
 950 static struct pnv_ioda_pe *pnv_ioda_setup_dev_PE(struct pci_dev *dev)
 951 {
 952         struct pci_controller *hose = pci_bus_to_host(dev->bus);
 953         struct pnv_phb *phb = hose->private_data;
 954         struct pci_dn *pdn = pci_get_pdn(dev);
 955         struct pnv_ioda_pe *pe;
 956         int pe_num;
 957
 958         if (!pdn) {
 959                 pr_err("%s: Device tree node not associated properly\n",
 960                            pci_name(dev));
 961                 return NULL;
 962         }
 963         if (pdn->pe_number != IODA_INVALID_PE)
 964                 return NULL;
 965
 966         /* PE#0 has been pre-set */
 967         if (dev->bus->number == 0)
 968                 pe_num = 0;
 969         else
 970                 pe_num = pnv_ioda_alloc_pe(phb);
 971         if (pe_num == IODA_INVALID_PE) {
 972                 pr_warning("%s: Not enough PE# available, disabling device\n",
 973                            pci_name(dev));
 974                 return NULL;
 975         }
 976
 977         /* NOTE: We get only one ref to the pci_dev for the pdn, not for the
 978          * pointer in the PE data structure, both should be destroyed at the
 979          * same time. However, this needs to be looked at more closely again
 980          * once we actually start removing things (Hotplug, SR-IOV, ...)
 981          *
 982          * At some point we want to remove the PDN completely anyways
 983          */
 984         pe = &phb->ioda.pe_array[pe_num];
 985         pci_dev_get(dev);
 986         pdn->pcidev = dev;
 987         pdn->pe_number = pe_num;
 988         pe->pdev = dev;
 989         pe->pbus = NULL;
 990         pe->tce32_seg = -1;
 991         pe->mve_number = -1;
 992         pe->rid = dev->bus->number << 8 | pdn->devfn;
 993
 994         pe_info(pe, "Associated device to PE\n");
 995
 996         if (pnv_ioda_configure_pe(phb, pe)) {
 997                 /* XXX What do we do here ? */
 998                 if (pe_num)
 999                         pnv_ioda_free_pe(phb, pe_num);
1000                 pdn->pe_number = IODA_INVALID_PE;
1001                 pe->pdev = NULL;
1002                 pci_dev_put(dev);
1003                 return NULL;
1004         }
1005
1006         /* Assign a DMA weight to the device */
1007         pe->dma_weight = pnv_ioda_dma_weight(dev);
1008         if (pe->dma_weight != 0) {
1009                 phb->ioda.dma_weight += pe->dma_weight;
1010                 phb->ioda.dma_pe_count++;
1011         }
1012
1013         /* Link the PE */
1014         pnv_ioda_link_pe_by_weight(phb, pe);
1015
1016         return pe;
1017 }
1018 #endif /* Useful for SRIOV case */
1019
1020 static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe)
1021 {
1022         struct pci_dev *dev;
1023
1024         list_for_each_entry(dev, &bus->devices, bus_list) {
1025                 struct pci_dn *pdn = pci_get_pdn(dev);
1026
1027                 if (pdn == NULL) {
1028                         pr_warn("%s: No device node associated with device !\n",
1029                                 pci_name(dev));
1030                         continue;
1031                 }
1032                 pdn->pe_number = pe->pe_number;
1033                 pe->dma_weight += pnv_ioda_dma_weight(dev);
1034                 if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
1035                         pnv_ioda_setup_same_PE(dev->subordinate, pe);
1036         }
1037 }
1038
1039 /*
1040  * There're 2 types of PCI bus sensitive PEs: One that is compromised of
1041  * single PCI bus. Another one that contains the primary PCI bus and its
1042  * subordinate PCI devices and buses. The second type of PE is normally
1043  * orgiriated by PCIe-to-PCI bridge or PLX switch downstream ports.
1044  */
1045 static void pnv_ioda_setup_bus_PE(struct pci_bus *bus, int all)
1046 {
1047         struct pci_controller *hose = pci_bus_to_host(bus);
1048         struct pnv_phb *phb = hose->private_data;
1049         struct pnv_ioda_pe *pe;
1050         int pe_num = IODA_INVALID_PE;
1051
1052         /* Check if PE is determined by M64 */
1053         if (phb->pick_m64_pe)
1054                 pe_num = phb->pick_m64_pe(phb, bus, all);
1055
1056         /* The PE number isn't pinned by M64 */
1057         if (pe_num == IODA_INVALID_PE)
1058                 pe_num = pnv_ioda_alloc_pe(phb);
1059
1060         if (pe_num == IODA_INVALID_PE) {
1061                 pr_warning("%s: Not enough PE# available for PCI bus %04x:%02x\n",
1062                         __func__, pci_domain_nr(bus), bus->number);
1063                 return;
1064         }
1065
1066         pe = &phb->ioda.pe_array[pe_num];
1067         pe->flags |= (all ? PNV_IODA_PE_BUS_ALL : PNV_IODA_PE_BUS);
1068         pe->pbus = bus;
1069         pe->pdev = NULL;
1070         pe->tce32_seg = -1;
1071         pe->mve_number = -1;
1072         pe->rid = bus->busn_res.start << 8;
1073         pe->dma_weight = 0;
1074
1075         if (all)
1076                 pe_info(pe, "Secondary bus %d..%d associated with PE#%d\n",
1077                         bus->busn_res.start, bus->busn_res.end, pe_num);
1078         else
1079                 pe_info(pe, "Secondary bus %d associated with PE#%d\n",
1080                         bus->busn_res.start, pe_num);
1081
1082         if (pnv_ioda_configure_pe(phb, pe)) {
1083                 /* XXX What do we do here ? */
1084                 if (pe_num)
1085                         pnv_ioda_free_pe(phb, pe_num);
1086                 pe->pbus = NULL;
1087                 return;
1088         }
1089
1090         pe->tce32_table = kzalloc_node(sizeof(struct iommu_table),
1091                         GFP_KERNEL, hose->node);
1092         pe->tce32_table->data = pe;
1093
1094         /* Associate it with all child devices */
1095         pnv_ioda_setup_same_PE(bus, pe);
1096
1097         /* Put PE to the list */
1098         list_add_tail(&pe->list, &phb->ioda.pe_list);
1099
1100         /* Account for one DMA PE if at least one DMA capable device exist
1101          * below the bridge
1102          */
1103         if (pe->dma_weight != 0) {
1104                 phb->ioda.dma_weight += pe->dma_weight;
1105                 phb->ioda.dma_pe_count++;
1106         }
1107
1108         /* Link the PE */
1109         pnv_ioda_link_pe_by_weight(phb, pe);
1110 }
1111
1112 static void pnv_ioda_setup_PEs(struct pci_bus *bus)
1113 {
1114         struct pci_dev *dev;
1115
1116         pnv_ioda_setup_bus_PE(bus, 0);
1117
1118         list_for_each_entry(dev, &bus->devices, bus_list) {
1119                 if (dev->subordinate) {
1120                         if (pci_pcie_type(dev) == PCI_EXP_TYPE_PCI_BRIDGE)
1121                                 pnv_ioda_setup_bus_PE(dev->subordinate, 1);
1122                         else
1123                                 pnv_ioda_setup_PEs(dev->subordinate);
1124                 }
1125         }
1126 }
1127
1128 /*
1129  * Configure PEs so that the downstream PCI buses and devices
1130  * could have their associated PE#. Unfortunately, we didn't
1131  * figure out the way to identify the PLX bridge yet. So we
1132  * simply put the PCI bus and the subordinate behind the root
1133  * port to PE# here. The game rule here is expected to be changed
1134  * as soon as we can detected PLX bridge correctly.
1135  */
1136 static void pnv_pci_ioda_setup_PEs(void)
1137 {
1138         struct pci_controller *hose, *tmp;
1139         struct pnv_phb *phb;
1140
1141         list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
1142                 phb = hose->private_data;
1143
1144                 /* M64 layout might affect PE allocation */
1145                 if (phb->reserve_m64_pe)
1146                         phb->reserve_m64_pe(phb);
1147
1148                 pnv_ioda_setup_PEs(hose->bus);
1149         }
1150 }
1151
1152 #ifdef CONFIG_PCI_IOV
1153 static int pnv_pci_vf_release_m64(struct pci_dev *pdev)
1154 {
1155         struct pci_bus        *bus;
1156         struct pci_controller *hose;
1157         struct pnv_phb        *phb;
1158         struct pci_dn         *pdn;
1159         int                    i, j;
1160
1161         bus = pdev->bus;
1162         hose = pci_bus_to_host(bus);
1163         phb = hose->private_data;
1164         pdn = pci_get_pdn(pdev);
1165
1166         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
1167                 for (j = 0; j < M64_PER_IOV; j++) {
1168                         if (pdn->m64_wins[i][j] == IODA_INVALID_M64)
1169                                 continue;
1170                         opal_pci_phb_mmio_enable(phb->opal_id,
1171                                 OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 0);
1172                         clear_bit(pdn->m64_wins[i][j], &phb->ioda.m64_bar_alloc);
1173                         pdn->m64_wins[i][j] = IODA_INVALID_M64;
1174                 }
1175
1176         return 0;
1177 }
1178
1179 static int pnv_pci_vf_assign_m64(struct pci_dev *pdev, u16 num_vfs)
1180 {
1181         struct pci_bus        *bus;
1182         struct pci_controller *hose;
1183         struct pnv_phb        *phb;
1184         struct pci_dn         *pdn;
1185         unsigned int           win;
1186         struct resource       *res;
1187         int                    i, j;
1188         int64_t                rc;
1189         int                    total_vfs;
1190         resource_size_t        size, start;
1191         int                    pe_num;
1192         int                    vf_groups;
1193         int                    vf_per_group;
1194
1195         bus = pdev->bus;
1196         hose = pci_bus_to_host(bus);
1197         phb = hose->private_data;
1198         pdn = pci_get_pdn(pdev);
1199         total_vfs = pci_sriov_get_totalvfs(pdev);
1200
1201         /* Initialize the m64_wins to IODA_INVALID_M64 */
1202         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
1203                 for (j = 0; j < M64_PER_IOV; j++)
1204                         pdn->m64_wins[i][j] = IODA_INVALID_M64;
1205
1206         if (pdn->m64_per_iov == M64_PER_IOV) {
1207                 vf_groups = (num_vfs <= M64_PER_IOV) ? num_vfs: M64_PER_IOV;
1208                 vf_per_group = (num_vfs <= M64_PER_IOV)? 1:
1209                         roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1210         } else {
1211                 vf_groups = 1;
1212                 vf_per_group = 1;
1213         }
1214
1215         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
1216                 res = &pdev->resource[i + PCI_IOV_RESOURCES];
1217                 if (!res->flags || !res->parent)
1218                         continue;
1219
1220                 if (!pnv_pci_is_mem_pref_64(res->flags))
1221                         continue;
1222
1223                 for (j = 0; j < vf_groups; j++) {
1224                         do {
1225                                 win = find_next_zero_bit(&phb->ioda.m64_bar_alloc,
1226                                                 phb->ioda.m64_bar_idx + 1, 0);
1227
1228                                 if (win >= phb->ioda.m64_bar_idx + 1)
1229                                         goto m64_failed;
1230                         } while (test_and_set_bit(win, &phb->ioda.m64_bar_alloc));
1231
1232                         pdn->m64_wins[i][j] = win;
1233
1234                         if (pdn->m64_per_iov == M64_PER_IOV) {
1235                                 size = pci_iov_resource_size(pdev,
1236                                                         PCI_IOV_RESOURCES + i);
1237                                 size = size * vf_per_group;
1238                                 start = res->start + size * j;
1239                         } else {
1240                                 size = resource_size(res);
1241                                 start = res->start;
1242                         }
1243
1244                         /* Map the M64 here */
1245                         if (pdn->m64_per_iov == M64_PER_IOV) {
1246                                 pe_num = pdn->offset + j;
1247                                 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
1248                                                 pe_num, OPAL_M64_WINDOW_TYPE,
1249                                                 pdn->m64_wins[i][j], 0);
1250                         }
1251
1252                         rc = opal_pci_set_phb_mem_window(phb->opal_id,
1253                                                  OPAL_M64_WINDOW_TYPE,
1254                                                  pdn->m64_wins[i][j],
1255                                                  start,
1256                                                  0, /* unused */
1257                                                  size);
1258
1259
1260                         if (rc != OPAL_SUCCESS) {
1261                                 dev_err(&pdev->dev, "Failed to map M64 window #%d: %lld\n",
1262                                         win, rc);
1263                                 goto m64_failed;
1264                         }
1265
1266                         if (pdn->m64_per_iov == M64_PER_IOV)
1267                                 rc = opal_pci_phb_mmio_enable(phb->opal_id,
1268                                      OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 2);
1269                         else
1270                                 rc = opal_pci_phb_mmio_enable(phb->opal_id,
1271                                      OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 1);
1272
1273                         if (rc != OPAL_SUCCESS) {
1274                                 dev_err(&pdev->dev, "Failed to enable M64 window #%d: %llx\n",
1275                                         win, rc);
1276                                 goto m64_failed;
1277                         }
1278                 }
1279         }
1280         return 0;
1281
1282 m64_failed:
1283         pnv_pci_vf_release_m64(pdev);
1284         return -EBUSY;
1285 }
1286
1287 static void pnv_pci_ioda2_release_dma_pe(struct pci_dev *dev, struct pnv_ioda_pe *pe)
1288 {
1289         struct pci_bus        *bus;
1290         struct pci_controller *hose;
1291         struct pnv_phb        *phb;
1292         struct iommu_table    *tbl;
1293         unsigned long         addr;
1294         int64_t               rc;
1295
1296         bus = dev->bus;
1297         hose = pci_bus_to_host(bus);
1298         phb = hose->private_data;
1299         tbl = pe->tce32_table;
1300         addr = tbl->it_base;
1301
1302         opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number,
1303                                    pe->pe_number << 1, 1, __pa(addr),
1304                                    0, 0x1000);
1305
1306         rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
1307                                         pe->pe_number,
1308                                         (pe->pe_number << 1) + 1,
1309                                         pe->tce_bypass_base,
1310                                         0);
1311         if (rc)
1312                 pe_warn(pe, "OPAL error %ld release DMA window\n", rc);
1313
1314         if (tbl->it_group) {
1315                 iommu_group_put(tbl->it_group);
1316                 BUG_ON(tbl->it_group);
1317         }
1318         iommu_free_table(tbl, of_node_full_name(dev->dev.of_node));
1319         free_pages(addr, get_order(TCE32_TABLE_SIZE));
1320         pe->tce32_table = NULL;
1321 }
1322
1323 static void pnv_ioda_release_vf_PE(struct pci_dev *pdev, u16 num_vfs)
1324 {
1325         struct pci_bus        *bus;
1326         struct pci_controller *hose;
1327         struct pnv_phb        *phb;
1328         struct pnv_ioda_pe    *pe, *pe_n;
1329         struct pci_dn         *pdn;
1330         u16                    vf_index;
1331         int64_t                rc;
1332
1333         bus = pdev->bus;
1334         hose = pci_bus_to_host(bus);
1335         phb = hose->private_data;
1336         pdn = pci_get_pdn(pdev);
1337
1338         if (!pdev->is_physfn)
1339                 return;
1340
1341         if (pdn->m64_per_iov == M64_PER_IOV && num_vfs > M64_PER_IOV) {
1342                 int   vf_group;
1343                 int   vf_per_group;
1344                 int   vf_index1;
1345
1346                 vf_per_group = roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1347
1348                 for (vf_group = 0; vf_group < M64_PER_IOV; vf_group++)
1349                         for (vf_index = vf_group * vf_per_group;
1350                                 vf_index < (vf_group + 1) * vf_per_group &&
1351                                 vf_index < num_vfs;
1352                                 vf_index++)
1353                                 for (vf_index1 = vf_group * vf_per_group;
1354                                         vf_index1 < (vf_group + 1) * vf_per_group &&
1355                                         vf_index1 < num_vfs;
1356                                         vf_index1++){
1357
1358                                         rc = opal_pci_set_peltv(phb->opal_id,
1359                                                 pdn->offset + vf_index,
1360                                                 pdn->offset + vf_index1,
1361                                                 OPAL_REMOVE_PE_FROM_DOMAIN);
1362
1363                                         if (rc)
1364                                             dev_warn(&pdev->dev, "%s: Failed to unlink same group PE#%d(%lld)\n",
1365                                                 __func__,
1366                                                 pdn->offset + vf_index1, rc);
1367                                 }
1368         }
1369
1370         list_for_each_entry_safe(pe, pe_n, &phb->ioda.pe_list, list) {
1371                 if (pe->parent_dev != pdev)
1372                         continue;
1373
1374                 pnv_pci_ioda2_release_dma_pe(pdev, pe);
1375
1376                 /* Remove from list */
1377                 mutex_lock(&phb->ioda.pe_list_mutex);
1378                 list_del(&pe->list);
1379                 mutex_unlock(&phb->ioda.pe_list_mutex);
1380
1381                 pnv_ioda_deconfigure_pe(phb, pe);
1382
1383                 pnv_ioda_free_pe(phb, pe->pe_number);
1384         }
1385 }
1386
1387 void pnv_pci_sriov_disable(struct pci_dev *pdev)
1388 {
1389         struct pci_bus        *bus;
1390         struct pci_controller *hose;
1391         struct pnv_phb        *phb;
1392         struct pci_dn         *pdn;
1393         struct pci_sriov      *iov;
1394         u16 num_vfs;
1395
1396         bus = pdev->bus;
1397         hose = pci_bus_to_host(bus);
1398         phb = hose->private_data;
1399         pdn = pci_get_pdn(pdev);
1400         iov = pdev->sriov;
1401         num_vfs = pdn->num_vfs;
1402
1403         /* Release VF PEs */
1404         pnv_ioda_release_vf_PE(pdev, num_vfs);
1405
1406         if (phb->type == PNV_PHB_IODA2) {
1407                 if (pdn->m64_per_iov == 1)
1408                         pnv_pci_vf_resource_shift(pdev, -pdn->offset);
1409
1410                 /* Release M64 windows */
1411                 pnv_pci_vf_release_m64(pdev);
1412
1413                 /* Release PE numbers */
1414                 bitmap_clear(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1415                 pdn->offset = 0;
1416         }
1417 }
1418
1419 static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
1420                                        struct pnv_ioda_pe *pe);
1421 static void pnv_ioda_setup_vf_PE(struct pci_dev *pdev, u16 num_vfs)
1422 {
1423         struct pci_bus        *bus;
1424         struct pci_controller *hose;
1425         struct pnv_phb        *phb;
1426         struct pnv_ioda_pe    *pe;
1427         int                    pe_num;
1428         u16                    vf_index;
1429         struct pci_dn         *pdn;
1430         int64_t                rc;
1431
1432         bus = pdev->bus;
1433         hose = pci_bus_to_host(bus);
1434         phb = hose->private_data;
1435         pdn = pci_get_pdn(pdev);
1436
1437         if (!pdev->is_physfn)
1438                 return;
1439
1440         /* Reserve PE for each VF */
1441         for (vf_index = 0; vf_index < num_vfs; vf_index++) {
1442                 pe_num = pdn->offset + vf_index;
1443
1444                 pe = &phb->ioda.pe_array[pe_num];
1445                 pe->pe_number = pe_num;
1446                 pe->phb = phb;
1447                 pe->flags = PNV_IODA_PE_VF;
1448                 pe->pbus = NULL;
1449                 pe->parent_dev = pdev;
1450                 pe->tce32_seg = -1;
1451                 pe->mve_number = -1;
1452                 pe->rid = (pci_iov_virtfn_bus(pdev, vf_index) << 8) |
1453                            pci_iov_virtfn_devfn(pdev, vf_index);
1454
1455                 pe_info(pe, "VF %04d:%02d:%02d.%d associated with PE#%d\n",
1456                         hose->global_number, pdev->bus->number,
1457                         PCI_SLOT(pci_iov_virtfn_devfn(pdev, vf_index)),
1458                         PCI_FUNC(pci_iov_virtfn_devfn(pdev, vf_index)), pe_num);
1459
1460                 if (pnv_ioda_configure_pe(phb, pe)) {
1461                         /* XXX What do we do here ? */
1462                         if (pe_num)
1463                                 pnv_ioda_free_pe(phb, pe_num);
1464                         pe->pdev = NULL;
1465                         continue;
1466                 }
1467
1468                 pe->tce32_table = kzalloc_node(sizeof(struct iommu_table),
1469                                 GFP_KERNEL, hose->node);
1470                 pe->tce32_table->data = pe;
1471
1472                 /* Put PE to the list */
1473                 mutex_lock(&phb->ioda.pe_list_mutex);
1474                 list_add_tail(&pe->list, &phb->ioda.pe_list);
1475                 mutex_unlock(&phb->ioda.pe_list_mutex);
1476
1477                 pnv_pci_ioda2_setup_dma_pe(phb, pe);
1478         }
1479
1480         if (pdn->m64_per_iov == M64_PER_IOV && num_vfs > M64_PER_IOV) {
1481                 int   vf_group;
1482                 int   vf_per_group;
1483                 int   vf_index1;
1484
1485                 vf_per_group = roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1486
1487                 for (vf_group = 0; vf_group < M64_PER_IOV; vf_group++) {
1488                         for (vf_index = vf_group * vf_per_group;
1489                              vf_index < (vf_group + 1) * vf_per_group &&
1490                              vf_index < num_vfs;
1491                              vf_index++) {
1492                                 for (vf_index1 = vf_group * vf_per_group;
1493                                      vf_index1 < (vf_group + 1) * vf_per_group &&
1494                                      vf_index1 < num_vfs;
1495                                      vf_index1++) {
1496
1497                                         rc = opal_pci_set_peltv(phb->opal_id,
1498                                                 pdn->offset + vf_index,
1499                                                 pdn->offset + vf_index1,
1500                                                 OPAL_ADD_PE_TO_DOMAIN);
1501
1502                                         if (rc)
1503                                             dev_warn(&pdev->dev, "%s: Failed to link same group PE#%d(%lld)\n",
1504                                                 __func__,
1505                                                 pdn->offset + vf_index1, rc);
1506                                 }
1507                         }
1508                 }
1509         }
1510 }
1511
1512 int pnv_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
1513 {
1514         struct pci_bus        *bus;
1515         struct pci_controller *hose;
1516         struct pnv_phb        *phb;
1517         struct pci_dn         *pdn;
1518         int                    ret;
1519
1520         bus = pdev->bus;
1521         hose = pci_bus_to_host(bus);
1522         phb = hose->private_data;
1523         pdn = pci_get_pdn(pdev);
1524
1525         if (phb->type == PNV_PHB_IODA2) {
1526                 /* Calculate available PE for required VFs */
1527                 mutex_lock(&phb->ioda.pe_alloc_mutex);
1528                 pdn->offset = bitmap_find_next_zero_area(
1529                         phb->ioda.pe_alloc, phb->ioda.total_pe,
1530                         0, num_vfs, 0);
1531                 if (pdn->offset >= phb->ioda.total_pe) {
1532                         mutex_unlock(&phb->ioda.pe_alloc_mutex);
1533                         dev_info(&pdev->dev, "Failed to enable VF%d\n", num_vfs);
1534                         pdn->offset = 0;
1535                         return -EBUSY;
1536                 }
1537                 bitmap_set(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1538                 pdn->num_vfs = num_vfs;
1539                 mutex_unlock(&phb->ioda.pe_alloc_mutex);
1540
1541                 /* Assign M64 window accordingly */
1542                 ret = pnv_pci_vf_assign_m64(pdev, num_vfs);
1543                 if (ret) {
1544                         dev_info(&pdev->dev, "Not enough M64 window resources\n");
1545                         goto m64_failed;
1546                 }
1547
1548                 /*
1549                  * When using one M64 BAR to map one IOV BAR, we need to shift
1550                  * the IOV BAR according to the PE# allocated to the VFs.
1551                  * Otherwise, the PE# for the VF will conflict with others.
1552                  */
1553                 if (pdn->m64_per_iov == 1) {
1554                         ret = pnv_pci_vf_resource_shift(pdev, pdn->offset);
1555                         if (ret)
1556                                 goto m64_failed;
1557                 }
1558         }
1559
1560         /* Setup VF PEs */
1561         pnv_ioda_setup_vf_PE(pdev, num_vfs);
1562
1563         return 0;
1564
1565 m64_failed:
1566         bitmap_clear(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1567         pdn->offset = 0;
1568
1569         return ret;
1570 }
1571
1572 int pcibios_sriov_disable(struct pci_dev *pdev)
1573 {
1574         pnv_pci_sriov_disable(pdev);
1575
1576         /* Release PCI data */
1577         remove_dev_pci_data(pdev);
1578         return 0;
1579 }
1580
1581 int pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
1582 {
1583         /* Allocate PCI data */
1584         add_dev_pci_data(pdev);
1585
1586         pnv_pci_sriov_enable(pdev, num_vfs);
1587         return 0;
1588 }
1589 #endif /* CONFIG_PCI_IOV */
1590
1591 static void pnv_pci_ioda_dma_dev_setup(struct pnv_phb *phb, struct pci_dev *pdev)
1592 {
1593         struct pci_dn *pdn = pci_get_pdn(pdev);
1594         struct pnv_ioda_pe *pe;
1595
1596         /*
1597          * The function can be called while the PE#
1598          * hasn't been assigned. Do nothing for the
1599          * case.
1600          */
1601         if (!pdn || pdn->pe_number == IODA_INVALID_PE)
1602                 return;
1603
1604         pe = &phb->ioda.pe_array[pdn->pe_number];
1605         WARN_ON(get_dma_ops(&pdev->dev) != &dma_iommu_ops);
1606         set_iommu_table_base(&pdev->dev, pe->tce32_table);
1607         /*
1608          * Note: iommu_add_device() will fail here as
1609          * for physical PE: the device is already added by now;
1610          * for virtual PE: sysfs entries are not ready yet and
1611          * tce_iommu_bus_notifier will add the device to a group later.
1612          */
1613 }
1614
1615 static int pnv_pci_ioda_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
1616 {
1617         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
1618         struct pnv_phb *phb = hose->private_data;
1619         struct pci_dn *pdn = pci_get_pdn(pdev);
1620         struct pnv_ioda_pe *pe;
1621         uint64_t top;
1622         bool bypass = false;
1623
1624         if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
1625                 return -ENODEV;;
1626
1627         pe = &phb->ioda.pe_array[pdn->pe_number];
1628         if (pe->tce_bypass_enabled) {
1629                 top = pe->tce_bypass_base + memblock_end_of_DRAM() - 1;
1630                 bypass = (dma_mask >= top);
1631         }
1632
1633         if (bypass) {
1634                 dev_info(&pdev->dev, "Using 64-bit DMA iommu bypass\n");
1635                 set_dma_ops(&pdev->dev, &dma_direct_ops);
1636                 set_dma_offset(&pdev->dev, pe->tce_bypass_base);
1637         } else {
1638                 dev_info(&pdev->dev, "Using 32-bit DMA via iommu\n");
1639                 set_dma_ops(&pdev->dev, &dma_iommu_ops);
1640                 set_iommu_table_base(&pdev->dev, pe->tce32_table);
1641         }
1642         *pdev->dev.dma_mask = dma_mask;
1643         return 0;
1644 }
1645
1646 static u64 pnv_pci_ioda_dma_get_required_mask(struct pnv_phb *phb,
1647                                               struct pci_dev *pdev)
1648 {
1649         struct pci_dn *pdn = pci_get_pdn(pdev);
1650         struct pnv_ioda_pe *pe;
1651         u64 end, mask;
1652
1653         if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
1654                 return 0;
1655
1656         pe = &phb->ioda.pe_array[pdn->pe_number];
1657         if (!pe->tce_bypass_enabled)
1658                 return __dma_get_required_mask(&pdev->dev);
1659
1660
1661         end = pe->tce_bypass_base + memblock_end_of_DRAM();
1662         mask = 1ULL << (fls64(end) - 1);
1663         mask += mask - 1;
1664
1665         return mask;
1666 }
1667
1668 static void pnv_ioda_setup_bus_dma(struct pnv_ioda_pe *pe,
1669                                    struct pci_bus *bus)
1670 {
1671         struct pci_dev *dev;
1672
1673         list_for_each_entry(dev, &bus->devices, bus_list) {
1674                 set_iommu_table_base(&dev->dev, pe->tce32_table);
1675                 iommu_add_device(&dev->dev);
1676
1677                 if (dev->subordinate)
1678                         pnv_ioda_setup_bus_dma(pe, dev->subordinate);
1679         }
1680 }
1681
1682 static void pnv_pci_ioda1_tce_invalidate(struct pnv_ioda_pe *pe,
1683                                          struct iommu_table *tbl,
1684                                          __be64 *startp, __be64 *endp, bool rm)
1685 {
1686         __be64 __iomem *invalidate = rm ?
1687                 (__be64 __iomem *)pe->tce_inval_reg_phys :
1688                 (__be64 __iomem *)tbl->it_index;
1689         unsigned long start, end, inc;
1690         const unsigned shift = tbl->it_page_shift;
1691
1692         start = __pa(startp);
1693         end = __pa(endp);
1694
1695         /* BML uses this case for p6/p7/galaxy2: Shift addr and put in node */
1696         if (tbl->it_busno) {
1697                 start <<= shift;
1698                 end <<= shift;
1699                 inc = 128ull << shift;
1700                 start |= tbl->it_busno;
1701                 end |= tbl->it_busno;
1702         } else if (tbl->it_type & TCE_PCI_SWINV_PAIR) {
1703                 /* p7ioc-style invalidation, 2 TCEs per write */
1704                 start |= (1ull << 63);
1705                 end |= (1ull << 63);
1706                 inc = 16;
1707         } else {
1708                 /* Default (older HW) */
1709                 inc = 128;
1710         }
1711
1712         end |= inc - 1; /* round up end to be different than start */
1713
1714         mb(); /* Ensure above stores are visible */
1715         while (start <= end) {
1716                 if (rm)
1717                         __raw_rm_writeq(cpu_to_be64(start), invalidate);
1718                 else
1719                         __raw_writeq(cpu_to_be64(start), invalidate);
1720                 start += inc;
1721         }
1722
1723         /*
1724          * The iommu layer will do another mb() for us on build()
1725          * and we don't care on free()
1726          */
1727 }
1728
1729 static struct iommu_table_ops pnv_ioda1_iommu_ops = {
1730         .set = pnv_tce_build,
1731         .clear = pnv_tce_free,
1732         .get = pnv_tce_get,
1733 };
1734
1735 static void pnv_pci_ioda2_tce_invalidate(struct pnv_ioda_pe *pe,
1736                                          struct iommu_table *tbl,
1737                                          __be64 *startp, __be64 *endp, bool rm)
1738 {
1739         unsigned long start, end, inc;
1740         __be64 __iomem *invalidate = rm ?
1741                 (__be64 __iomem *)pe->tce_inval_reg_phys :
1742                 (__be64 __iomem *)tbl->it_index;
1743         const unsigned shift = tbl->it_page_shift;
1744
1745         /* We'll invalidate DMA address in PE scope */
1746         start = 0x2ull << 60;
1747         start |= (pe->pe_number & 0xFF);
1748         end = start;
1749
1750         /* Figure out the start, end and step */
1751         inc = tbl->it_offset + (((u64)startp - tbl->it_base) / sizeof(u64));
1752         start |= (inc << shift);
1753         inc = tbl->it_offset + (((u64)endp - tbl->it_base) / sizeof(u64));
1754         end |= (inc << shift);
1755         inc = (0x1ull << shift);
1756         mb();
1757
1758         while (start <= end) {
1759                 if (rm)
1760                         __raw_rm_writeq(cpu_to_be64(start), invalidate);
1761                 else
1762                         __raw_writeq(cpu_to_be64(start), invalidate);
1763                 start += inc;
1764         }
1765 }
1766
1767 void pnv_pci_ioda_tce_invalidate(struct iommu_table *tbl,
1768                                  __be64 *startp, __be64 *endp, bool rm)
1769 {
1770         struct pnv_ioda_pe *pe = tbl->data;
1771         struct pnv_phb *phb = pe->phb;
1772
1773         if (phb->type == PNV_PHB_IODA1)
1774                 pnv_pci_ioda1_tce_invalidate(pe, tbl, startp, endp, rm);
1775         else
1776                 pnv_pci_ioda2_tce_invalidate(pe, tbl, startp, endp, rm);
1777 }
1778
1779 static struct iommu_table_ops pnv_ioda2_iommu_ops = {
1780         .set = pnv_tce_build,
1781         .clear = pnv_tce_free,
1782         .get = pnv_tce_get,
1783 };
1784
1785 static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb,
1786                                       struct pnv_ioda_pe *pe, unsigned int base,
1787                                       unsigned int segs)
1788 {
1789
1790         struct page *tce_mem = NULL;
1791         const __be64 *swinvp;
1792         struct iommu_table *tbl;
1793         unsigned int i;
1794         int64_t rc;
1795         void *addr;
1796
1797         /* XXX FIXME: Handle 64-bit only DMA devices */
1798         /* XXX FIXME: Provide 64-bit DMA facilities & non-4K TCE tables etc.. */
1799         /* XXX FIXME: Allocate multi-level tables on PHB3 */
1800
1801         /* We shouldn't already have a 32-bit DMA associated */
1802         if (WARN_ON(pe->tce32_seg >= 0))
1803                 return;
1804
1805         tbl = pe->tce32_table;
1806         iommu_register_group(tbl, phb->hose->global_number, pe->pe_number);
1807
1808         /* Grab a 32-bit TCE table */
1809         pe->tce32_seg = base;
1810         pe_info(pe, " Setting up 32-bit TCE table at %08x..%08x\n",
1811                 (base << 28), ((base + segs) << 28) - 1);
1812
1813         /* XXX Currently, we allocate one big contiguous table for the
1814          * TCEs. We only really need one chunk per 256M of TCE space
1815          * (ie per segment) but that's an optimization for later, it
1816          * requires some added smarts with our get/put_tce implementation
1817          */
1818         tce_mem = alloc_pages_node(phb->hose->node, GFP_KERNEL,
1819                                    get_order(TCE32_TABLE_SIZE * segs));
1820         if (!tce_mem) {
1821                 pe_err(pe, " Failed to allocate a 32-bit TCE memory\n");
1822                 goto fail;
1823         }
1824         addr = page_address(tce_mem);
1825         memset(addr, 0, TCE32_TABLE_SIZE * segs);
1826
1827         /* Configure HW */
1828         for (i = 0; i < segs; i++) {
1829                 rc = opal_pci_map_pe_dma_window(phb->opal_id,
1830                                               pe->pe_number,
1831                                               base + i, 1,
1832                                               __pa(addr) + TCE32_TABLE_SIZE * i,
1833                                               TCE32_TABLE_SIZE, 0x1000);
1834                 if (rc) {
1835                         pe_err(pe, " Failed to configure 32-bit TCE table,"
1836                                " err %ld\n", rc);
1837                         goto fail;
1838                 }
1839         }
1840
1841         /* Setup linux iommu table */
1842         pnv_pci_setup_iommu_table(tbl, addr, TCE32_TABLE_SIZE * segs,
1843                                   base << 28, IOMMU_PAGE_SHIFT_4K);
1844
1845         /* OPAL variant of P7IOC SW invalidated TCEs */
1846         swinvp = of_get_property(phb->hose->dn, "ibm,opal-tce-kill", NULL);
1847         if (swinvp) {
1848                 /* We need a couple more fields -- an address and a data
1849                  * to or.  Since the bus is only printed out on table free
1850                  * errors, and on the first pass the data will be a relative
1851                  * bus number, print that out instead.
1852                  */
1853                 pe->tce_inval_reg_phys = be64_to_cpup(swinvp);
1854                 tbl->it_index = (unsigned long)ioremap(pe->tce_inval_reg_phys,
1855                                 8);
1856                 tbl->it_type |= (TCE_PCI_SWINV_CREATE |
1857                                  TCE_PCI_SWINV_FREE   |
1858                                  TCE_PCI_SWINV_PAIR);
1859         }
1860         tbl->it_ops = &pnv_ioda1_iommu_ops;
1861         iommu_init_table(tbl, phb->hose->node);
1862
1863         if (pe->flags & PNV_IODA_PE_DEV) {
1864                 /*
1865                  * Setting table base here only for carrying iommu_group
1866                  * further down to let iommu_add_device() do the job.
1867                  * pnv_pci_ioda_dma_dev_setup will override it later anyway.
1868                  */
1869                 set_iommu_table_base(&pe->pdev->dev, tbl);
1870                 iommu_add_device(&pe->pdev->dev);
1871         } else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
1872                 pnv_ioda_setup_bus_dma(pe, pe->pbus);
1873
1874         return;
1875  fail:
1876         /* XXX Failure: Try to fallback to 64-bit only ? */
1877         if (pe->tce32_seg >= 0)
1878                 pe->tce32_seg = -1;
1879         if (tce_mem)
1880                 __free_pages(tce_mem, get_order(TCE32_TABLE_SIZE * segs));
1881 }
1882
1883 static void pnv_pci_ioda2_set_bypass(struct iommu_table *tbl, bool enable)
1884 {
1885         struct pnv_ioda_pe *pe = tbl->data;
1886         uint16_t window_id = (pe->pe_number << 1 ) + 1;
1887         int64_t rc;
1888
1889         pe_info(pe, "%sabling 64-bit DMA bypass\n", enable ? "En" : "Dis");
1890         if (enable) {
1891                 phys_addr_t top = memblock_end_of_DRAM();
1892
1893                 top = roundup_pow_of_two(top);
1894                 rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
1895                                                      pe->pe_number,
1896                                                      window_id,
1897                                                      pe->tce_bypass_base,
1898                                                      top);
1899         } else {
1900                 rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
1901                                                      pe->pe_number,
1902                                                      window_id,
1903                                                      pe->tce_bypass_base,
1904                                                      0);
1905         }
1906         if (rc)
1907                 pe_err(pe, "OPAL error %lld configuring bypass window\n", rc);
1908         else
1909                 pe->tce_bypass_enabled = enable;
1910 }
1911
1912 static void pnv_pci_ioda2_setup_bypass_pe(struct pnv_phb *phb,
1913                                           struct pnv_ioda_pe *pe)
1914 {
1915         /* TVE #1 is selected by PCI address bit 59 */
1916         pe->tce_bypass_base = 1ull << 59;
1917
1918         /* Install set_bypass callback for VFIO */
1919         pe->tce32_table->set_bypass = pnv_pci_ioda2_set_bypass;
1920
1921         /* Enable bypass by default */
1922         pnv_pci_ioda2_set_bypass(pe->tce32_table, true);
1923 }
1924
1925 static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
1926                                        struct pnv_ioda_pe *pe)
1927 {
1928         struct page *tce_mem = NULL;
1929         void *addr;
1930         const __be64 *swinvp;
1931         struct iommu_table *tbl;
1932         unsigned int tce_table_size, end;
1933         int64_t rc;
1934
1935         /* We shouldn't already have a 32-bit DMA associated */
1936         if (WARN_ON(pe->tce32_seg >= 0))
1937                 return;
1938
1939         tbl = pe->tce32_table;
1940         iommu_register_group(tbl, phb->hose->global_number, pe->pe_number);
1941
1942         /* The PE will reserve all possible 32-bits space */
1943         pe->tce32_seg = 0;
1944         end = (1 << ilog2(phb->ioda.m32_pci_base));
1945         tce_table_size = (end / 0x1000) * 8;
1946         pe_info(pe, "Setting up 32-bit TCE table at 0..%08x\n",
1947                 end);
1948
1949         /* Allocate TCE table */
1950         tce_mem = alloc_pages_node(phb->hose->node, GFP_KERNEL,
1951                                    get_order(tce_table_size));
1952         if (!tce_mem) {
1953                 pe_err(pe, "Failed to allocate a 32-bit TCE memory\n");
1954                 goto fail;
1955         }
1956         addr = page_address(tce_mem);
1957         memset(addr, 0, tce_table_size);
1958
1959         /*
1960          * Map TCE table through TVT. The TVE index is the PE number
1961          * shifted by 1 bit for 32-bits DMA space.
1962          */
1963         rc = opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number,
1964                                         pe->pe_number << 1, 1, __pa(addr),
1965                                         tce_table_size, 0x1000);
1966         if (rc) {
1967                 pe_err(pe, "Failed to configure 32-bit TCE table,"
1968                        " err %ld\n", rc);
1969                 goto fail;
1970         }
1971
1972         /* Setup linux iommu table */
1973         pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, 0,
1974                         IOMMU_PAGE_SHIFT_4K);
1975
1976         /* OPAL variant of PHB3 invalidated TCEs */
1977         swinvp = of_get_property(phb->hose->dn, "ibm,opal-tce-kill", NULL);
1978         if (swinvp) {
1979                 /* We need a couple more fields -- an address and a data
1980                  * to or.  Since the bus is only printed out on table free
1981                  * errors, and on the first pass the data will be a relative
1982                  * bus number, print that out instead.
1983                  */
1984                 pe->tce_inval_reg_phys = be64_to_cpup(swinvp);
1985                 tbl->it_index = (unsigned long)ioremap(pe->tce_inval_reg_phys,
1986                                 8);
1987                 tbl->it_type |= (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE);
1988         }
1989         tbl->it_ops = &pnv_ioda2_iommu_ops;
1990         iommu_init_table(tbl, phb->hose->node);
1991
1992         if (pe->flags & PNV_IODA_PE_DEV) {
1993                 /*
1994                  * Setting table base here only for carrying iommu_group
1995                  * further down to let iommu_add_device() do the job.
1996                  * pnv_pci_ioda_dma_dev_setup will override it later anyway.
1997                  */
1998                 set_iommu_table_base(&pe->pdev->dev, tbl);
1999                 iommu_add_device(&pe->pdev->dev);
2000         } else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
2001                 pnv_ioda_setup_bus_dma(pe, pe->pbus);
2002
2003         /* Also create a bypass window */
2004         if (!pnv_iommu_bypass_disabled)
2005                 pnv_pci_ioda2_setup_bypass_pe(phb, pe);
2006
2007         return;
2008 fail:
2009         if (pe->tce32_seg >= 0)
2010                 pe->tce32_seg = -1;
2011         if (tce_mem)
2012                 __free_pages(tce_mem, get_order(tce_table_size));
2013 }
2014
2015 static void pnv_ioda_setup_dma(struct pnv_phb *phb)
2016 {
2017         struct pci_controller *hose = phb->hose;
2018         unsigned int residual, remaining, segs, tw, base;
2019         struct pnv_ioda_pe *pe;
2020
2021         /* If we have more PE# than segments available, hand out one
2022          * per PE until we run out and let the rest fail. If not,
2023          * then we assign at least one segment per PE, plus more based
2024          * on the amount of devices under that PE
2025          */
2026         if (phb->ioda.dma_pe_count > phb->ioda.tce32_count)
2027                 residual = 0;
2028         else
2029                 residual = phb->ioda.tce32_count -
2030                         phb->ioda.dma_pe_count;
2031
2032         pr_info("PCI: Domain %04x has %ld available 32-bit DMA segments\n",
2033                 hose->global_number, phb->ioda.tce32_count);
2034         pr_info("PCI: %d PE# for a total weight of %d\n",
2035                 phb->ioda.dma_pe_count, phb->ioda.dma_weight);
2036
2037         /* Walk our PE list and configure their DMA segments, hand them
2038          * out one base segment plus any residual segments based on
2039          * weight
2040          */
2041         remaining = phb->ioda.tce32_count;
2042         tw = phb->ioda.dma_weight;
2043         base = 0;
2044         list_for_each_entry(pe, &phb->ioda.pe_dma_list, dma_link) {
2045                 if (!pe->dma_weight)
2046                         continue;
2047                 if (!remaining) {
2048                         pe_warn(pe, "No DMA32 resources available\n");
2049                         continue;
2050                 }
2051                 segs = 1;
2052                 if (residual) {
2053                         segs += ((pe->dma_weight * residual)  + (tw / 2)) / tw;
2054                         if (segs > remaining)
2055                                 segs = remaining;
2056                 }
2057
2058                 /*
2059                  * For IODA2 compliant PHB3, we needn't care about the weight.
2060                  * The all available 32-bits DMA space will be assigned to
2061                  * the specific PE.
2062                  */
2063                 if (phb->type == PNV_PHB_IODA1) {
2064                         pe_info(pe, "DMA weight %d, assigned %d DMA32 segments\n",
2065                                 pe->dma_weight, segs);
2066                         pnv_pci_ioda_setup_dma_pe(phb, pe, base, segs);
2067                 } else {
2068                         pe_info(pe, "Assign DMA32 space\n");
2069                         segs = 0;
2070                         pnv_pci_ioda2_setup_dma_pe(phb, pe);
2071                 }
2072
2073                 remaining -= segs;
2074                 base += segs;
2075         }
2076 }
2077
2078 #ifdef CONFIG_PCI_MSI
2079 static void pnv_ioda2_msi_eoi(struct irq_data *d)
2080 {
2081         unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d);
2082         struct irq_chip *chip = irq_data_get_irq_chip(d);
2083         struct pnv_phb *phb = container_of(chip, struct pnv_phb,
2084                                            ioda.irq_chip);
2085         int64_t rc;
2086
2087         rc = opal_pci_msi_eoi(phb->opal_id, hw_irq);
2088         WARN_ON_ONCE(rc);
2089
2090         icp_native_eoi(d);
2091 }
2092
2093
2094 static void set_msi_irq_chip(struct pnv_phb *phb, unsigned int virq)
2095 {
2096         struct irq_data *idata;
2097         struct irq_chip *ichip;
2098
2099         if (phb->type != PNV_PHB_IODA2)
2100                 return;
2101
2102         if (!phb->ioda.irq_chip_init) {
2103                 /*
2104                  * First time we setup an MSI IRQ, we need to setup the
2105                  * corresponding IRQ chip to route correctly.
2106                  */
2107                 idata = irq_get_irq_data(virq);
2108                 ichip = irq_data_get_irq_chip(idata);
2109                 phb->ioda.irq_chip_init = 1;
2110                 phb->ioda.irq_chip = *ichip;
2111                 phb->ioda.irq_chip.irq_eoi = pnv_ioda2_msi_eoi;
2112         }
2113         irq_set_chip(virq, &phb->ioda.irq_chip);
2114 }
2115
2116 #ifdef CONFIG_CXL_BASE
2117
2118 struct device_node *pnv_pci_get_phb_node(struct pci_dev *dev)
2119 {
2120         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2121
2122         return of_node_get(hose->dn);
2123 }
2124 EXPORT_SYMBOL(pnv_pci_get_phb_node);
2125
2126 int pnv_phb_to_cxl_mode(struct pci_dev *dev, uint64_t mode)
2127 {
2128         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2129         struct pnv_phb *phb = hose->private_data;
2130         struct pnv_ioda_pe *pe;
2131         int rc;
2132
2133         pe = pnv_ioda_get_pe(dev);
2134         if (!pe)
2135                 return -ENODEV;
2136
2137         pe_info(pe, "Switching PHB to CXL\n");
2138
2139         rc = opal_pci_set_phb_cxl_mode(phb->opal_id, mode, pe->pe_number);
2140         if (rc)
2141                 dev_err(&dev->dev, "opal_pci_set_phb_cxl_mode failed: %i\n", rc);
2142
2143         return rc;
2144 }
2145 EXPORT_SYMBOL(pnv_phb_to_cxl_mode);
2146
2147 /* Find PHB for cxl dev and allocate MSI hwirqs?
2148  * Returns the absolute hardware IRQ number
2149  */
2150 int pnv_cxl_alloc_hwirqs(struct pci_dev *dev, int num)
2151 {
2152         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2153         struct pnv_phb *phb = hose->private_data;
2154         int hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, num);
2155
2156         if (hwirq < 0) {
2157                 dev_warn(&dev->dev, "Failed to find a free MSI\n");
2158                 return -ENOSPC;
2159         }
2160
2161         return phb->msi_base + hwirq;
2162 }
2163 EXPORT_SYMBOL(pnv_cxl_alloc_hwirqs);
2164
2165 void pnv_cxl_release_hwirqs(struct pci_dev *dev, int hwirq, int num)
2166 {
2167         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2168         struct pnv_phb *phb = hose->private_data;
2169
2170         msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq - phb->msi_base, num);
2171 }
2172 EXPORT_SYMBOL(pnv_cxl_release_hwirqs);
2173
2174 void pnv_cxl_release_hwirq_ranges(struct cxl_irq_ranges *irqs,
2175                                   struct pci_dev *dev)
2176 {
2177         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2178         struct pnv_phb *phb = hose->private_data;
2179         int i, hwirq;
2180
2181         for (i = 1; i < CXL_IRQ_RANGES; i++) {
2182                 if (!irqs->range[i])
2183                         continue;
2184                 pr_devel("cxl release irq range 0x%x: offset: 0x%lx  limit: %ld\n",
2185                          i, irqs->offset[i],
2186                          irqs->range[i]);
2187                 hwirq = irqs->offset[i] - phb->msi_base;
2188                 msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq,
2189                                        irqs->range[i]);
2190         }
2191 }
2192 EXPORT_SYMBOL(pnv_cxl_release_hwirq_ranges);
2193
2194 int pnv_cxl_alloc_hwirq_ranges(struct cxl_irq_ranges *irqs,
2195                                struct pci_dev *dev, int num)
2196 {
2197         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2198         struct pnv_phb *phb = hose->private_data;
2199         int i, hwirq, try;
2200
2201         memset(irqs, 0, sizeof(struct cxl_irq_ranges));
2202
2203         /* 0 is reserved for the multiplexed PSL DSI interrupt */
2204         for (i = 1; i < CXL_IRQ_RANGES && num; i++) {
2205                 try = num;
2206                 while (try) {
2207                         hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, try);
2208                         if (hwirq >= 0)
2209                                 break;
2210                         try /= 2;
2211                 }
2212                 if (!try)
2213                         goto fail;
2214
2215                 irqs->offset[i] = phb->msi_base + hwirq;
2216                 irqs->range[i] = try;
2217                 pr_devel("cxl alloc irq range 0x%x: offset: 0x%lx  limit: %li\n",
2218                          i, irqs->offset[i], irqs->range[i]);
2219                 num -= try;
2220         }
2221         if (num)
2222                 goto fail;
2223
2224         return 0;
2225 fail:
2226         pnv_cxl_release_hwirq_ranges(irqs, dev);
2227         return -ENOSPC;
2228 }
2229 EXPORT_SYMBOL(pnv_cxl_alloc_hwirq_ranges);
2230
2231 int pnv_cxl_get_irq_count(struct pci_dev *dev)
2232 {
2233         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2234         struct pnv_phb *phb = hose->private_data;
2235
2236         return phb->msi_bmp.irq_count;
2237 }
2238 EXPORT_SYMBOL(pnv_cxl_get_irq_count);
2239
2240 int pnv_cxl_ioda_msi_setup(struct pci_dev *dev, unsigned int hwirq,
2241                            unsigned int virq)
2242 {
2243         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2244         struct pnv_phb *phb = hose->private_data;
2245         unsigned int xive_num = hwirq - phb->msi_base;
2246         struct pnv_ioda_pe *pe;
2247         int rc;
2248
2249         if (!(pe = pnv_ioda_get_pe(dev)))
2250                 return -ENODEV;
2251
2252         /* Assign XIVE to PE */
2253         rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num);
2254         if (rc) {
2255                 pe_warn(pe, "%s: OPAL error %d setting msi_base 0x%x "
2256                         "hwirq 0x%x XIVE 0x%x PE\n",
2257                         pci_name(dev), rc, phb->msi_base, hwirq, xive_num);
2258                 return -EIO;
2259         }
2260         set_msi_irq_chip(phb, virq);
2261
2262         return 0;
2263 }
2264 EXPORT_SYMBOL(pnv_cxl_ioda_msi_setup);
2265 #endif
2266
2267 static int pnv_pci_ioda_msi_setup(struct pnv_phb *phb, struct pci_dev *dev,
2268                                   unsigned int hwirq, unsigned int virq,
2269                                   unsigned int is_64, struct msi_msg *msg)
2270 {
2271         struct pnv_ioda_pe *pe = pnv_ioda_get_pe(dev);
2272         unsigned int xive_num = hwirq - phb->msi_base;
2273         __be32 data;
2274         int rc;
2275
2276         /* No PE assigned ? bail out ... no MSI for you ! */
2277         if (pe == NULL)
2278                 return -ENXIO;
2279
2280         /* Check if we have an MVE */
2281         if (pe->mve_number < 0)
2282                 return -ENXIO;
2283
2284         /* Force 32-bit MSI on some broken devices */
2285         if (dev->no_64bit_msi)
2286                 is_64 = 0;
2287
2288         /* Assign XIVE to PE */
2289         rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num);
2290         if (rc) {
2291                 pr_warn("%s: OPAL error %d setting XIVE %d PE\n",
2292                         pci_name(dev), rc, xive_num);
2293                 return -EIO;
2294         }
2295
2296         if (is_64) {
2297                 __be64 addr64;
2298
2299                 rc = opal_get_msi_64(phb->opal_id, pe->mve_number, xive_num, 1,
2300                                      &addr64, &data);
2301                 if (rc) {
2302                         pr_warn("%s: OPAL error %d getting 64-bit MSI data\n",
2303                                 pci_name(dev), rc);
2304                         return -EIO;
2305                 }
2306                 msg->address_hi = be64_to_cpu(addr64) >> 32;
2307                 msg->address_lo = be64_to_cpu(addr64) & 0xfffffffful;
2308         } else {
2309                 __be32 addr32;
2310
2311                 rc = opal_get_msi_32(phb->opal_id, pe->mve_number, xive_num, 1,
2312                                      &addr32, &data);
2313                 if (rc) {
2314                         pr_warn("%s: OPAL error %d getting 32-bit MSI data\n",
2315                                 pci_name(dev), rc);
2316                         return -EIO;
2317                 }
2318                 msg->address_hi = 0;
2319                 msg->address_lo = be32_to_cpu(addr32);
2320         }
2321         msg->data = be32_to_cpu(data);
2322
2323         set_msi_irq_chip(phb, virq);
2324
2325         pr_devel("%s: %s-bit MSI on hwirq %x (xive #%d),"
2326                  " address=%x_%08x data=%x PE# %d\n",
2327                  pci_name(dev), is_64 ? "64" : "32", hwirq, xive_num,
2328                  msg->address_hi, msg->address_lo, data, pe->pe_number);
2329
2330         return 0;
2331 }
2332
2333 static void pnv_pci_init_ioda_msis(struct pnv_phb *phb)
2334 {
2335         unsigned int count;
2336         const __be32 *prop = of_get_property(phb->hose->dn,
2337                                              "ibm,opal-msi-ranges", NULL);
2338         if (!prop) {
2339                 /* BML Fallback */
2340                 prop = of_get_property(phb->hose->dn, "msi-ranges", NULL);
2341         }
2342         if (!prop)
2343                 return;
2344
2345         phb->msi_base = be32_to_cpup(prop);
2346         count = be32_to_cpup(prop + 1);
2347         if (msi_bitmap_alloc(&phb->msi_bmp, count, phb->hose->dn)) {
2348                 pr_err("PCI %d: Failed to allocate MSI bitmap !\n",
2349                        phb->hose->global_number);
2350                 return;
2351         }
2352
2353         phb->msi_setup = pnv_pci_ioda_msi_setup;
2354         phb->msi32_support = 1;
2355         pr_info("  Allocated bitmap for %d MSIs (base IRQ 0x%x)\n",
2356                 count, phb->msi_base);
2357 }
2358 #else
2359 static void pnv_pci_init_ioda_msis(struct pnv_phb *phb) { }
2360 #endif /* CONFIG_PCI_MSI */
2361
2362 #ifdef CONFIG_PCI_IOV
2363 static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev)
2364 {
2365         struct pci_controller *hose;
2366         struct pnv_phb *phb;
2367         struct resource *res;
2368         int i;
2369         resource_size_t size;
2370         struct pci_dn *pdn;
2371         int mul, total_vfs;
2372
2373         if (!pdev->is_physfn || pdev->is_added)
2374                 return;
2375
2376         hose = pci_bus_to_host(pdev->bus);
2377         phb = hose->private_data;
2378
2379         pdn = pci_get_pdn(pdev);
2380         pdn->vfs_expanded = 0;
2381
2382         total_vfs = pci_sriov_get_totalvfs(pdev);
2383         pdn->m64_per_iov = 1;
2384         mul = phb->ioda.total_pe;
2385
2386         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
2387                 res = &pdev->resource[i + PCI_IOV_RESOURCES];
2388                 if (!res->flags || res->parent)
2389                         continue;
2390                 if (!pnv_pci_is_mem_pref_64(res->flags)) {
2391                         dev_warn(&pdev->dev, " non M64 VF BAR%d: %pR\n",
2392                                  i, res);
2393                         continue;
2394                 }
2395
2396                 size = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES);
2397
2398                 /* bigger than 64M */
2399                 if (size > (1 << 26)) {
2400                         dev_info(&pdev->dev, "PowerNV: VF BAR%d: %pR IOV size is bigger than 64M, roundup power2\n",
2401                                  i, res);
2402                         pdn->m64_per_iov = M64_PER_IOV;
2403                         mul = roundup_pow_of_two(total_vfs);
2404                         break;
2405                 }
2406         }
2407
2408         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
2409                 res = &pdev->resource[i + PCI_IOV_RESOURCES];
2410                 if (!res->flags || res->parent)
2411                         continue;
2412                 if (!pnv_pci_is_mem_pref_64(res->flags)) {
2413                         dev_warn(&pdev->dev, "Skipping expanding VF BAR%d: %pR\n",
2414                                  i, res);
2415                         continue;
2416                 }
2417
2418                 dev_dbg(&pdev->dev, " Fixing VF BAR%d: %pR to\n", i, res);
2419                 size = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES);
2420                 res->end = res->start + size * mul - 1;
2421                 dev_dbg(&pdev->dev, "                       %pR\n", res);
2422                 dev_info(&pdev->dev, "VF BAR%d: %pR (expanded to %d VFs for PE alignment)",
2423                          i, res, mul);
2424         }
2425         pdn->vfs_expanded = mul;
2426 }
2427 #endif /* CONFIG_PCI_IOV */
2428
2429 /*
2430  * This function is supposed to be called on basis of PE from top
2431  * to bottom style. So the the I/O or MMIO segment assigned to
2432  * parent PE could be overrided by its child PEs if necessary.
2433  */
2434 static void pnv_ioda_setup_pe_seg(struct pci_controller *hose,
2435                                   struct pnv_ioda_pe *pe)
2436 {
2437         struct pnv_phb *phb = hose->private_data;
2438         struct pci_bus_region region;
2439         struct resource *res;
2440         int i, index;
2441         int rc;
2442
2443         /*
2444          * NOTE: We only care PCI bus based PE for now. For PCI
2445          * device based PE, for example SRIOV sensitive VF should
2446          * be figured out later.
2447          */
2448         BUG_ON(!(pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)));
2449
2450         pci_bus_for_each_resource(pe->pbus, res, i) {
2451                 if (!res || !res->flags ||
2452                     res->start > res->end)
2453                         continue;
2454
2455                 if (res->flags & IORESOURCE_IO) {
2456                         region.start = res->start - phb->ioda.io_pci_base;
2457                         region.end   = res->end - phb->ioda.io_pci_base;
2458                         index = region.start / phb->ioda.io_segsize;
2459
2460                         while (index < phb->ioda.total_pe &&
2461                                region.start <= region.end) {
2462                                 phb->ioda.io_segmap[index] = pe->pe_number;
2463                                 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
2464                                         pe->pe_number, OPAL_IO_WINDOW_TYPE, 0, index);
2465                                 if (rc != OPAL_SUCCESS) {
2466                                         pr_err("%s: OPAL error %d when mapping IO "
2467                                                "segment #%d to PE#%d\n",
2468                                                __func__, rc, index, pe->pe_number);
2469                                         break;
2470                                 }
2471
2472                                 region.start += phb->ioda.io_segsize;
2473                                 index++;
2474                         }
2475                 } else if ((res->flags & IORESOURCE_MEM) &&
2476                            !pnv_pci_is_mem_pref_64(res->flags)) {
2477                         region.start = res->start -
2478                                        hose->mem_offset[0] -
2479                                        phb->ioda.m32_pci_base;
2480                         region.end   = res->end -
2481                                        hose->mem_offset[0] -
2482                                        phb->ioda.m32_pci_base;
2483                         index = region.start / phb->ioda.m32_segsize;
2484
2485                         while (index < phb->ioda.total_pe &&
2486                                region.start <= region.end) {
2487                                 phb->ioda.m32_segmap[index] = pe->pe_number;
2488                                 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
2489                                         pe->pe_number, OPAL_M32_WINDOW_TYPE, 0, index);
2490                                 if (rc != OPAL_SUCCESS) {
2491                                         pr_err("%s: OPAL error %d when mapping M32 "
2492                                                "segment#%d to PE#%d",
2493                                                __func__, rc, index, pe->pe_number);
2494                                         break;
2495                                 }
2496
2497                                 region.start += phb->ioda.m32_segsize;
2498                                 index++;
2499                         }
2500                 }
2501         }
2502 }
2503
2504 static void pnv_pci_ioda_setup_seg(void)
2505 {
2506         struct pci_controller *tmp, *hose;
2507         struct pnv_phb *phb;
2508         struct pnv_ioda_pe *pe;
2509
2510         list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2511                 phb = hose->private_data;
2512                 list_for_each_entry(pe, &phb->ioda.pe_list, list) {
2513                         pnv_ioda_setup_pe_seg(hose, pe);
2514                 }
2515         }
2516 }
2517
2518 static void pnv_pci_ioda_setup_DMA(void)
2519 {
2520         struct pci_controller *hose, *tmp;
2521         struct pnv_phb *phb;
2522
2523         list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2524                 pnv_ioda_setup_dma(hose->private_data);
2525
2526                 /* Mark the PHB initialization done */
2527                 phb = hose->private_data;
2528                 phb->initialized = 1;
2529         }
2530 }
2531
2532 static void pnv_pci_ioda_create_dbgfs(void)
2533 {
2534 #ifdef CONFIG_DEBUG_FS
2535         struct pci_controller *hose, *tmp;
2536         struct pnv_phb *phb;
2537         char name[16];
2538
2539         list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2540                 phb = hose->private_data;
2541
2542                 sprintf(name, "PCI%04x", hose->global_number);
2543                 phb->dbgfs = debugfs_create_dir(name, powerpc_debugfs_root);
2544                 if (!phb->dbgfs)
2545                         pr_warning("%s: Error on creating debugfs on PHB#%x\n",
2546                                 __func__, hose->global_number);
2547         }
2548 #endif /* CONFIG_DEBUG_FS */
2549 }
2550
2551 static void pnv_pci_ioda_fixup(void)
2552 {
2553         pnv_pci_ioda_setup_PEs();
2554         pnv_pci_ioda_setup_seg();
2555         pnv_pci_ioda_setup_DMA();
2556
2557         pnv_pci_ioda_create_dbgfs();
2558
2559 #ifdef CONFIG_EEH
2560         eeh_init();
2561         eeh_addr_cache_build();
2562 #endif
2563 }
2564
2565 /*
2566  * Returns the alignment for I/O or memory windows for P2P
2567  * bridges. That actually depends on how PEs are segmented.
2568  * For now, we return I/O or M32 segment size for PE sensitive
2569  * P2P bridges. Otherwise, the default values (4KiB for I/O,
2570  * 1MiB for memory) will be returned.
2571  *
2572  * The current PCI bus might be put into one PE, which was
2573  * create against the parent PCI bridge. For that case, we
2574  * needn't enlarge the alignment so that we can save some
2575  * resources.
2576  */
2577 static resource_size_t pnv_pci_window_alignment(struct pci_bus *bus,
2578                                                 unsigned long type)
2579 {
2580         struct pci_dev *bridge;
2581         struct pci_controller *hose = pci_bus_to_host(bus);
2582         struct pnv_phb *phb = hose->private_data;
2583         int num_pci_bridges = 0;
2584
2585         bridge = bus->self;
2586         while (bridge) {
2587                 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE) {
2588                         num_pci_bridges++;
2589                         if (num_pci_bridges >= 2)
2590                                 return 1;
2591                 }
2592
2593                 bridge = bridge->bus->self;
2594         }
2595
2596         /* We fail back to M32 if M64 isn't supported */
2597         if (phb->ioda.m64_segsize &&
2598             pnv_pci_is_mem_pref_64(type))
2599                 return phb->ioda.m64_segsize;
2600         if (type & IORESOURCE_MEM)
2601                 return phb->ioda.m32_segsize;
2602
2603         return phb->ioda.io_segsize;
2604 }
2605
2606 #ifdef CONFIG_PCI_IOV
2607 static resource_size_t pnv_pci_iov_resource_alignment(struct pci_dev *pdev,
2608                                                       int resno)
2609 {
2610         struct pci_dn *pdn = pci_get_pdn(pdev);
2611         resource_size_t align, iov_align;
2612
2613         iov_align = resource_size(&pdev->resource[resno]);
2614         if (iov_align)
2615                 return iov_align;
2616
2617         align = pci_iov_resource_size(pdev, resno);
2618         if (pdn->vfs_expanded)
2619                 return pdn->vfs_expanded * align;
2620
2621         return align;
2622 }
2623 #endif /* CONFIG_PCI_IOV */
2624
2625 /* Prevent enabling devices for which we couldn't properly
2626  * assign a PE
2627  */
2628 static bool pnv_pci_enable_device_hook(struct pci_dev *dev)
2629 {
2630         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2631         struct pnv_phb *phb = hose->private_data;
2632         struct pci_dn *pdn;
2633
2634         /* The function is probably called while the PEs have
2635          * not be created yet. For example, resource reassignment
2636          * during PCI probe period. We just skip the check if
2637          * PEs isn't ready.
2638          */
2639         if (!phb->initialized)
2640                 return true;
2641
2642         pdn = pci_get_pdn(dev);
2643         if (!pdn || pdn->pe_number == IODA_INVALID_PE)
2644                 return false;
2645
2646         return true;
2647 }
2648
2649 static u32 pnv_ioda_bdfn_to_pe(struct pnv_phb *phb, struct pci_bus *bus,
2650                                u32 devfn)
2651 {
2652         return phb->ioda.pe_rmap[(bus->number << 8) | devfn];
2653 }
2654
2655 static void pnv_pci_ioda_shutdown(struct pci_controller *hose)
2656 {
2657         struct pnv_phb *phb = hose->private_data;
2658
2659         opal_pci_reset(phb->opal_id, OPAL_RESET_PCI_IODA_TABLE,
2660                        OPAL_ASSERT_RESET);
2661 }
2662
2663 static const struct pci_controller_ops pnv_pci_ioda_controller_ops = {
2664        .dma_dev_setup = pnv_pci_dma_dev_setup,
2665 #ifdef CONFIG_PCI_MSI
2666        .setup_msi_irqs = pnv_setup_msi_irqs,
2667        .teardown_msi_irqs = pnv_teardown_msi_irqs,
2668 #endif
2669        .enable_device_hook = pnv_pci_enable_device_hook,
2670        .window_alignment = pnv_pci_window_alignment,
2671        .reset_secondary_bus = pnv_pci_reset_secondary_bus,
2672        .dma_set_mask = pnv_pci_ioda_dma_set_mask,
2673        .shutdown = pnv_pci_ioda_shutdown,
2674 };
2675
2676 static void __init pnv_pci_init_ioda_phb(struct device_node *np,
2677                                          u64 hub_id, int ioda_type)
2678 {
2679         struct pci_controller *hose;
2680         struct pnv_phb *phb;
2681         unsigned long size, m32map_off, pemap_off, iomap_off = 0;
2682         const __be64 *prop64;
2683         const __be32 *prop32;
2684         int len;
2685         u64 phb_id;
2686         void *aux;
2687         long rc;
2688
2689         pr_info("Initializing IODA%d OPAL PHB %s\n", ioda_type, np->full_name);
2690
2691         prop64 = of_get_property(np, "ibm,opal-phbid", NULL);
2692         if (!prop64) {
2693                 pr_err("  Missing \"ibm,opal-phbid\" property !\n");
2694                 return;
2695         }
2696         phb_id = be64_to_cpup(prop64);
2697         pr_debug("  PHB-ID  : 0x%016llx\n", phb_id);
2698
2699         phb = memblock_virt_alloc(sizeof(struct pnv_phb), 0);
2700
2701         /* Allocate PCI controller */
2702         phb->hose = hose = pcibios_alloc_controller(np);
2703         if (!phb->hose) {
2704                 pr_err("  Can't allocate PCI controller for %s\n",
2705                        np->full_name);
2706                 memblock_free(__pa(phb), sizeof(struct pnv_phb));
2707                 return;
2708         }
2709
2710         spin_lock_init(&phb->lock);
2711         prop32 = of_get_property(np, "bus-range", &len);
2712         if (prop32 && len == 8) {
2713                 hose->first_busno = be32_to_cpu(prop32[0]);
2714                 hose->last_busno = be32_to_cpu(prop32[1]);
2715         } else {
2716                 pr_warn("  Broken <bus-range> on %s\n", np->full_name);
2717                 hose->first_busno = 0;
2718                 hose->last_busno = 0xff;
2719         }
2720         hose->private_data = phb;
2721         phb->hub_id = hub_id;
2722         phb->opal_id = phb_id;
2723         phb->type = ioda_type;
2724         mutex_init(&phb->ioda.pe_alloc_mutex);
2725
2726         /* Detect specific models for error handling */
2727         if (of_device_is_compatible(np, "ibm,p7ioc-pciex"))
2728                 phb->model = PNV_PHB_MODEL_P7IOC;
2729         else if (of_device_is_compatible(np, "ibm,power8-pciex"))
2730                 phb->model = PNV_PHB_MODEL_PHB3;
2731         else
2732                 phb->model = PNV_PHB_MODEL_UNKNOWN;
2733
2734         /* Parse 32-bit and IO ranges (if any) */
2735         pci_process_bridge_OF_ranges(hose, np, !hose->global_number);
2736
2737         /* Get registers */
2738         phb->regs = of_iomap(np, 0);
2739         if (phb->regs == NULL)
2740                 pr_err("  Failed to map registers !\n");
2741
2742         /* Initialize more IODA stuff */
2743         phb->ioda.total_pe = 1;
2744         prop32 = of_get_property(np, "ibm,opal-num-pes", NULL);
2745         if (prop32)
2746                 phb->ioda.total_pe = be32_to_cpup(prop32);
2747         prop32 = of_get_property(np, "ibm,opal-reserved-pe", NULL);
2748         if (prop32)
2749                 phb->ioda.reserved_pe = be32_to_cpup(prop32);
2750
2751         /* Parse 64-bit MMIO range */
2752         pnv_ioda_parse_m64_window(phb);
2753
2754         phb->ioda.m32_size = resource_size(&hose->mem_resources[0]);
2755         /* FW Has already off top 64k of M32 space (MSI space) */
2756         phb->ioda.m32_size += 0x10000;
2757
2758         phb->ioda.m32_segsize = phb->ioda.m32_size / phb->ioda.total_pe;
2759         phb->ioda.m32_pci_base = hose->mem_resources[0].start - hose->mem_offset[0];
2760         phb->ioda.io_size = hose->pci_io_size;
2761         phb->ioda.io_segsize = phb->ioda.io_size / phb->ioda.total_pe;
2762         phb->ioda.io_pci_base = 0; /* XXX calculate this ? */
2763
2764         /* Allocate aux data & arrays. We don't have IO ports on PHB3 */
2765         size = _ALIGN_UP(phb->ioda.total_pe / 8, sizeof(unsigned long));
2766         m32map_off = size;
2767         size += phb->ioda.total_pe * sizeof(phb->ioda.m32_segmap[0]);
2768         if (phb->type == PNV_PHB_IODA1) {
2769                 iomap_off = size;
2770                 size += phb->ioda.total_pe * sizeof(phb->ioda.io_segmap[0]);
2771         }
2772         pemap_off = size;
2773         size += phb->ioda.total_pe * sizeof(struct pnv_ioda_pe);
2774         aux = memblock_virt_alloc(size, 0);
2775         phb->ioda.pe_alloc = aux;
2776         phb->ioda.m32_segmap = aux + m32map_off;
2777         if (phb->type == PNV_PHB_IODA1)
2778                 phb->ioda.io_segmap = aux + iomap_off;
2779         phb->ioda.pe_array = aux + pemap_off;
2780         set_bit(phb->ioda.reserved_pe, phb->ioda.pe_alloc);
2781
2782         INIT_LIST_HEAD(&phb->ioda.pe_dma_list);
2783         INIT_LIST_HEAD(&phb->ioda.pe_list);
2784         mutex_init(&phb->ioda.pe_list_mutex);
2785
2786         /* Calculate how many 32-bit TCE segments we have */
2787         phb->ioda.tce32_count = phb->ioda.m32_pci_base >> 28;
2788
2789 #if 0 /* We should really do that ... */
2790         rc = opal_pci_set_phb_mem_window(opal->phb_id,
2791                                          window_type,
2792                                          window_num,
2793                                          starting_real_address,
2794                                          starting_pci_address,
2795                                          segment_size);
2796 #endif
2797
2798         pr_info("  %03d (%03d) PE's M32: 0x%x [segment=0x%x]\n",
2799                 phb->ioda.total_pe, phb->ioda.reserved_pe,
2800                 phb->ioda.m32_size, phb->ioda.m32_segsize);
2801         if (phb->ioda.m64_size)
2802                 pr_info("                 M64: 0x%lx [segment=0x%lx]\n",
2803                         phb->ioda.m64_size, phb->ioda.m64_segsize);
2804         if (phb->ioda.io_size)
2805                 pr_info("                  IO: 0x%x [segment=0x%x]\n",
2806                         phb->ioda.io_size, phb->ioda.io_segsize);
2807
2808
2809         phb->hose->ops = &pnv_pci_ops;
2810         phb->get_pe_state = pnv_ioda_get_pe_state;
2811         phb->freeze_pe = pnv_ioda_freeze_pe;
2812         phb->unfreeze_pe = pnv_ioda_unfreeze_pe;
2813
2814         /* Setup RID -> PE mapping function */
2815         phb->bdfn_to_pe = pnv_ioda_bdfn_to_pe;
2816
2817         /* Setup TCEs */
2818         phb->dma_dev_setup = pnv_pci_ioda_dma_dev_setup;
2819         phb->dma_get_required_mask = pnv_pci_ioda_dma_get_required_mask;
2820
2821         /* Setup MSI support */
2822         pnv_pci_init_ioda_msis(phb);
2823
2824         /*
2825          * We pass the PCI probe flag PCI_REASSIGN_ALL_RSRC here
2826          * to let the PCI core do resource assignment. It's supposed
2827          * that the PCI core will do correct I/O and MMIO alignment
2828          * for the P2P bridge bars so that each PCI bus (excluding
2829          * the child P2P bridges) can form individual PE.
2830          */
2831         ppc_md.pcibios_fixup = pnv_pci_ioda_fixup;
2832         hose->controller_ops = pnv_pci_ioda_controller_ops;
2833
2834 #ifdef CONFIG_PCI_IOV
2835         ppc_md.pcibios_fixup_sriov = pnv_pci_ioda_fixup_iov_resources;
2836         ppc_md.pcibios_iov_resource_alignment = pnv_pci_iov_resource_alignment;
2837 #endif
2838
2839         pci_add_flags(PCI_REASSIGN_ALL_RSRC);
2840
2841         /* Reset IODA tables to a clean state */
2842         rc = opal_pci_reset(phb_id, OPAL_RESET_PCI_IODA_TABLE, OPAL_ASSERT_RESET);
2843         if (rc)
2844                 pr_warning("  OPAL Error %ld performing IODA table reset !\n", rc);
2845
2846         /* If we're running in kdump kerenl, the previous kerenl never
2847          * shutdown PCI devices correctly. We already got IODA table
2848          * cleaned out. So we have to issue PHB reset to stop all PCI
2849          * transactions from previous kerenl.
2850          */
2851         if (is_kdump_kernel()) {
2852                 pr_info("  Issue PHB reset ...\n");
2853                 pnv_eeh_phb_reset(hose, EEH_RESET_FUNDAMENTAL);
2854                 pnv_eeh_phb_reset(hose, EEH_RESET_DEACTIVATE);
2855         }
2856
2857         /* Remove M64 resource if we can't configure it successfully */
2858         if (!phb->init_m64 || phb->init_m64(phb))
2859                 hose->mem_resources[1].flags = 0;
2860 }
2861
2862 void __init pnv_pci_init_ioda2_phb(struct device_node *np)
2863 {
2864         pnv_pci_init_ioda_phb(np, 0, PNV_PHB_IODA2);
2865 }
2866
2867 void __init pnv_pci_init_ioda_hub(struct device_node *np)
2868 {
2869         struct device_node *phbn;
2870         const __be64 *prop64;
2871         u64 hub_id;
2872
2873         pr_info("Probing IODA IO-Hub %s\n", np->full_name);
2874
2875         prop64 = of_get_property(np, "ibm,opal-hubid", NULL);
2876         if (!prop64) {
2877                 pr_err(" Missing \"ibm,opal-hubid\" property !\n");
2878                 return;
2879         }
2880         hub_id = be64_to_cpup(prop64);
2881         pr_devel(" HUB-ID : 0x%016llx\n", hub_id);
2882
2883         /* Count child PHBs */
2884         for_each_child_of_node(np, phbn) {
2885                 /* Look for IODA1 PHBs */
2886                 if (of_device_is_compatible(phbn, "ibm,ioda-phb"))
2887                         pnv_pci_init_ioda_phb(phbn, hub_id, PNV_PHB_IODA1);
2888         }
2889 }