2 * Derived from arch/powerpc/kernel/iommu.c
4 * Copyright IBM Corporation, 2006-2007
5 * Copyright (C) 2006 Jon Mason <jdmason@kudzu.us>
7 * Author: Jon Mason <jdmason@kudzu.us>
8 * Author: Muli Ben-Yehuda <muli@il.ibm.com>
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software
22 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25 #include <linux/kernel.h>
26 #include <linux/init.h>
27 #include <linux/types.h>
28 #include <linux/slab.h>
30 #include <linux/spinlock.h>
31 #include <linux/string.h>
32 #include <linux/crash_dump.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/bitops.h>
35 #include <linux/pci_ids.h>
36 #include <linux/pci.h>
37 #include <linux/delay.h>
38 #include <linux/scatterlist.h>
39 #include <linux/iommu-helper.h>
41 #include <asm/iommu.h>
42 #include <asm/calgary.h>
44 #include <asm/pci-direct.h>
45 #include <asm/system.h>
48 #include <asm/bios_ebda.h>
49 #include <asm/x86_init.h>
51 #ifdef CONFIG_CALGARY_IOMMU_ENABLED_BY_DEFAULT
52 int use_calgary __read_mostly = 1;
54 int use_calgary __read_mostly = 0;
55 #endif /* CONFIG_CALGARY_DEFAULT_ENABLED */
57 #define PCI_DEVICE_ID_IBM_CALGARY 0x02a1
58 #define PCI_DEVICE_ID_IBM_CALIOC2 0x0308
60 /* register offsets inside the host bridge space */
61 #define CALGARY_CONFIG_REG 0x0108
62 #define PHB_CSR_OFFSET 0x0110 /* Channel Status */
63 #define PHB_PLSSR_OFFSET 0x0120
64 #define PHB_CONFIG_RW_OFFSET 0x0160
65 #define PHB_IOBASE_BAR_LOW 0x0170
66 #define PHB_IOBASE_BAR_HIGH 0x0180
67 #define PHB_MEM_1_LOW 0x0190
68 #define PHB_MEM_1_HIGH 0x01A0
69 #define PHB_IO_ADDR_SIZE 0x01B0
70 #define PHB_MEM_1_SIZE 0x01C0
71 #define PHB_MEM_ST_OFFSET 0x01D0
72 #define PHB_AER_OFFSET 0x0200
73 #define PHB_CONFIG_0_HIGH 0x0220
74 #define PHB_CONFIG_0_LOW 0x0230
75 #define PHB_CONFIG_0_END 0x0240
76 #define PHB_MEM_2_LOW 0x02B0
77 #define PHB_MEM_2_HIGH 0x02C0
78 #define PHB_MEM_2_SIZE_HIGH 0x02D0
79 #define PHB_MEM_2_SIZE_LOW 0x02E0
80 #define PHB_DOSHOLE_OFFSET 0x08E0
82 /* CalIOC2 specific */
83 #define PHB_SAVIOR_L2 0x0DB0
84 #define PHB_PAGE_MIG_CTRL 0x0DA8
85 #define PHB_PAGE_MIG_DEBUG 0x0DA0
86 #define PHB_ROOT_COMPLEX_STATUS 0x0CB0
89 #define PHB_TCE_ENABLE 0x20000000
90 #define PHB_SLOT_DISABLE 0x1C000000
91 #define PHB_DAC_DISABLE 0x01000000
92 #define PHB_MEM2_ENABLE 0x00400000
93 #define PHB_MCSR_ENABLE 0x00100000
94 /* TAR (Table Address Register) */
95 #define TAR_SW_BITS 0x0000ffffffff800fUL
96 #define TAR_VALID 0x0000000000000008UL
97 /* CSR (Channel/DMA Status Register) */
98 #define CSR_AGENT_MASK 0xffe0ffff
99 /* CCR (Calgary Configuration Register) */
100 #define CCR_2SEC_TIMEOUT 0x000000000000000EUL
101 /* PMCR/PMDR (Page Migration Control/Debug Registers */
102 #define PMR_SOFTSTOP 0x80000000
103 #define PMR_SOFTSTOPFAULT 0x40000000
104 #define PMR_HARDSTOP 0x20000000
106 #define MAX_NUM_OF_PHBS 8 /* how many PHBs in total? */
107 #define MAX_NUM_CHASSIS 8 /* max number of chassis */
108 /* MAX_PHB_BUS_NUM is the maximal possible dev->bus->number */
109 #define MAX_PHB_BUS_NUM (MAX_NUM_OF_PHBS * MAX_NUM_CHASSIS * 2)
110 #define PHBS_PER_CALGARY 4
112 /* register offsets in Calgary's internal register space */
113 static const unsigned long tar_offsets[] = {
120 static const unsigned long split_queue_offsets[] = {
121 0x4870 /* SPLIT QUEUE 0 */,
122 0x5870 /* SPLIT QUEUE 1 */,
123 0x6870 /* SPLIT QUEUE 2 */,
124 0x7870 /* SPLIT QUEUE 3 */
127 static const unsigned long phb_offsets[] = {
134 /* PHB debug registers */
136 static const unsigned long phb_debug_offsets[] = {
137 0x4000 /* PHB 0 DEBUG */,
138 0x5000 /* PHB 1 DEBUG */,
139 0x6000 /* PHB 2 DEBUG */,
140 0x7000 /* PHB 3 DEBUG */
144 * STUFF register for each debug PHB,
145 * byte 1 = start bus number, byte 2 = end bus number
148 #define PHB_DEBUG_STUFF_OFFSET 0x0020
150 #define EMERGENCY_PAGES 32 /* = 128KB */
152 unsigned int specified_table_size = TCE_TABLE_SIZE_UNSPECIFIED;
153 static int translate_empty_slots __read_mostly = 0;
154 static int calgary_detected __read_mostly = 0;
156 static struct rio_table_hdr *rio_table_hdr __initdata;
157 static struct scal_detail *scal_devs[MAX_NUMNODES] __initdata;
158 static struct rio_detail *rio_devs[MAX_NUMNODES * 4] __initdata;
160 struct calgary_bus_info {
162 unsigned char translation_disabled;
167 static void calgary_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev);
168 static void calgary_tce_cache_blast(struct iommu_table *tbl);
169 static void calgary_dump_error_regs(struct iommu_table *tbl);
170 static void calioc2_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev);
171 static void calioc2_tce_cache_blast(struct iommu_table *tbl);
172 static void calioc2_dump_error_regs(struct iommu_table *tbl);
173 static void calgary_init_bitmap_from_tce_table(struct iommu_table *tbl);
174 static void get_tce_space_from_tar(void);
176 static struct cal_chipset_ops calgary_chip_ops = {
177 .handle_quirks = calgary_handle_quirks,
178 .tce_cache_blast = calgary_tce_cache_blast,
179 .dump_error_regs = calgary_dump_error_regs
182 static struct cal_chipset_ops calioc2_chip_ops = {
183 .handle_quirks = calioc2_handle_quirks,
184 .tce_cache_blast = calioc2_tce_cache_blast,
185 .dump_error_regs = calioc2_dump_error_regs
188 static struct calgary_bus_info bus_info[MAX_PHB_BUS_NUM] = { { NULL, 0, 0 }, };
190 static inline int translation_enabled(struct iommu_table *tbl)
192 /* only PHBs with translation enabled have an IOMMU table */
193 return (tbl != NULL);
196 static void iommu_range_reserve(struct iommu_table *tbl,
197 unsigned long start_addr, unsigned int npages)
203 index = start_addr >> PAGE_SHIFT;
205 /* bail out if we're asked to reserve a region we don't cover */
206 if (index >= tbl->it_size)
209 end = index + npages;
210 if (end > tbl->it_size) /* don't go off the table */
213 spin_lock_irqsave(&tbl->it_lock, flags);
215 iommu_area_reserve(tbl->it_map, index, npages);
217 spin_unlock_irqrestore(&tbl->it_lock, flags);
220 static unsigned long iommu_range_alloc(struct device *dev,
221 struct iommu_table *tbl,
225 unsigned long offset;
226 unsigned long boundary_size;
228 boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
229 PAGE_SIZE) >> PAGE_SHIFT;
233 spin_lock_irqsave(&tbl->it_lock, flags);
235 offset = iommu_area_alloc(tbl->it_map, tbl->it_size, tbl->it_hint,
236 npages, 0, boundary_size, 0);
237 if (offset == ~0UL) {
238 tbl->chip_ops->tce_cache_blast(tbl);
240 offset = iommu_area_alloc(tbl->it_map, tbl->it_size, 0,
241 npages, 0, boundary_size, 0);
242 if (offset == ~0UL) {
243 printk(KERN_WARNING "Calgary: IOMMU full.\n");
244 spin_unlock_irqrestore(&tbl->it_lock, flags);
245 if (panic_on_overflow)
246 panic("Calgary: fix the allocator.\n");
248 return DMA_ERROR_CODE;
252 tbl->it_hint = offset + npages;
253 BUG_ON(tbl->it_hint > tbl->it_size);
255 spin_unlock_irqrestore(&tbl->it_lock, flags);
260 static dma_addr_t iommu_alloc(struct device *dev, struct iommu_table *tbl,
261 void *vaddr, unsigned int npages, int direction)
264 dma_addr_t ret = DMA_ERROR_CODE;
266 entry = iommu_range_alloc(dev, tbl, npages);
268 if (unlikely(entry == DMA_ERROR_CODE))
271 /* set the return dma address */
272 ret = (entry << PAGE_SHIFT) | ((unsigned long)vaddr & ~PAGE_MASK);
274 /* put the TCEs in the HW table */
275 tce_build(tbl, entry, npages, (unsigned long)vaddr & PAGE_MASK,
281 printk(KERN_WARNING "Calgary: failed to allocate %u pages in "
282 "iommu %p\n", npages, tbl);
283 return DMA_ERROR_CODE;
286 static void iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr,
290 unsigned long badend;
293 /* were we called with bad_dma_address? */
294 badend = DMA_ERROR_CODE + (EMERGENCY_PAGES * PAGE_SIZE);
295 if (unlikely((dma_addr >= DMA_ERROR_CODE) && (dma_addr < badend))) {
296 WARN(1, KERN_ERR "Calgary: driver tried unmapping bad DMA "
297 "address 0x%Lx\n", dma_addr);
301 entry = dma_addr >> PAGE_SHIFT;
303 BUG_ON(entry + npages > tbl->it_size);
305 tce_free(tbl, entry, npages);
307 spin_lock_irqsave(&tbl->it_lock, flags);
309 iommu_area_free(tbl->it_map, entry, npages);
311 spin_unlock_irqrestore(&tbl->it_lock, flags);
314 static inline struct iommu_table *find_iommu_table(struct device *dev)
316 struct pci_dev *pdev;
317 struct pci_bus *pbus;
318 struct iommu_table *tbl;
320 pdev = to_pci_dev(dev);
324 /* is the device behind a bridge? Look for the root bus */
328 tbl = pci_iommu(pbus);
330 BUG_ON(tbl && (tbl->it_busno != pbus->number));
335 static void calgary_unmap_sg(struct device *dev, struct scatterlist *sglist,
336 int nelems,enum dma_data_direction dir,
337 struct dma_attrs *attrs)
339 struct iommu_table *tbl = find_iommu_table(dev);
340 struct scatterlist *s;
343 if (!translation_enabled(tbl))
346 for_each_sg(sglist, s, nelems, i) {
348 dma_addr_t dma = s->dma_address;
349 unsigned int dmalen = s->dma_length;
354 npages = iommu_num_pages(dma, dmalen, PAGE_SIZE);
355 iommu_free(tbl, dma, npages);
359 static int calgary_map_sg(struct device *dev, struct scatterlist *sg,
360 int nelems, enum dma_data_direction dir,
361 struct dma_attrs *attrs)
363 struct iommu_table *tbl = find_iommu_table(dev);
364 struct scatterlist *s;
370 for_each_sg(sg, s, nelems, i) {
373 vaddr = (unsigned long) sg_virt(s);
374 npages = iommu_num_pages(vaddr, s->length, PAGE_SIZE);
376 entry = iommu_range_alloc(dev, tbl, npages);
377 if (entry == DMA_ERROR_CODE) {
378 /* makes sure unmap knows to stop */
383 s->dma_address = (entry << PAGE_SHIFT) | s->offset;
385 /* insert into HW table */
386 tce_build(tbl, entry, npages, vaddr & PAGE_MASK, dir);
388 s->dma_length = s->length;
393 calgary_unmap_sg(dev, sg, nelems, dir, NULL);
394 for_each_sg(sg, s, nelems, i) {
395 sg->dma_address = DMA_ERROR_CODE;
401 static dma_addr_t calgary_map_page(struct device *dev, struct page *page,
402 unsigned long offset, size_t size,
403 enum dma_data_direction dir,
404 struct dma_attrs *attrs)
406 void *vaddr = page_address(page) + offset;
409 struct iommu_table *tbl = find_iommu_table(dev);
411 uaddr = (unsigned long)vaddr;
412 npages = iommu_num_pages(uaddr, size, PAGE_SIZE);
414 return iommu_alloc(dev, tbl, vaddr, npages, dir);
417 static void calgary_unmap_page(struct device *dev, dma_addr_t dma_addr,
418 size_t size, enum dma_data_direction dir,
419 struct dma_attrs *attrs)
421 struct iommu_table *tbl = find_iommu_table(dev);
424 npages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
425 iommu_free(tbl, dma_addr, npages);
428 static void* calgary_alloc_coherent(struct device *dev, size_t size,
429 dma_addr_t *dma_handle, gfp_t flag)
433 unsigned int npages, order;
434 struct iommu_table *tbl = find_iommu_table(dev);
436 size = PAGE_ALIGN(size); /* size rounded up to full pages */
437 npages = size >> PAGE_SHIFT;
438 order = get_order(size);
440 flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
442 /* alloc enough pages (and possibly more) */
443 ret = (void *)__get_free_pages(flag, order);
446 memset(ret, 0, size);
448 /* set up tces to cover the allocated range */
449 mapping = iommu_alloc(dev, tbl, ret, npages, DMA_BIDIRECTIONAL);
450 if (mapping == DMA_ERROR_CODE)
452 *dma_handle = mapping;
455 free_pages((unsigned long)ret, get_order(size));
461 static void calgary_free_coherent(struct device *dev, size_t size,
462 void *vaddr, dma_addr_t dma_handle)
465 struct iommu_table *tbl = find_iommu_table(dev);
467 size = PAGE_ALIGN(size);
468 npages = size >> PAGE_SHIFT;
470 iommu_free(tbl, dma_handle, npages);
471 free_pages((unsigned long)vaddr, get_order(size));
474 static struct dma_map_ops calgary_dma_ops = {
475 .alloc_coherent = calgary_alloc_coherent,
476 .free_coherent = calgary_free_coherent,
477 .map_sg = calgary_map_sg,
478 .unmap_sg = calgary_unmap_sg,
479 .map_page = calgary_map_page,
480 .unmap_page = calgary_unmap_page,
483 static inline void __iomem * busno_to_bbar(unsigned char num)
485 return bus_info[num].bbar;
488 static inline int busno_to_phbid(unsigned char num)
490 return bus_info[num].phbid;
493 static inline unsigned long split_queue_offset(unsigned char num)
495 size_t idx = busno_to_phbid(num);
497 return split_queue_offsets[idx];
500 static inline unsigned long tar_offset(unsigned char num)
502 size_t idx = busno_to_phbid(num);
504 return tar_offsets[idx];
507 static inline unsigned long phb_offset(unsigned char num)
509 size_t idx = busno_to_phbid(num);
511 return phb_offsets[idx];
514 static inline void __iomem* calgary_reg(void __iomem *bar, unsigned long offset)
516 unsigned long target = ((unsigned long)bar) | offset;
517 return (void __iomem*)target;
520 static inline int is_calioc2(unsigned short device)
522 return (device == PCI_DEVICE_ID_IBM_CALIOC2);
525 static inline int is_calgary(unsigned short device)
527 return (device == PCI_DEVICE_ID_IBM_CALGARY);
530 static inline int is_cal_pci_dev(unsigned short device)
532 return (is_calgary(device) || is_calioc2(device));
535 static void calgary_tce_cache_blast(struct iommu_table *tbl)
540 void __iomem *bbar = tbl->bbar;
541 void __iomem *target;
543 /* disable arbitration on the bus */
544 target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_AER_OFFSET);
548 /* read plssr to ensure it got there */
549 target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_PLSSR_OFFSET);
552 /* poll split queues until all DMA activity is done */
553 target = calgary_reg(bbar, split_queue_offset(tbl->it_busno));
557 } while ((val & 0xff) != 0xff && i < 100);
559 printk(KERN_WARNING "Calgary: PCI bus not quiesced, "
560 "continuing anyway\n");
562 /* invalidate TCE cache */
563 target = calgary_reg(bbar, tar_offset(tbl->it_busno));
564 writeq(tbl->tar_val, target);
566 /* enable arbitration */
567 target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_AER_OFFSET);
569 (void)readl(target); /* flush */
572 static void calioc2_tce_cache_blast(struct iommu_table *tbl)
574 void __iomem *bbar = tbl->bbar;
575 void __iomem *target;
580 unsigned char bus = tbl->it_busno;
583 printk(KERN_DEBUG "Calgary: CalIOC2 bus 0x%x entering tce cache blast "
584 "sequence - count %d\n", bus, count);
586 /* 1. using the Page Migration Control reg set SoftStop */
587 target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_CTRL);
588 val = be32_to_cpu(readl(target));
589 printk(KERN_DEBUG "1a. read 0x%x [LE] from %p\n", val, target);
591 printk(KERN_DEBUG "1b. writing 0x%x [LE] to %p\n", val, target);
592 writel(cpu_to_be32(val), target);
594 /* 2. poll split queues until all DMA activity is done */
595 printk(KERN_DEBUG "2a. starting to poll split queues\n");
596 target = calgary_reg(bbar, split_queue_offset(bus));
598 val64 = readq(target);
600 } while ((val64 & 0xff) != 0xff && i < 100);
602 printk(KERN_WARNING "CalIOC2: PCI bus not quiesced, "
603 "continuing anyway\n");
605 /* 3. poll Page Migration DEBUG for SoftStopFault */
606 target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_DEBUG);
607 val = be32_to_cpu(readl(target));
608 printk(KERN_DEBUG "3. read 0x%x [LE] from %p\n", val, target);
610 /* 4. if SoftStopFault - goto (1) */
611 if (val & PMR_SOFTSTOPFAULT) {
615 printk(KERN_WARNING "CalIOC2: too many SoftStopFaults, "
616 "aborting TCE cache flush sequence!\n");
617 return; /* pray for the best */
621 /* 5. Slam into HardStop by reading PHB_PAGE_MIG_CTRL */
622 target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_CTRL);
623 printk(KERN_DEBUG "5a. slamming into HardStop by reading %p\n", target);
624 val = be32_to_cpu(readl(target));
625 printk(KERN_DEBUG "5b. read 0x%x [LE] from %p\n", val, target);
626 target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_DEBUG);
627 val = be32_to_cpu(readl(target));
628 printk(KERN_DEBUG "5c. read 0x%x [LE] from %p (debug)\n", val, target);
630 /* 6. invalidate TCE cache */
631 printk(KERN_DEBUG "6. invalidating TCE cache\n");
632 target = calgary_reg(bbar, tar_offset(bus));
633 writeq(tbl->tar_val, target);
635 /* 7. Re-read PMCR */
636 printk(KERN_DEBUG "7a. Re-reading PMCR\n");
637 target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_CTRL);
638 val = be32_to_cpu(readl(target));
639 printk(KERN_DEBUG "7b. read 0x%x [LE] from %p\n", val, target);
641 /* 8. Remove HardStop */
642 printk(KERN_DEBUG "8a. removing HardStop from PMCR\n");
643 target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_CTRL);
645 printk(KERN_DEBUG "8b. writing 0x%x [LE] to %p\n", val, target);
646 writel(cpu_to_be32(val), target);
647 val = be32_to_cpu(readl(target));
648 printk(KERN_DEBUG "8c. read 0x%x [LE] from %p\n", val, target);
651 static void __init calgary_reserve_mem_region(struct pci_dev *dev, u64 start,
654 unsigned int numpages;
656 limit = limit | 0xfffff;
659 numpages = ((limit - start) >> PAGE_SHIFT);
660 iommu_range_reserve(pci_iommu(dev->bus), start, numpages);
663 static void __init calgary_reserve_peripheral_mem_1(struct pci_dev *dev)
665 void __iomem *target;
666 u64 low, high, sizelow;
668 struct iommu_table *tbl = pci_iommu(dev->bus);
669 unsigned char busnum = dev->bus->number;
670 void __iomem *bbar = tbl->bbar;
672 /* peripheral MEM_1 region */
673 target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_1_LOW);
674 low = be32_to_cpu(readl(target));
675 target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_1_HIGH);
676 high = be32_to_cpu(readl(target));
677 target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_1_SIZE);
678 sizelow = be32_to_cpu(readl(target));
680 start = (high << 32) | low;
683 calgary_reserve_mem_region(dev, start, limit);
686 static void __init calgary_reserve_peripheral_mem_2(struct pci_dev *dev)
688 void __iomem *target;
690 u64 low, high, sizelow, sizehigh;
692 struct iommu_table *tbl = pci_iommu(dev->bus);
693 unsigned char busnum = dev->bus->number;
694 void __iomem *bbar = tbl->bbar;
697 target = calgary_reg(bbar, phb_offset(busnum) | PHB_CONFIG_RW_OFFSET);
698 val32 = be32_to_cpu(readl(target));
699 if (!(val32 & PHB_MEM2_ENABLE))
702 target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_2_LOW);
703 low = be32_to_cpu(readl(target));
704 target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_2_HIGH);
705 high = be32_to_cpu(readl(target));
706 target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_2_SIZE_LOW);
707 sizelow = be32_to_cpu(readl(target));
708 target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_2_SIZE_HIGH);
709 sizehigh = be32_to_cpu(readl(target));
711 start = (high << 32) | low;
712 limit = (sizehigh << 32) | sizelow;
714 calgary_reserve_mem_region(dev, start, limit);
718 * some regions of the IO address space do not get translated, so we
719 * must not give devices IO addresses in those regions. The regions
720 * are the 640KB-1MB region and the two PCI peripheral memory holes.
721 * Reserve all of them in the IOMMU bitmap to avoid giving them out
724 static void __init calgary_reserve_regions(struct pci_dev *dev)
728 struct iommu_table *tbl = pci_iommu(dev->bus);
730 /* reserve EMERGENCY_PAGES from bad_dma_address and up */
731 iommu_range_reserve(tbl, DMA_ERROR_CODE, EMERGENCY_PAGES);
733 /* avoid the BIOS/VGA first 640KB-1MB region */
734 /* for CalIOC2 - avoid the entire first MB */
735 if (is_calgary(dev->device)) {
736 start = (640 * 1024);
737 npages = ((1024 - 640) * 1024) >> PAGE_SHIFT;
738 } else { /* calioc2 */
740 npages = (1 * 1024 * 1024) >> PAGE_SHIFT;
742 iommu_range_reserve(tbl, start, npages);
744 /* reserve the two PCI peripheral memory regions in IO space */
745 calgary_reserve_peripheral_mem_1(dev);
746 calgary_reserve_peripheral_mem_2(dev);
749 static int __init calgary_setup_tar(struct pci_dev *dev, void __iomem *bbar)
753 void __iomem *target;
755 struct iommu_table *tbl;
757 /* build TCE tables for each PHB */
758 ret = build_tce_table(dev, bbar);
762 tbl = pci_iommu(dev->bus);
763 tbl->it_base = (unsigned long)bus_info[dev->bus->number].tce_space;
765 if (is_kdump_kernel())
766 calgary_init_bitmap_from_tce_table(tbl);
768 tce_free(tbl, 0, tbl->it_size);
770 if (is_calgary(dev->device))
771 tbl->chip_ops = &calgary_chip_ops;
772 else if (is_calioc2(dev->device))
773 tbl->chip_ops = &calioc2_chip_ops;
777 calgary_reserve_regions(dev);
779 /* set TARs for each PHB */
780 target = calgary_reg(bbar, tar_offset(dev->bus->number));
781 val64 = be64_to_cpu(readq(target));
783 /* zero out all TAR bits under sw control */
784 val64 &= ~TAR_SW_BITS;
785 table_phys = (u64)__pa(tbl->it_base);
789 BUG_ON(specified_table_size > TCE_TABLE_SIZE_8M);
790 val64 |= (u64) specified_table_size;
792 tbl->tar_val = cpu_to_be64(val64);
794 writeq(tbl->tar_val, target);
795 readq(target); /* flush */
800 static void __init calgary_free_bus(struct pci_dev *dev)
803 struct iommu_table *tbl = pci_iommu(dev->bus);
804 void __iomem *target;
805 unsigned int bitmapsz;
807 target = calgary_reg(tbl->bbar, tar_offset(dev->bus->number));
808 val64 = be64_to_cpu(readq(target));
809 val64 &= ~TAR_SW_BITS;
810 writeq(cpu_to_be64(val64), target);
811 readq(target); /* flush */
813 bitmapsz = tbl->it_size / BITS_PER_BYTE;
814 free_pages((unsigned long)tbl->it_map, get_order(bitmapsz));
819 set_pci_iommu(dev->bus, NULL);
821 /* Can't free bootmem allocated memory after system is up :-( */
822 bus_info[dev->bus->number].tce_space = NULL;
825 static void calgary_dump_error_regs(struct iommu_table *tbl)
827 void __iomem *bbar = tbl->bbar;
828 void __iomem *target;
831 target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_CSR_OFFSET);
832 csr = be32_to_cpu(readl(target));
834 target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_PLSSR_OFFSET);
835 plssr = be32_to_cpu(readl(target));
837 /* If no error, the agent ID in the CSR is not valid */
838 printk(KERN_EMERG "Calgary: DMA error on Calgary PHB 0x%x, "
839 "0x%08x@CSR 0x%08x@PLSSR\n", tbl->it_busno, csr, plssr);
842 static void calioc2_dump_error_regs(struct iommu_table *tbl)
844 void __iomem *bbar = tbl->bbar;
845 u32 csr, csmr, plssr, mck, rcstat;
846 void __iomem *target;
847 unsigned long phboff = phb_offset(tbl->it_busno);
848 unsigned long erroff;
853 target = calgary_reg(bbar, phboff | PHB_CSR_OFFSET);
854 csr = be32_to_cpu(readl(target));
856 target = calgary_reg(bbar, phboff | PHB_PLSSR_OFFSET);
857 plssr = be32_to_cpu(readl(target));
859 target = calgary_reg(bbar, phboff | 0x290);
860 csmr = be32_to_cpu(readl(target));
862 target = calgary_reg(bbar, phboff | 0x800);
863 mck = be32_to_cpu(readl(target));
865 printk(KERN_EMERG "Calgary: DMA error on CalIOC2 PHB 0x%x\n",
868 printk(KERN_EMERG "Calgary: 0x%08x@CSR 0x%08x@PLSSR 0x%08x@CSMR 0x%08x@MCK\n",
869 csr, plssr, csmr, mck);
871 /* dump rest of error regs */
872 printk(KERN_EMERG "Calgary: ");
873 for (i = 0; i < ARRAY_SIZE(errregs); i++) {
874 /* err regs are at 0x810 - 0x870 */
875 erroff = (0x810 + (i * 0x10));
876 target = calgary_reg(bbar, phboff | erroff);
877 errregs[i] = be32_to_cpu(readl(target));
878 printk("0x%08x@0x%lx ", errregs[i], erroff);
882 /* root complex status */
883 target = calgary_reg(bbar, phboff | PHB_ROOT_COMPLEX_STATUS);
884 rcstat = be32_to_cpu(readl(target));
885 printk(KERN_EMERG "Calgary: 0x%08x@0x%x\n", rcstat,
886 PHB_ROOT_COMPLEX_STATUS);
889 static void calgary_watchdog(unsigned long data)
891 struct pci_dev *dev = (struct pci_dev *)data;
892 struct iommu_table *tbl = pci_iommu(dev->bus);
893 void __iomem *bbar = tbl->bbar;
895 void __iomem *target;
897 target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_CSR_OFFSET);
898 val32 = be32_to_cpu(readl(target));
900 /* If no error, the agent ID in the CSR is not valid */
901 if (val32 & CSR_AGENT_MASK) {
902 tbl->chip_ops->dump_error_regs(tbl);
907 /* Disable bus that caused the error */
908 target = calgary_reg(bbar, phb_offset(tbl->it_busno) |
909 PHB_CONFIG_RW_OFFSET);
910 val32 = be32_to_cpu(readl(target));
911 val32 |= PHB_SLOT_DISABLE;
912 writel(cpu_to_be32(val32), target);
913 readl(target); /* flush */
915 /* Reset the timer */
916 mod_timer(&tbl->watchdog_timer, jiffies + 2 * HZ);
920 static void __init calgary_set_split_completion_timeout(void __iomem *bbar,
921 unsigned char busnum, unsigned long timeout)
924 void __iomem *target;
925 unsigned int phb_shift = ~0; /* silence gcc */
928 switch (busno_to_phbid(busnum)) {
929 case 0: phb_shift = (63 - 19);
931 case 1: phb_shift = (63 - 23);
933 case 2: phb_shift = (63 - 27);
935 case 3: phb_shift = (63 - 35);
938 BUG_ON(busno_to_phbid(busnum));
941 target = calgary_reg(bbar, CALGARY_CONFIG_REG);
942 val64 = be64_to_cpu(readq(target));
944 /* zero out this PHB's timer bits */
945 mask = ~(0xFUL << phb_shift);
947 val64 |= (timeout << phb_shift);
948 writeq(cpu_to_be64(val64), target);
949 readq(target); /* flush */
952 static void __init calioc2_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev)
954 unsigned char busnum = dev->bus->number;
955 void __iomem *bbar = tbl->bbar;
956 void __iomem *target;
960 * CalIOC2 designers recommend setting bit 8 in 0xnDB0 to 1
962 target = calgary_reg(bbar, phb_offset(busnum) | PHB_SAVIOR_L2);
963 val = cpu_to_be32(readl(target));
965 writel(cpu_to_be32(val), target);
968 static void __init calgary_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev)
970 unsigned char busnum = dev->bus->number;
973 * Give split completion a longer timeout on bus 1 for aic94xx
974 * http://bugzilla.kernel.org/show_bug.cgi?id=7180
976 if (is_calgary(dev->device) && (busnum == 1))
977 calgary_set_split_completion_timeout(tbl->bbar, busnum,
981 static void __init calgary_enable_translation(struct pci_dev *dev)
984 unsigned char busnum;
985 void __iomem *target;
987 struct iommu_table *tbl;
989 busnum = dev->bus->number;
990 tbl = pci_iommu(dev->bus);
993 /* enable TCE in PHB Config Register */
994 target = calgary_reg(bbar, phb_offset(busnum) | PHB_CONFIG_RW_OFFSET);
995 val32 = be32_to_cpu(readl(target));
996 val32 |= PHB_TCE_ENABLE | PHB_DAC_DISABLE | PHB_MCSR_ENABLE;
998 printk(KERN_INFO "Calgary: enabling translation on %s PHB %#x\n",
999 (dev->device == PCI_DEVICE_ID_IBM_CALGARY) ?
1000 "Calgary" : "CalIOC2", busnum);
1001 printk(KERN_INFO "Calgary: errant DMAs will now be prevented on this "
1004 writel(cpu_to_be32(val32), target);
1005 readl(target); /* flush */
1007 init_timer(&tbl->watchdog_timer);
1008 tbl->watchdog_timer.function = &calgary_watchdog;
1009 tbl->watchdog_timer.data = (unsigned long)dev;
1010 mod_timer(&tbl->watchdog_timer, jiffies);
1013 static void __init calgary_disable_translation(struct pci_dev *dev)
1016 unsigned char busnum;
1017 void __iomem *target;
1019 struct iommu_table *tbl;
1021 busnum = dev->bus->number;
1022 tbl = pci_iommu(dev->bus);
1025 /* disable TCE in PHB Config Register */
1026 target = calgary_reg(bbar, phb_offset(busnum) | PHB_CONFIG_RW_OFFSET);
1027 val32 = be32_to_cpu(readl(target));
1028 val32 &= ~(PHB_TCE_ENABLE | PHB_DAC_DISABLE | PHB_MCSR_ENABLE);
1030 printk(KERN_INFO "Calgary: disabling translation on PHB %#x!\n", busnum);
1031 writel(cpu_to_be32(val32), target);
1032 readl(target); /* flush */
1034 del_timer_sync(&tbl->watchdog_timer);
1037 static void __init calgary_init_one_nontraslated(struct pci_dev *dev)
1040 set_pci_iommu(dev->bus, NULL);
1042 /* is the device behind a bridge? */
1043 if (dev->bus->parent)
1044 dev->bus->parent->self = dev;
1046 dev->bus->self = dev;
1049 static int __init calgary_init_one(struct pci_dev *dev)
1052 struct iommu_table *tbl;
1055 BUG_ON(dev->bus->number >= MAX_PHB_BUS_NUM);
1057 bbar = busno_to_bbar(dev->bus->number);
1058 ret = calgary_setup_tar(dev, bbar);
1064 if (dev->bus->parent) {
1065 if (dev->bus->parent->self)
1066 printk(KERN_WARNING "Calgary: IEEEE, dev %p has "
1067 "bus->parent->self!\n", dev);
1068 dev->bus->parent->self = dev;
1070 dev->bus->self = dev;
1072 tbl = pci_iommu(dev->bus);
1073 tbl->chip_ops->handle_quirks(tbl, dev);
1075 calgary_enable_translation(dev);
1083 static int __init calgary_locate_bbars(void)
1086 int rioidx, phb, bus;
1088 void __iomem *target;
1089 unsigned long offset;
1090 u8 start_bus, end_bus;
1094 for (rioidx = 0; rioidx < rio_table_hdr->num_rio_dev; rioidx++) {
1095 struct rio_detail *rio = rio_devs[rioidx];
1097 if ((rio->type != COMPAT_CALGARY) && (rio->type != ALT_CALGARY))
1100 /* map entire 1MB of Calgary config space */
1101 bbar = ioremap_nocache(rio->BBAR, 1024 * 1024);
1105 for (phb = 0; phb < PHBS_PER_CALGARY; phb++) {
1106 offset = phb_debug_offsets[phb] | PHB_DEBUG_STUFF_OFFSET;
1107 target = calgary_reg(bbar, offset);
1109 val = be32_to_cpu(readl(target));
1111 start_bus = (u8)((val & 0x00FF0000) >> 16);
1112 end_bus = (u8)((val & 0x0000FF00) >> 8);
1115 for (bus = start_bus; bus <= end_bus; bus++) {
1116 bus_info[bus].bbar = bbar;
1117 bus_info[bus].phbid = phb;
1120 bus_info[start_bus].bbar = bbar;
1121 bus_info[start_bus].phbid = phb;
1129 /* scan bus_info and iounmap any bbars we previously ioremap'd */
1130 for (bus = 0; bus < ARRAY_SIZE(bus_info); bus++)
1131 if (bus_info[bus].bbar)
1132 iounmap(bus_info[bus].bbar);
1137 static int __init calgary_init(void)
1140 struct pci_dev *dev = NULL;
1141 struct calgary_bus_info *info;
1143 ret = calgary_locate_bbars();
1147 /* Purely for kdump kernel case */
1148 if (is_kdump_kernel())
1149 get_tce_space_from_tar();
1152 dev = pci_get_device(PCI_VENDOR_ID_IBM, PCI_ANY_ID, dev);
1155 if (!is_cal_pci_dev(dev->device))
1158 info = &bus_info[dev->bus->number];
1159 if (info->translation_disabled) {
1160 calgary_init_one_nontraslated(dev);
1164 if (!info->tce_space && !translate_empty_slots)
1167 ret = calgary_init_one(dev);
1173 for_each_pci_dev(dev) {
1174 struct iommu_table *tbl;
1176 tbl = find_iommu_table(&dev->dev);
1178 if (translation_enabled(tbl))
1179 dev->dev.archdata.dma_ops = &calgary_dma_ops;
1186 dev = pci_get_device(PCI_VENDOR_ID_IBM, PCI_ANY_ID, dev);
1189 if (!is_cal_pci_dev(dev->device))
1192 info = &bus_info[dev->bus->number];
1193 if (info->translation_disabled) {
1197 if (!info->tce_space && !translate_empty_slots)
1200 calgary_disable_translation(dev);
1201 calgary_free_bus(dev);
1202 pci_dev_put(dev); /* Undo calgary_init_one()'s pci_dev_get() */
1203 dev->dev.archdata.dma_ops = NULL;
1209 static inline int __init determine_tce_table_size(u64 ram)
1213 if (specified_table_size != TCE_TABLE_SIZE_UNSPECIFIED)
1214 return specified_table_size;
1217 * Table sizes are from 0 to 7 (TCE_TABLE_SIZE_64K to
1218 * TCE_TABLE_SIZE_8M). Table size 0 has 8K entries and each
1219 * larger table size has twice as many entries, so shift the
1220 * max ram address by 13 to divide by 8K and then look at the
1221 * order of the result to choose between 0-7.
1223 ret = get_order(ram >> 13);
1224 if (ret > TCE_TABLE_SIZE_8M)
1225 ret = TCE_TABLE_SIZE_8M;
1230 static int __init build_detail_arrays(void)
1233 unsigned numnodes, i;
1234 int scal_detail_size, rio_detail_size;
1236 numnodes = rio_table_hdr->num_scal_dev;
1237 if (numnodes > MAX_NUMNODES){
1239 "Calgary: MAX_NUMNODES too low! Defined as %d, "
1240 "but system has %d nodes.\n",
1241 MAX_NUMNODES, numnodes);
1245 switch (rio_table_hdr->version){
1247 scal_detail_size = 11;
1248 rio_detail_size = 13;
1251 scal_detail_size = 12;
1252 rio_detail_size = 15;
1256 "Calgary: Invalid Rio Grande Table Version: %d\n",
1257 rio_table_hdr->version);
1261 ptr = ((unsigned long)rio_table_hdr) + 3;
1262 for (i = 0; i < numnodes; i++, ptr += scal_detail_size)
1263 scal_devs[i] = (struct scal_detail *)ptr;
1265 for (i = 0; i < rio_table_hdr->num_rio_dev;
1266 i++, ptr += rio_detail_size)
1267 rio_devs[i] = (struct rio_detail *)ptr;
1272 static int __init calgary_bus_has_devices(int bus, unsigned short pci_dev)
1277 if (pci_dev == PCI_DEVICE_ID_IBM_CALIOC2) {
1279 * FIXME: properly scan for devices accross the
1280 * PCI-to-PCI bridge on every CalIOC2 port.
1285 for (dev = 1; dev < 8; dev++) {
1286 val = read_pci_config(bus, dev, 0, 0);
1287 if (val != 0xffffffff)
1290 return (val != 0xffffffff);
1294 * calgary_init_bitmap_from_tce_table():
1295 * Funtion for kdump case. In the second/kdump kernel initialize
1296 * the bitmap based on the tce table entries obtained from first kernel
1298 static void calgary_init_bitmap_from_tce_table(struct iommu_table *tbl)
1302 tp = ((u64 *)tbl->it_base);
1303 for (index = 0 ; index < tbl->it_size; index++) {
1305 set_bit(index, tbl->it_map);
1311 * get_tce_space_from_tar():
1312 * Function for kdump case. Get the tce tables from first kernel
1313 * by reading the contents of the base adress register of calgary iommu
1315 static void __init get_tce_space_from_tar(void)
1318 void __iomem *target;
1319 unsigned long tce_space;
1321 for (bus = 0; bus < MAX_PHB_BUS_NUM; bus++) {
1322 struct calgary_bus_info *info = &bus_info[bus];
1323 unsigned short pci_device;
1326 val = read_pci_config(bus, 0, 0, 0);
1327 pci_device = (val & 0xFFFF0000) >> 16;
1329 if (!is_cal_pci_dev(pci_device))
1331 if (info->translation_disabled)
1334 if (calgary_bus_has_devices(bus, pci_device) ||
1335 translate_empty_slots) {
1336 target = calgary_reg(bus_info[bus].bbar,
1338 tce_space = be64_to_cpu(readq(target));
1339 tce_space = tce_space & TAR_SW_BITS;
1341 tce_space = tce_space & (~specified_table_size);
1342 info->tce_space = (u64 *)__va(tce_space);
1348 static int __init calgary_iommu_init(void)
1352 /* ok, we're trying to use Calgary - let's roll */
1353 printk(KERN_INFO "PCI-DMA: Using Calgary IOMMU\n");
1355 ret = calgary_init();
1357 printk(KERN_ERR "PCI-DMA: Calgary init failed %d, "
1358 "falling back to no_iommu\n", ret);
1365 void __init detect_calgary(void)
1369 int calgary_found = 0;
1371 unsigned int offset, prev_offset;
1375 * if the user specified iommu=off or iommu=soft or we found
1376 * another HW IOMMU already, bail out.
1378 if (no_iommu || iommu_detected)
1384 if (!early_pci_allowed())
1387 printk(KERN_DEBUG "Calgary: detecting Calgary via BIOS EBDA area\n");
1389 ptr = (unsigned long)phys_to_virt(get_bios_ebda());
1391 rio_table_hdr = NULL;
1395 * The next offset is stored in the 1st word.
1396 * Only parse up until the offset increases:
1398 while (offset > prev_offset) {
1399 /* The block id is stored in the 2nd word */
1400 if (*((unsigned short *)(ptr + offset + 2)) == 0x4752){
1401 /* set the pointer past the offset & block id */
1402 rio_table_hdr = (struct rio_table_hdr *)(ptr + offset + 4);
1405 prev_offset = offset;
1406 offset = *((unsigned short *)(ptr + offset));
1408 if (!rio_table_hdr) {
1409 printk(KERN_DEBUG "Calgary: Unable to locate Rio Grande table "
1410 "in EBDA - bailing!\n");
1414 ret = build_detail_arrays();
1416 printk(KERN_DEBUG "Calgary: build_detail_arrays ret %d\n", ret);
1420 specified_table_size = determine_tce_table_size((is_kdump_kernel() ?
1421 saved_max_pfn : max_pfn) * PAGE_SIZE);
1423 for (bus = 0; bus < MAX_PHB_BUS_NUM; bus++) {
1424 struct calgary_bus_info *info = &bus_info[bus];
1425 unsigned short pci_device;
1428 val = read_pci_config(bus, 0, 0, 0);
1429 pci_device = (val & 0xFFFF0000) >> 16;
1431 if (!is_cal_pci_dev(pci_device))
1434 if (info->translation_disabled)
1437 if (calgary_bus_has_devices(bus, pci_device) ||
1438 translate_empty_slots) {
1440 * If it is kdump kernel, find and use tce tables
1441 * from first kernel, else allocate tce tables here
1443 if (!is_kdump_kernel()) {
1444 tbl = alloc_tce_table();
1447 info->tce_space = tbl;
1453 printk(KERN_DEBUG "Calgary: finished detection, Calgary %s\n",
1454 calgary_found ? "found" : "not found");
1456 if (calgary_found) {
1458 calgary_detected = 1;
1459 printk(KERN_INFO "PCI-DMA: Calgary IOMMU detected.\n");
1460 printk(KERN_INFO "PCI-DMA: Calgary TCE table spec is %d\n",
1461 specified_table_size);
1463 x86_init.iommu.iommu_init = calgary_iommu_init;
1468 for (--bus; bus >= 0; --bus) {
1469 struct calgary_bus_info *info = &bus_info[bus];
1471 if (info->tce_space)
1472 free_tce_table(info->tce_space);
1476 static int __init calgary_parse_options(char *p)
1478 unsigned int bridge;
1483 if (!strncmp(p, "64k", 3))
1484 specified_table_size = TCE_TABLE_SIZE_64K;
1485 else if (!strncmp(p, "128k", 4))
1486 specified_table_size = TCE_TABLE_SIZE_128K;
1487 else if (!strncmp(p, "256k", 4))
1488 specified_table_size = TCE_TABLE_SIZE_256K;
1489 else if (!strncmp(p, "512k", 4))
1490 specified_table_size = TCE_TABLE_SIZE_512K;
1491 else if (!strncmp(p, "1M", 2))
1492 specified_table_size = TCE_TABLE_SIZE_1M;
1493 else if (!strncmp(p, "2M", 2))
1494 specified_table_size = TCE_TABLE_SIZE_2M;
1495 else if (!strncmp(p, "4M", 2))
1496 specified_table_size = TCE_TABLE_SIZE_4M;
1497 else if (!strncmp(p, "8M", 2))
1498 specified_table_size = TCE_TABLE_SIZE_8M;
1500 len = strlen("translate_empty_slots");
1501 if (!strncmp(p, "translate_empty_slots", len))
1502 translate_empty_slots = 1;
1504 len = strlen("disable");
1505 if (!strncmp(p, "disable", len)) {
1511 bridge = simple_strtoul(p, &endp, 0);
1515 if (bridge < MAX_PHB_BUS_NUM) {
1516 printk(KERN_INFO "Calgary: disabling "
1517 "translation for PHB %#x\n", bridge);
1518 bus_info[bridge].translation_disabled = 1;
1522 p = strpbrk(p, ",");
1530 __setup("calgary=", calgary_parse_options);
1532 static void __init calgary_fixup_one_tce_space(struct pci_dev *dev)
1534 struct iommu_table *tbl;
1535 unsigned int npages;
1538 tbl = pci_iommu(dev->bus);
1540 for (i = 0; i < 4; i++) {
1541 struct resource *r = &dev->resource[PCI_BRIDGE_RESOURCES + i];
1543 /* Don't give out TCEs that map MEM resources */
1544 if (!(r->flags & IORESOURCE_MEM))
1547 /* 0-based? we reserve the whole 1st MB anyway */
1551 /* cover the whole region */
1552 npages = (r->end - r->start) >> PAGE_SHIFT;
1555 iommu_range_reserve(tbl, r->start, npages);
1559 static int __init calgary_fixup_tce_spaces(void)
1561 struct pci_dev *dev = NULL;
1562 struct calgary_bus_info *info;
1564 if (no_iommu || swiotlb || !calgary_detected)
1567 printk(KERN_DEBUG "Calgary: fixing up tce spaces\n");
1570 dev = pci_get_device(PCI_VENDOR_ID_IBM, PCI_ANY_ID, dev);
1573 if (!is_cal_pci_dev(dev->device))
1576 info = &bus_info[dev->bus->number];
1577 if (info->translation_disabled)
1580 if (!info->tce_space)
1583 calgary_fixup_one_tce_space(dev);
1591 * We need to be call after pcibios_assign_resources (fs_initcall level)
1592 * and before device_initcall.
1594 rootfs_initcall(calgary_fixup_tce_spaces);