2 * srmmu.c: SRMMU specific routines for memory management.
4 * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu)
5 * Copyright (C) 1995,2002 Pete Zaitcev (zaitcev@yahoo.com)
6 * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be)
7 * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
8 * Copyright (C) 1999,2000 Anton Blanchard (anton@samba.org)
11 #include <linux/kernel.h>
13 #include <linux/vmalloc.h>
14 #include <linux/pagemap.h>
15 #include <linux/init.h>
16 #include <linux/spinlock.h>
17 #include <linux/bootmem.h>
19 #include <linux/seq_file.h>
20 #include <linux/kdebug.h>
21 #include <linux/log2.h>
22 #include <linux/gfp.h>
24 #include <asm/bitext.h>
26 #include <asm/pgalloc.h>
27 #include <asm/pgtable.h>
29 #include <asm/vaddrs.h>
30 #include <asm/traps.h>
33 #include <asm/cache.h>
34 #include <asm/oplib.h>
37 #include <asm/mmu_context.h>
38 #include <asm/io-unit.h>
39 #include <asm/cacheflush.h>
40 #include <asm/tlbflush.h>
42 /* Now the cpu specific definitions. */
43 #include <asm/viking.h>
46 #include <asm/tsunami.h>
47 #include <asm/swift.h>
48 #include <asm/turbosparc.h>
51 #include <asm/btfixup.h>
53 enum mbus_module srmmu_modtype;
54 static unsigned int hwbug_bitmask;
58 struct ctx_list *ctx_list_pool;
59 struct ctx_list ctx_free;
60 struct ctx_list ctx_used;
62 extern struct resource sparc_iomap;
64 extern unsigned long last_valid_pfn;
66 static pgd_t *srmmu_swapper_pg_dir;
69 #define FLUSH_BEGIN(mm)
72 #define FLUSH_BEGIN(mm) if((mm)->context != NO_CONTEXT) {
76 BTFIXUPDEF_CALL(void, flush_page_for_dma, unsigned long)
77 #define flush_page_for_dma(page) BTFIXUP_CALL(flush_page_for_dma)(page)
79 int flush_page_for_dma_global = 1;
82 BTFIXUPDEF_CALL(void, local_flush_page_for_dma, unsigned long)
83 #define local_flush_page_for_dma(page) BTFIXUP_CALL(local_flush_page_for_dma)(page)
88 ctxd_t *srmmu_ctx_table_phys;
89 static ctxd_t *srmmu_context_table;
91 int viking_mxcc_present;
92 static DEFINE_SPINLOCK(srmmu_context_spinlock);
94 static int is_hypersparc;
96 static int srmmu_cache_pagetables;
98 /* these will be initialized in srmmu_nocache_calcsize() */
99 static unsigned long srmmu_nocache_size;
100 static unsigned long srmmu_nocache_end;
102 /* 1 bit <=> 256 bytes of nocache <=> 64 PTEs */
103 #define SRMMU_NOCACHE_BITMAP_SHIFT (PAGE_SHIFT - 4)
105 /* The context table is a nocache user with the biggest alignment needs. */
106 #define SRMMU_NOCACHE_ALIGN_MAX (sizeof(ctxd_t)*SRMMU_MAX_CONTEXTS)
108 void *srmmu_nocache_pool;
109 void *srmmu_nocache_bitmap;
110 static struct bit_map srmmu_nocache_map;
112 static inline unsigned long srmmu_pgd_page(pgd_t pgd)
113 { return srmmu_device_memory(pgd_val(pgd))?~0:(unsigned long)__nocache_va((pgd_val(pgd) & SRMMU_PTD_PMASK) << 4); }
116 static inline int srmmu_pte_none(pte_t pte)
117 { return !(pte_val(pte) & 0xFFFFFFF); }
119 static inline int srmmu_pmd_none(pmd_t pmd)
120 { return !(pmd_val(pmd) & 0xFFFFFFF); }
122 static inline pte_t srmmu_pte_wrprotect(pte_t pte)
123 { return __pte(pte_val(pte) & ~SRMMU_WRITE);}
125 static inline pte_t srmmu_pte_mkclean(pte_t pte)
126 { return __pte(pte_val(pte) & ~SRMMU_DIRTY);}
128 static inline pte_t srmmu_pte_mkold(pte_t pte)
129 { return __pte(pte_val(pte) & ~SRMMU_REF);}
131 static inline pte_t srmmu_pte_mkwrite(pte_t pte)
132 { return __pte(pte_val(pte) | SRMMU_WRITE);}
134 static inline pte_t srmmu_pte_mkdirty(pte_t pte)
135 { return __pte(pte_val(pte) | SRMMU_DIRTY);}
137 static inline pte_t srmmu_pte_mkyoung(pte_t pte)
138 { return __pte(pte_val(pte) | SRMMU_REF);}
140 /* XXX should we hyper_flush_whole_icache here - Anton */
141 static inline void srmmu_ctxd_set(ctxd_t *ctxp, pgd_t *pgdp)
142 { set_pte((pte_t *)ctxp, (SRMMU_ET_PTD | (__nocache_pa((unsigned long) pgdp) >> 4))); }
144 static inline void srmmu_pgd_set(pgd_t * pgdp, pmd_t * pmdp)
145 { set_pte((pte_t *)pgdp, (SRMMU_ET_PTD | (__nocache_pa((unsigned long) pmdp) >> 4))); }
147 static void srmmu_pmd_set(pmd_t *pmdp, pte_t *ptep)
149 unsigned long ptp; /* Physical address, shifted right by 4 */
152 ptp = __nocache_pa((unsigned long) ptep) >> 4;
153 for (i = 0; i < PTRS_PER_PTE/SRMMU_REAL_PTRS_PER_PTE; i++) {
154 set_pte((pte_t *)&pmdp->pmdv[i], SRMMU_ET_PTD | ptp);
155 ptp += (SRMMU_REAL_PTRS_PER_PTE*sizeof(pte_t) >> 4);
159 static void srmmu_pmd_populate(pmd_t *pmdp, struct page *ptep)
161 unsigned long ptp; /* Physical address, shifted right by 4 */
164 ptp = page_to_pfn(ptep) << (PAGE_SHIFT-4); /* watch for overflow */
165 for (i = 0; i < PTRS_PER_PTE/SRMMU_REAL_PTRS_PER_PTE; i++) {
166 set_pte((pte_t *)&pmdp->pmdv[i], SRMMU_ET_PTD | ptp);
167 ptp += (SRMMU_REAL_PTRS_PER_PTE*sizeof(pte_t) >> 4);
171 static inline pte_t srmmu_pte_modify(pte_t pte, pgprot_t newprot)
172 { return __pte((pte_val(pte) & SRMMU_CHG_MASK) | pgprot_val(newprot)); }
174 /* to find an entry in a top-level page table... */
175 static inline pgd_t *srmmu_pgd_offset(struct mm_struct * mm, unsigned long address)
176 { return mm->pgd + (address >> SRMMU_PGDIR_SHIFT); }
178 /* Find an entry in the second-level page table.. */
179 static inline pmd_t *srmmu_pmd_offset(pgd_t * dir, unsigned long address)
181 return (pmd_t *) srmmu_pgd_page(*dir) +
182 ((address >> PMD_SHIFT) & (PTRS_PER_PMD - 1));
185 /* Find an entry in the third-level page table.. */
186 static inline pte_t *srmmu_pte_offset(pmd_t * dir, unsigned long address)
190 pte = __nocache_va((dir->pmdv[0] & SRMMU_PTD_PMASK) << 4);
191 return (pte_t *) pte +
192 ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1));
195 static unsigned long srmmu_swp_type(swp_entry_t entry)
197 return (entry.val >> SRMMU_SWP_TYPE_SHIFT) & SRMMU_SWP_TYPE_MASK;
200 static unsigned long srmmu_swp_offset(swp_entry_t entry)
202 return (entry.val >> SRMMU_SWP_OFF_SHIFT) & SRMMU_SWP_OFF_MASK;
205 static swp_entry_t srmmu_swp_entry(unsigned long type, unsigned long offset)
207 return (swp_entry_t) {
208 (type & SRMMU_SWP_TYPE_MASK) << SRMMU_SWP_TYPE_SHIFT
209 | (offset & SRMMU_SWP_OFF_MASK) << SRMMU_SWP_OFF_SHIFT };
213 * size: bytes to allocate in the nocache area.
214 * align: bytes, number to align at.
215 * Returns the virtual address of the allocated area.
217 static unsigned long __srmmu_get_nocache(int size, int align)
221 if (size < SRMMU_NOCACHE_BITMAP_SHIFT) {
222 printk("Size 0x%x too small for nocache request\n", size);
223 size = SRMMU_NOCACHE_BITMAP_SHIFT;
225 if (size & (SRMMU_NOCACHE_BITMAP_SHIFT-1)) {
226 printk("Size 0x%x unaligned int nocache request\n", size);
227 size += SRMMU_NOCACHE_BITMAP_SHIFT-1;
229 BUG_ON(align > SRMMU_NOCACHE_ALIGN_MAX);
231 offset = bit_map_string_get(&srmmu_nocache_map,
232 size >> SRMMU_NOCACHE_BITMAP_SHIFT,
233 align >> SRMMU_NOCACHE_BITMAP_SHIFT);
235 printk("srmmu: out of nocache %d: %d/%d\n",
236 size, (int) srmmu_nocache_size,
237 srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT);
241 return (SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT));
244 static unsigned long srmmu_get_nocache(int size, int align)
248 tmp = __srmmu_get_nocache(size, align);
251 memset((void *)tmp, 0, size);
256 static void srmmu_free_nocache(unsigned long vaddr, int size)
260 if (vaddr < SRMMU_NOCACHE_VADDR) {
261 printk("Vaddr %lx is smaller than nocache base 0x%lx\n",
262 vaddr, (unsigned long)SRMMU_NOCACHE_VADDR);
265 if (vaddr+size > srmmu_nocache_end) {
266 printk("Vaddr %lx is bigger than nocache end 0x%lx\n",
267 vaddr, srmmu_nocache_end);
270 if (!is_power_of_2(size)) {
271 printk("Size 0x%x is not a power of 2\n", size);
274 if (size < SRMMU_NOCACHE_BITMAP_SHIFT) {
275 printk("Size 0x%x is too small\n", size);
278 if (vaddr & (size-1)) {
279 printk("Vaddr %lx is not aligned to size 0x%x\n", vaddr, size);
283 offset = (vaddr - SRMMU_NOCACHE_VADDR) >> SRMMU_NOCACHE_BITMAP_SHIFT;
284 size = size >> SRMMU_NOCACHE_BITMAP_SHIFT;
286 bit_map_clear(&srmmu_nocache_map, offset, size);
289 static void srmmu_early_allocate_ptable_skeleton(unsigned long start,
292 extern unsigned long probe_memory(void); /* in fault.c */
295 * Reserve nocache dynamically proportionally to the amount of
296 * system RAM. -- Tomas Szepe <szepe@pinerecords.com>, June 2002
298 static void srmmu_nocache_calcsize(void)
300 unsigned long sysmemavail = probe_memory() / 1024;
301 int srmmu_nocache_npages;
303 srmmu_nocache_npages =
304 sysmemavail / SRMMU_NOCACHE_ALCRATIO / 1024 * 256;
306 /* P3 XXX The 4x overuse: corroborated by /proc/meminfo. */
307 // if (srmmu_nocache_npages < 256) srmmu_nocache_npages = 256;
308 if (srmmu_nocache_npages < SRMMU_MIN_NOCACHE_PAGES)
309 srmmu_nocache_npages = SRMMU_MIN_NOCACHE_PAGES;
311 /* anything above 1280 blows up */
312 if (srmmu_nocache_npages > SRMMU_MAX_NOCACHE_PAGES)
313 srmmu_nocache_npages = SRMMU_MAX_NOCACHE_PAGES;
315 srmmu_nocache_size = srmmu_nocache_npages * PAGE_SIZE;
316 srmmu_nocache_end = SRMMU_NOCACHE_VADDR + srmmu_nocache_size;
319 static void __init srmmu_nocache_init(void)
321 unsigned int bitmap_bits;
325 unsigned long paddr, vaddr;
326 unsigned long pteval;
328 bitmap_bits = srmmu_nocache_size >> SRMMU_NOCACHE_BITMAP_SHIFT;
330 srmmu_nocache_pool = __alloc_bootmem(srmmu_nocache_size,
331 SRMMU_NOCACHE_ALIGN_MAX, 0UL);
332 memset(srmmu_nocache_pool, 0, srmmu_nocache_size);
334 srmmu_nocache_bitmap = __alloc_bootmem(bitmap_bits >> 3, SMP_CACHE_BYTES, 0UL);
335 bit_map_init(&srmmu_nocache_map, srmmu_nocache_bitmap, bitmap_bits);
337 srmmu_swapper_pg_dir = (pgd_t *)__srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
338 memset(__nocache_fix(srmmu_swapper_pg_dir), 0, SRMMU_PGD_TABLE_SIZE);
339 init_mm.pgd = srmmu_swapper_pg_dir;
341 srmmu_early_allocate_ptable_skeleton(SRMMU_NOCACHE_VADDR, srmmu_nocache_end);
343 paddr = __pa((unsigned long)srmmu_nocache_pool);
344 vaddr = SRMMU_NOCACHE_VADDR;
346 while (vaddr < srmmu_nocache_end) {
347 pgd = pgd_offset_k(vaddr);
348 pmd = srmmu_pmd_offset(__nocache_fix(pgd), vaddr);
349 pte = srmmu_pte_offset(__nocache_fix(pmd), vaddr);
351 pteval = ((paddr >> 4) | SRMMU_ET_PTE | SRMMU_PRIV);
353 if (srmmu_cache_pagetables)
354 pteval |= SRMMU_CACHE;
356 set_pte(__nocache_fix(pte), __pte(pteval));
366 static inline pgd_t *srmmu_get_pgd_fast(void)
370 pgd = (pgd_t *)__srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
372 pgd_t *init = pgd_offset_k(0);
373 memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
374 memcpy(pgd + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,
375 (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
381 static void srmmu_free_pgd_fast(pgd_t *pgd)
383 srmmu_free_nocache((unsigned long)pgd, SRMMU_PGD_TABLE_SIZE);
386 static pmd_t *srmmu_pmd_alloc_one(struct mm_struct *mm, unsigned long address)
388 return (pmd_t *)srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
391 static void srmmu_pmd_free(pmd_t * pmd)
393 srmmu_free_nocache((unsigned long)pmd, SRMMU_PMD_TABLE_SIZE);
397 * Hardware needs alignment to 256 only, but we align to whole page size
398 * to reduce fragmentation problems due to the buddy principle.
399 * XXX Provide actual fragmentation statistics in /proc.
401 * Alignments up to the page size are the same for physical and virtual
402 * addresses of the nocache area.
405 srmmu_pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
407 return (pte_t *)srmmu_get_nocache(PTE_SIZE, PTE_SIZE);
411 srmmu_pte_alloc_one(struct mm_struct *mm, unsigned long address)
416 if ((pte = (unsigned long)srmmu_pte_alloc_one_kernel(mm, address)) == 0)
418 page = pfn_to_page( __nocache_pa(pte) >> PAGE_SHIFT );
419 pgtable_page_ctor(page);
423 static void srmmu_free_pte_fast(pte_t *pte)
425 srmmu_free_nocache((unsigned long)pte, PTE_SIZE);
428 static void srmmu_pte_free(pgtable_t pte)
432 pgtable_page_dtor(pte);
433 p = (unsigned long)page_address(pte); /* Cached address (for test) */
436 p = page_to_pfn(pte) << PAGE_SHIFT; /* Physical address */
437 p = (unsigned long) __nocache_va(p); /* Nocached virtual */
438 srmmu_free_nocache(p, PTE_SIZE);
443 static inline void alloc_context(struct mm_struct *old_mm, struct mm_struct *mm)
445 struct ctx_list *ctxp;
447 ctxp = ctx_free.next;
448 if(ctxp != &ctx_free) {
449 remove_from_ctx_list(ctxp);
450 add_to_used_ctxlist(ctxp);
451 mm->context = ctxp->ctx_number;
455 ctxp = ctx_used.next;
456 if(ctxp->ctx_mm == old_mm)
458 if(ctxp == &ctx_used)
459 panic("out of mmu contexts");
460 flush_cache_mm(ctxp->ctx_mm);
461 flush_tlb_mm(ctxp->ctx_mm);
462 remove_from_ctx_list(ctxp);
463 add_to_used_ctxlist(ctxp);
464 ctxp->ctx_mm->context = NO_CONTEXT;
466 mm->context = ctxp->ctx_number;
469 static inline void free_context(int context)
471 struct ctx_list *ctx_old;
473 ctx_old = ctx_list_pool + context;
474 remove_from_ctx_list(ctx_old);
475 add_to_free_ctxlist(ctx_old);
479 void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm,
480 struct task_struct *tsk)
482 if(mm->context == NO_CONTEXT) {
483 spin_lock(&srmmu_context_spinlock);
484 alloc_context(old_mm, mm);
485 spin_unlock(&srmmu_context_spinlock);
486 srmmu_ctxd_set(&srmmu_context_table[mm->context], mm->pgd);
489 if (sparc_cpu_model == sparc_leon)
493 hyper_flush_whole_icache();
495 srmmu_set_context(mm->context);
498 /* Low level IO area allocation on the SRMMU. */
499 static inline void srmmu_mapioaddr(unsigned long physaddr,
500 unsigned long virt_addr, int bus_type)
507 physaddr &= PAGE_MASK;
508 pgdp = pgd_offset_k(virt_addr);
509 pmdp = srmmu_pmd_offset(pgdp, virt_addr);
510 ptep = srmmu_pte_offset(pmdp, virt_addr);
511 tmp = (physaddr >> 4) | SRMMU_ET_PTE;
514 * I need to test whether this is consistent over all
515 * sun4m's. The bus_type represents the upper 4 bits of
516 * 36-bit physical address on the I/O space lines...
518 tmp |= (bus_type << 28);
520 __flush_page_to_ram(virt_addr);
521 set_pte(ptep, __pte(tmp));
524 static void srmmu_mapiorange(unsigned int bus, unsigned long xpa,
525 unsigned long xva, unsigned int len)
529 srmmu_mapioaddr(xpa, xva, bus);
536 static inline void srmmu_unmapioaddr(unsigned long virt_addr)
542 pgdp = pgd_offset_k(virt_addr);
543 pmdp = srmmu_pmd_offset(pgdp, virt_addr);
544 ptep = srmmu_pte_offset(pmdp, virt_addr);
546 /* No need to flush uncacheable page. */
550 static void srmmu_unmapiorange(unsigned long virt_addr, unsigned int len)
554 srmmu_unmapioaddr(virt_addr);
555 virt_addr += PAGE_SIZE;
561 * On the SRMMU we do not have the problems with limited tlb entries
562 * for mapping kernel pages, so we just take things from the free page
563 * pool. As a side effect we are putting a little too much pressure
564 * on the gfp() subsystem. This setup also makes the logic of the
565 * iommu mapping code a lot easier as we can transparently handle
566 * mappings on the kernel stack without any special code.
568 struct thread_info *alloc_thread_info_node(struct task_struct *tsk, int node)
570 struct thread_info *ret;
572 ret = (struct thread_info *)__get_free_pages(GFP_KERNEL,
574 #ifdef CONFIG_DEBUG_STACK_USAGE
576 memset(ret, 0, PAGE_SIZE << THREAD_INFO_ORDER);
577 #endif /* DEBUG_STACK_USAGE */
582 void free_thread_info(struct thread_info *ti)
584 free_pages((unsigned long)ti, THREAD_INFO_ORDER);
588 extern void tsunami_flush_cache_all(void);
589 extern void tsunami_flush_cache_mm(struct mm_struct *mm);
590 extern void tsunami_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
591 extern void tsunami_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
592 extern void tsunami_flush_page_to_ram(unsigned long page);
593 extern void tsunami_flush_page_for_dma(unsigned long page);
594 extern void tsunami_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
595 extern void tsunami_flush_tlb_all(void);
596 extern void tsunami_flush_tlb_mm(struct mm_struct *mm);
597 extern void tsunami_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
598 extern void tsunami_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
599 extern void tsunami_setup_blockops(void);
602 * Workaround, until we find what's going on with Swift. When low on memory,
603 * it sometimes loops in fault/handle_mm_fault incl. flush_tlb_page to find
604 * out it is already in page tables/ fault again on the same instruction.
605 * I really don't understand it, have checked it and contexts
606 * are right, flush_tlb_all is done as well, and it faults again...
609 * The following code is a deadwood that may be necessary when
610 * we start to make precise page flushes again. --zaitcev
612 static void swift_update_mmu_cache(struct vm_area_struct * vma, unsigned long address, pte_t *ptep)
615 static unsigned long last;
617 /* unsigned int n; */
619 if (address == last) {
620 val = srmmu_hwprobe(address);
621 if (val != 0 && pte_val(*ptep) != val) {
622 printk("swift_update_mmu_cache: "
623 "addr %lx put %08x probed %08x from %pf\n",
624 address, pte_val(*ptep), val,
625 __builtin_return_address(0));
626 srmmu_flush_whole_tlb();
634 extern void swift_flush_cache_all(void);
635 extern void swift_flush_cache_mm(struct mm_struct *mm);
636 extern void swift_flush_cache_range(struct vm_area_struct *vma,
637 unsigned long start, unsigned long end);
638 extern void swift_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
639 extern void swift_flush_page_to_ram(unsigned long page);
640 extern void swift_flush_page_for_dma(unsigned long page);
641 extern void swift_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
642 extern void swift_flush_tlb_all(void);
643 extern void swift_flush_tlb_mm(struct mm_struct *mm);
644 extern void swift_flush_tlb_range(struct vm_area_struct *vma,
645 unsigned long start, unsigned long end);
646 extern void swift_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
648 #if 0 /* P3: deadwood to debug precise flushes on Swift. */
649 void swift_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
654 if ((ctx1 = vma->vm_mm->context) != -1) {
655 cctx = srmmu_get_context();
656 /* Is context # ever different from current context? P3 */
658 printk("flush ctx %02x curr %02x\n", ctx1, cctx);
659 srmmu_set_context(ctx1);
660 swift_flush_page(page);
661 __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : :
662 "r" (page), "i" (ASI_M_FLUSH_PROBE));
663 srmmu_set_context(cctx);
665 /* Rm. prot. bits from virt. c. */
666 /* swift_flush_cache_all(); */
667 /* swift_flush_cache_page(vma, page); */
668 swift_flush_page(page);
670 __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : :
671 "r" (page), "i" (ASI_M_FLUSH_PROBE));
672 /* same as above: srmmu_flush_tlb_page() */
679 * The following are all MBUS based SRMMU modules, and therefore could
680 * be found in a multiprocessor configuration. On the whole, these
681 * chips seems to be much more touchy about DVMA and page tables
682 * with respect to cache coherency.
685 /* Cypress flushes. */
686 static void cypress_flush_cache_all(void)
688 volatile unsigned long cypress_sucks;
689 unsigned long faddr, tagval;
691 flush_user_windows();
692 for(faddr = 0; faddr < 0x10000; faddr += 0x20) {
693 __asm__ __volatile__("lda [%1 + %2] %3, %0\n\t" :
695 "r" (faddr), "r" (0x40000),
696 "i" (ASI_M_DATAC_TAG));
698 /* If modified and valid, kick it. */
699 if((tagval & 0x60) == 0x60)
700 cypress_sucks = *(unsigned long *)(0xf0020000 + faddr);
704 static void cypress_flush_cache_mm(struct mm_struct *mm)
706 register unsigned long a, b, c, d, e, f, g;
707 unsigned long flags, faddr;
711 flush_user_windows();
712 local_irq_save(flags);
713 octx = srmmu_get_context();
714 srmmu_set_context(mm->context);
715 a = 0x20; b = 0x40; c = 0x60;
716 d = 0x80; e = 0xa0; f = 0xc0; g = 0xe0;
718 faddr = (0x10000 - 0x100);
723 __asm__ __volatile__("sta %%g0, [%0] %1\n\t"
724 "sta %%g0, [%0 + %2] %1\n\t"
725 "sta %%g0, [%0 + %3] %1\n\t"
726 "sta %%g0, [%0 + %4] %1\n\t"
727 "sta %%g0, [%0 + %5] %1\n\t"
728 "sta %%g0, [%0 + %6] %1\n\t"
729 "sta %%g0, [%0 + %7] %1\n\t"
730 "sta %%g0, [%0 + %8] %1\n\t" : :
731 "r" (faddr), "i" (ASI_M_FLUSH_CTX),
732 "r" (a), "r" (b), "r" (c), "r" (d),
733 "r" (e), "r" (f), "r" (g));
735 srmmu_set_context(octx);
736 local_irq_restore(flags);
740 static void cypress_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
742 struct mm_struct *mm = vma->vm_mm;
743 register unsigned long a, b, c, d, e, f, g;
744 unsigned long flags, faddr;
748 flush_user_windows();
749 local_irq_save(flags);
750 octx = srmmu_get_context();
751 srmmu_set_context(mm->context);
752 a = 0x20; b = 0x40; c = 0x60;
753 d = 0x80; e = 0xa0; f = 0xc0; g = 0xe0;
755 start &= SRMMU_REAL_PMD_MASK;
757 faddr = (start + (0x10000 - 0x100));
762 __asm__ __volatile__("sta %%g0, [%0] %1\n\t"
763 "sta %%g0, [%0 + %2] %1\n\t"
764 "sta %%g0, [%0 + %3] %1\n\t"
765 "sta %%g0, [%0 + %4] %1\n\t"
766 "sta %%g0, [%0 + %5] %1\n\t"
767 "sta %%g0, [%0 + %6] %1\n\t"
768 "sta %%g0, [%0 + %7] %1\n\t"
769 "sta %%g0, [%0 + %8] %1\n\t" : :
771 "i" (ASI_M_FLUSH_SEG),
772 "r" (a), "r" (b), "r" (c), "r" (d),
773 "r" (e), "r" (f), "r" (g));
774 } while (faddr != start);
775 start += SRMMU_REAL_PMD_SIZE;
777 srmmu_set_context(octx);
778 local_irq_restore(flags);
782 static void cypress_flush_cache_page(struct vm_area_struct *vma, unsigned long page)
784 register unsigned long a, b, c, d, e, f, g;
785 struct mm_struct *mm = vma->vm_mm;
786 unsigned long flags, line;
790 flush_user_windows();
791 local_irq_save(flags);
792 octx = srmmu_get_context();
793 srmmu_set_context(mm->context);
794 a = 0x20; b = 0x40; c = 0x60;
795 d = 0x80; e = 0xa0; f = 0xc0; g = 0xe0;
798 line = (page + PAGE_SIZE) - 0x100;
803 __asm__ __volatile__("sta %%g0, [%0] %1\n\t"
804 "sta %%g0, [%0 + %2] %1\n\t"
805 "sta %%g0, [%0 + %3] %1\n\t"
806 "sta %%g0, [%0 + %4] %1\n\t"
807 "sta %%g0, [%0 + %5] %1\n\t"
808 "sta %%g0, [%0 + %6] %1\n\t"
809 "sta %%g0, [%0 + %7] %1\n\t"
810 "sta %%g0, [%0 + %8] %1\n\t" : :
812 "i" (ASI_M_FLUSH_PAGE),
813 "r" (a), "r" (b), "r" (c), "r" (d),
814 "r" (e), "r" (f), "r" (g));
815 } while(line != page);
816 srmmu_set_context(octx);
817 local_irq_restore(flags);
821 /* Cypress is copy-back, at least that is how we configure it. */
822 static void cypress_flush_page_to_ram(unsigned long page)
824 register unsigned long a, b, c, d, e, f, g;
827 a = 0x20; b = 0x40; c = 0x60; d = 0x80; e = 0xa0; f = 0xc0; g = 0xe0;
829 line = (page + PAGE_SIZE) - 0x100;
834 __asm__ __volatile__("sta %%g0, [%0] %1\n\t"
835 "sta %%g0, [%0 + %2] %1\n\t"
836 "sta %%g0, [%0 + %3] %1\n\t"
837 "sta %%g0, [%0 + %4] %1\n\t"
838 "sta %%g0, [%0 + %5] %1\n\t"
839 "sta %%g0, [%0 + %6] %1\n\t"
840 "sta %%g0, [%0 + %7] %1\n\t"
841 "sta %%g0, [%0 + %8] %1\n\t" : :
843 "i" (ASI_M_FLUSH_PAGE),
844 "r" (a), "r" (b), "r" (c), "r" (d),
845 "r" (e), "r" (f), "r" (g));
846 } while(line != page);
849 /* Cypress is also IO cache coherent. */
850 static void cypress_flush_page_for_dma(unsigned long page)
854 /* Cypress has unified L2 VIPT, from which both instructions and data
855 * are stored. It does not have an onboard icache of any sort, therefore
856 * no flush is necessary.
858 static void cypress_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr)
862 static void cypress_flush_tlb_all(void)
864 srmmu_flush_whole_tlb();
867 static void cypress_flush_tlb_mm(struct mm_struct *mm)
870 __asm__ __volatile__(
871 "lda [%0] %3, %%g5\n\t"
872 "sta %2, [%0] %3\n\t"
873 "sta %%g0, [%1] %4\n\t"
874 "sta %%g5, [%0] %3\n"
876 : "r" (SRMMU_CTX_REG), "r" (0x300), "r" (mm->context),
877 "i" (ASI_M_MMUREGS), "i" (ASI_M_FLUSH_PROBE)
882 static void cypress_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
884 struct mm_struct *mm = vma->vm_mm;
888 start &= SRMMU_PGDIR_MASK;
889 size = SRMMU_PGDIR_ALIGN(end) - start;
890 __asm__ __volatile__(
891 "lda [%0] %5, %%g5\n\t"
894 "subcc %3, %4, %3\n\t"
896 " sta %%g0, [%2 + %3] %6\n\t"
897 "sta %%g5, [%0] %5\n"
899 : "r" (SRMMU_CTX_REG), "r" (mm->context), "r" (start | 0x200),
900 "r" (size), "r" (SRMMU_PGDIR_SIZE), "i" (ASI_M_MMUREGS),
901 "i" (ASI_M_FLUSH_PROBE)
906 static void cypress_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
908 struct mm_struct *mm = vma->vm_mm;
911 __asm__ __volatile__(
912 "lda [%0] %3, %%g5\n\t"
913 "sta %1, [%0] %3\n\t"
914 "sta %%g0, [%2] %4\n\t"
915 "sta %%g5, [%0] %3\n"
917 : "r" (SRMMU_CTX_REG), "r" (mm->context), "r" (page & PAGE_MASK),
918 "i" (ASI_M_MMUREGS), "i" (ASI_M_FLUSH_PROBE)
924 extern void viking_flush_cache_all(void);
925 extern void viking_flush_cache_mm(struct mm_struct *mm);
926 extern void viking_flush_cache_range(struct vm_area_struct *vma, unsigned long start,
928 extern void viking_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
929 extern void viking_flush_page_to_ram(unsigned long page);
930 extern void viking_flush_page_for_dma(unsigned long page);
931 extern void viking_flush_sig_insns(struct mm_struct *mm, unsigned long addr);
932 extern void viking_flush_page(unsigned long page);
933 extern void viking_mxcc_flush_page(unsigned long page);
934 extern void viking_flush_tlb_all(void);
935 extern void viking_flush_tlb_mm(struct mm_struct *mm);
936 extern void viking_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
938 extern void viking_flush_tlb_page(struct vm_area_struct *vma,
940 extern void sun4dsmp_flush_tlb_all(void);
941 extern void sun4dsmp_flush_tlb_mm(struct mm_struct *mm);
942 extern void sun4dsmp_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
944 extern void sun4dsmp_flush_tlb_page(struct vm_area_struct *vma,
948 extern void hypersparc_flush_cache_all(void);
949 extern void hypersparc_flush_cache_mm(struct mm_struct *mm);
950 extern void hypersparc_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
951 extern void hypersparc_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
952 extern void hypersparc_flush_page_to_ram(unsigned long page);
953 extern void hypersparc_flush_page_for_dma(unsigned long page);
954 extern void hypersparc_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
955 extern void hypersparc_flush_tlb_all(void);
956 extern void hypersparc_flush_tlb_mm(struct mm_struct *mm);
957 extern void hypersparc_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
958 extern void hypersparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
959 extern void hypersparc_setup_blockops(void);
962 * NOTE: All of this startup code assumes the low 16mb (approx.) of
963 * kernel mappings are done with one single contiguous chunk of
964 * ram. On small ram machines (classics mainly) we only get
965 * around 8mb mapped for us.
968 static void __init early_pgtable_allocfail(char *type)
970 prom_printf("inherit_prom_mappings: Cannot alloc kernel %s.\n", type);
974 static void __init srmmu_early_allocate_ptable_skeleton(unsigned long start,
982 pgdp = pgd_offset_k(start);
983 if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) {
984 pmdp = (pmd_t *) __srmmu_get_nocache(
985 SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
987 early_pgtable_allocfail("pmd");
988 memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE);
989 srmmu_pgd_set(__nocache_fix(pgdp), pmdp);
991 pmdp = srmmu_pmd_offset(__nocache_fix(pgdp), start);
992 if(srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) {
993 ptep = (pte_t *)__srmmu_get_nocache(PTE_SIZE, PTE_SIZE);
995 early_pgtable_allocfail("pte");
996 memset(__nocache_fix(ptep), 0, PTE_SIZE);
997 srmmu_pmd_set(__nocache_fix(pmdp), ptep);
999 if (start > (0xffffffffUL - PMD_SIZE))
1001 start = (start + PMD_SIZE) & PMD_MASK;
1005 static void __init srmmu_allocate_ptable_skeleton(unsigned long start,
1012 while(start < end) {
1013 pgdp = pgd_offset_k(start);
1014 if (pgd_none(*pgdp)) {
1015 pmdp = (pmd_t *)__srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
1017 early_pgtable_allocfail("pmd");
1018 memset(pmdp, 0, SRMMU_PMD_TABLE_SIZE);
1019 srmmu_pgd_set(pgdp, pmdp);
1021 pmdp = srmmu_pmd_offset(pgdp, start);
1022 if(srmmu_pmd_none(*pmdp)) {
1023 ptep = (pte_t *) __srmmu_get_nocache(PTE_SIZE,
1026 early_pgtable_allocfail("pte");
1027 memset(ptep, 0, PTE_SIZE);
1028 srmmu_pmd_set(pmdp, ptep);
1030 if (start > (0xffffffffUL - PMD_SIZE))
1032 start = (start + PMD_SIZE) & PMD_MASK;
1037 * This is much cleaner than poking around physical address space
1038 * looking at the prom's page table directly which is what most
1039 * other OS's do. Yuck... this is much better.
1041 static void __init srmmu_inherit_prom_mappings(unsigned long start,
1047 int what = 0; /* 0 = normal-pte, 1 = pmd-level pte, 2 = pgd-level pte */
1048 unsigned long prompte;
1050 while(start <= end) {
1052 break; /* probably wrap around */
1053 if(start == 0xfef00000)
1054 start = KADB_DEBUGGER_BEGVM;
1055 if(!(prompte = srmmu_hwprobe(start))) {
1060 /* A red snapper, see what it really is. */
1063 if(!(start & ~(SRMMU_REAL_PMD_MASK))) {
1064 if(srmmu_hwprobe((start-PAGE_SIZE) + SRMMU_REAL_PMD_SIZE) == prompte)
1068 if(!(start & ~(SRMMU_PGDIR_MASK))) {
1069 if(srmmu_hwprobe((start-PAGE_SIZE) + SRMMU_PGDIR_SIZE) ==
1074 pgdp = pgd_offset_k(start);
1076 *(pgd_t *)__nocache_fix(pgdp) = __pgd(prompte);
1077 start += SRMMU_PGDIR_SIZE;
1080 if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) {
1081 pmdp = (pmd_t *)__srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
1083 early_pgtable_allocfail("pmd");
1084 memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE);
1085 srmmu_pgd_set(__nocache_fix(pgdp), pmdp);
1087 pmdp = srmmu_pmd_offset(__nocache_fix(pgdp), start);
1088 if(srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) {
1089 ptep = (pte_t *) __srmmu_get_nocache(PTE_SIZE,
1092 early_pgtable_allocfail("pte");
1093 memset(__nocache_fix(ptep), 0, PTE_SIZE);
1094 srmmu_pmd_set(__nocache_fix(pmdp), ptep);
1098 * We bend the rule where all 16 PTPs in a pmd_t point
1099 * inside the same PTE page, and we leak a perfectly
1100 * good hardware PTE piece. Alternatives seem worse.
1102 unsigned int x; /* Index of HW PMD in soft cluster */
1103 x = (start >> PMD_SHIFT) & 15;
1104 *(unsigned long *)__nocache_fix(&pmdp->pmdv[x]) = prompte;
1105 start += SRMMU_REAL_PMD_SIZE;
1108 ptep = srmmu_pte_offset(__nocache_fix(pmdp), start);
1109 *(pte_t *)__nocache_fix(ptep) = __pte(prompte);
1114 #define KERNEL_PTE(page_shifted) ((page_shifted)|SRMMU_CACHE|SRMMU_PRIV|SRMMU_VALID)
1116 /* Create a third-level SRMMU 16MB page mapping. */
1117 static void __init do_large_mapping(unsigned long vaddr, unsigned long phys_base)
1119 pgd_t *pgdp = pgd_offset_k(vaddr);
1120 unsigned long big_pte;
1122 big_pte = KERNEL_PTE(phys_base >> 4);
1123 *(pgd_t *)__nocache_fix(pgdp) = __pgd(big_pte);
1126 /* Map sp_bank entry SP_ENTRY, starting at virtual address VBASE. */
1127 static unsigned long __init map_spbank(unsigned long vbase, int sp_entry)
1129 unsigned long pstart = (sp_banks[sp_entry].base_addr & SRMMU_PGDIR_MASK);
1130 unsigned long vstart = (vbase & SRMMU_PGDIR_MASK);
1131 unsigned long vend = SRMMU_PGDIR_ALIGN(vbase + sp_banks[sp_entry].num_bytes);
1132 /* Map "low" memory only */
1133 const unsigned long min_vaddr = PAGE_OFFSET;
1134 const unsigned long max_vaddr = PAGE_OFFSET + SRMMU_MAXMEM;
1136 if (vstart < min_vaddr || vstart >= max_vaddr)
1139 if (vend > max_vaddr || vend < min_vaddr)
1142 while(vstart < vend) {
1143 do_large_mapping(vstart, pstart);
1144 vstart += SRMMU_PGDIR_SIZE; pstart += SRMMU_PGDIR_SIZE;
1149 static inline void memprobe_error(char *msg)
1152 prom_printf("Halting now...\n");
1156 static inline void map_kernel(void)
1160 if (phys_base > 0) {
1161 do_large_mapping(PAGE_OFFSET, phys_base);
1164 for (i = 0; sp_banks[i].num_bytes != 0; i++) {
1165 map_spbank((unsigned long)__va(sp_banks[i].base_addr), i);
1169 /* Paging initialization on the Sparc Reference MMU. */
1170 extern void sparc_context_init(int);
1172 void (*poke_srmmu)(void) __cpuinitdata = NULL;
1174 extern unsigned long bootmem_init(unsigned long *pages_avail);
1176 void __init srmmu_paging_init(void)
1184 unsigned long pages_avail;
1186 sparc_iomap.start = SUN4M_IOBASE_VADDR; /* 16MB of IOSPACE on all sun4m's. */
1188 if (sparc_cpu_model == sun4d)
1189 num_contexts = 65536; /* We know it is Viking */
1191 /* Find the number of contexts on the srmmu. */
1192 cpunode = prom_getchild(prom_root_node);
1194 while(cpunode != 0) {
1195 prom_getstring(cpunode, "device_type", node_str, sizeof(node_str));
1196 if(!strcmp(node_str, "cpu")) {
1197 num_contexts = prom_getintdefault(cpunode, "mmu-nctx", 0x8);
1200 cpunode = prom_getsibling(cpunode);
1205 prom_printf("Something wrong, can't find cpu node in paging_init.\n");
1210 last_valid_pfn = bootmem_init(&pages_avail);
1212 srmmu_nocache_calcsize();
1213 srmmu_nocache_init();
1214 srmmu_inherit_prom_mappings(0xfe400000,(LINUX_OPPROM_ENDVM-PAGE_SIZE));
1217 /* ctx table has to be physically aligned to its size */
1218 srmmu_context_table = (ctxd_t *)__srmmu_get_nocache(num_contexts*sizeof(ctxd_t), num_contexts*sizeof(ctxd_t));
1219 srmmu_ctx_table_phys = (ctxd_t *)__nocache_pa((unsigned long)srmmu_context_table);
1221 for(i = 0; i < num_contexts; i++)
1222 srmmu_ctxd_set((ctxd_t *)__nocache_fix(&srmmu_context_table[i]), srmmu_swapper_pg_dir);
1225 srmmu_set_ctable_ptr((unsigned long)srmmu_ctx_table_phys);
1227 /* Stop from hanging here... */
1228 local_flush_tlb_all();
1234 srmmu_allocate_ptable_skeleton(sparc_iomap.start, IOBASE_END);
1235 srmmu_allocate_ptable_skeleton(DVMA_VADDR, DVMA_END);
1237 srmmu_allocate_ptable_skeleton(
1238 __fix_to_virt(__end_of_fixed_addresses - 1), FIXADDR_TOP);
1239 srmmu_allocate_ptable_skeleton(PKMAP_BASE, PKMAP_END);
1241 pgd = pgd_offset_k(PKMAP_BASE);
1242 pmd = srmmu_pmd_offset(pgd, PKMAP_BASE);
1243 pte = srmmu_pte_offset(pmd, PKMAP_BASE);
1244 pkmap_page_table = pte;
1249 sparc_context_init(num_contexts);
1254 unsigned long zones_size[MAX_NR_ZONES];
1255 unsigned long zholes_size[MAX_NR_ZONES];
1256 unsigned long npages;
1259 for (znum = 0; znum < MAX_NR_ZONES; znum++)
1260 zones_size[znum] = zholes_size[znum] = 0;
1262 npages = max_low_pfn - pfn_base;
1264 zones_size[ZONE_DMA] = npages;
1265 zholes_size[ZONE_DMA] = npages - pages_avail;
1267 npages = highend_pfn - max_low_pfn;
1268 zones_size[ZONE_HIGHMEM] = npages;
1269 zholes_size[ZONE_HIGHMEM] = npages - calc_highpages();
1271 free_area_init_node(0, zones_size, pfn_base, zholes_size);
1275 static void srmmu_mmu_info(struct seq_file *m)
1280 "nocache total\t: %ld\n"
1281 "nocache used\t: %d\n",
1285 srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT);
1288 static void srmmu_update_mmu_cache(struct vm_area_struct * vma, unsigned long address, pte_t pte)
1292 static void srmmu_destroy_context(struct mm_struct *mm)
1295 if(mm->context != NO_CONTEXT) {
1297 srmmu_ctxd_set(&srmmu_context_table[mm->context], srmmu_swapper_pg_dir);
1299 spin_lock(&srmmu_context_spinlock);
1300 free_context(mm->context);
1301 spin_unlock(&srmmu_context_spinlock);
1302 mm->context = NO_CONTEXT;
1306 /* Init various srmmu chip types. */
1307 static void __init srmmu_is_bad(void)
1309 prom_printf("Could not determine SRMMU chip type.\n");
1313 static void __init init_vac_layout(void)
1320 unsigned long max_size = 0;
1321 unsigned long min_line_size = 0x10000000;
1324 nd = prom_getchild(prom_root_node);
1325 while((nd = prom_getsibling(nd)) != 0) {
1326 prom_getstring(nd, "device_type", node_str, sizeof(node_str));
1327 if(!strcmp(node_str, "cpu")) {
1328 vac_line_size = prom_getint(nd, "cache-line-size");
1329 if (vac_line_size == -1) {
1330 prom_printf("can't determine cache-line-size, "
1334 cache_lines = prom_getint(nd, "cache-nlines");
1335 if (cache_lines == -1) {
1336 prom_printf("can't determine cache-nlines, halting.\n");
1340 vac_cache_size = cache_lines * vac_line_size;
1342 if(vac_cache_size > max_size)
1343 max_size = vac_cache_size;
1344 if(vac_line_size < min_line_size)
1345 min_line_size = vac_line_size;
1346 //FIXME: cpus not contiguous!!
1348 if (cpu >= nr_cpu_ids || !cpu_online(cpu))
1356 prom_printf("No CPU nodes found, halting.\n");
1360 vac_cache_size = max_size;
1361 vac_line_size = min_line_size;
1363 printk("SRMMU: Using VAC size of %d bytes, line size %d bytes.\n",
1364 (int)vac_cache_size, (int)vac_line_size);
1367 static void __cpuinit poke_hypersparc(void)
1369 volatile unsigned long clear;
1370 unsigned long mreg = srmmu_get_mmureg();
1372 hyper_flush_unconditional_combined();
1374 mreg &= ~(HYPERSPARC_CWENABLE);
1375 mreg |= (HYPERSPARC_CENABLE | HYPERSPARC_WBENABLE);
1376 mreg |= (HYPERSPARC_CMODE);
1378 srmmu_set_mmureg(mreg);
1380 #if 0 /* XXX I think this is bad news... -DaveM */
1381 hyper_clear_all_tags();
1384 put_ross_icr(HYPERSPARC_ICCR_FTD | HYPERSPARC_ICCR_ICE);
1385 hyper_flush_whole_icache();
1386 clear = srmmu_get_faddr();
1387 clear = srmmu_get_fstatus();
1390 static void __init init_hypersparc(void)
1392 srmmu_name = "ROSS HyperSparc";
1393 srmmu_modtype = HyperSparc;
1399 BTFIXUPSET_CALL(flush_cache_all, hypersparc_flush_cache_all, BTFIXUPCALL_NORM);
1400 BTFIXUPSET_CALL(flush_cache_mm, hypersparc_flush_cache_mm, BTFIXUPCALL_NORM);
1401 BTFIXUPSET_CALL(flush_cache_range, hypersparc_flush_cache_range, BTFIXUPCALL_NORM);
1402 BTFIXUPSET_CALL(flush_cache_page, hypersparc_flush_cache_page, BTFIXUPCALL_NORM);
1404 BTFIXUPSET_CALL(flush_tlb_all, hypersparc_flush_tlb_all, BTFIXUPCALL_NORM);
1405 BTFIXUPSET_CALL(flush_tlb_mm, hypersparc_flush_tlb_mm, BTFIXUPCALL_NORM);
1406 BTFIXUPSET_CALL(flush_tlb_range, hypersparc_flush_tlb_range, BTFIXUPCALL_NORM);
1407 BTFIXUPSET_CALL(flush_tlb_page, hypersparc_flush_tlb_page, BTFIXUPCALL_NORM);
1409 BTFIXUPSET_CALL(__flush_page_to_ram, hypersparc_flush_page_to_ram, BTFIXUPCALL_NORM);
1410 BTFIXUPSET_CALL(flush_sig_insns, hypersparc_flush_sig_insns, BTFIXUPCALL_NORM);
1411 BTFIXUPSET_CALL(flush_page_for_dma, hypersparc_flush_page_for_dma, BTFIXUPCALL_NOP);
1414 poke_srmmu = poke_hypersparc;
1416 hypersparc_setup_blockops();
1419 static void __cpuinit poke_cypress(void)
1421 unsigned long mreg = srmmu_get_mmureg();
1422 unsigned long faddr, tagval;
1423 volatile unsigned long cypress_sucks;
1424 volatile unsigned long clear;
1426 clear = srmmu_get_faddr();
1427 clear = srmmu_get_fstatus();
1429 if (!(mreg & CYPRESS_CENABLE)) {
1430 for(faddr = 0x0; faddr < 0x10000; faddr += 20) {
1431 __asm__ __volatile__("sta %%g0, [%0 + %1] %2\n\t"
1432 "sta %%g0, [%0] %2\n\t" : :
1433 "r" (faddr), "r" (0x40000),
1434 "i" (ASI_M_DATAC_TAG));
1437 for(faddr = 0; faddr < 0x10000; faddr += 0x20) {
1438 __asm__ __volatile__("lda [%1 + %2] %3, %0\n\t" :
1440 "r" (faddr), "r" (0x40000),
1441 "i" (ASI_M_DATAC_TAG));
1443 /* If modified and valid, kick it. */
1444 if((tagval & 0x60) == 0x60)
1445 cypress_sucks = *(unsigned long *)
1446 (0xf0020000 + faddr);
1450 /* And one more, for our good neighbor, Mr. Broken Cypress. */
1451 clear = srmmu_get_faddr();
1452 clear = srmmu_get_fstatus();
1454 mreg |= (CYPRESS_CENABLE | CYPRESS_CMODE);
1455 srmmu_set_mmureg(mreg);
1458 static void __init init_cypress_common(void)
1462 BTFIXUPSET_CALL(flush_cache_all, cypress_flush_cache_all, BTFIXUPCALL_NORM);
1463 BTFIXUPSET_CALL(flush_cache_mm, cypress_flush_cache_mm, BTFIXUPCALL_NORM);
1464 BTFIXUPSET_CALL(flush_cache_range, cypress_flush_cache_range, BTFIXUPCALL_NORM);
1465 BTFIXUPSET_CALL(flush_cache_page, cypress_flush_cache_page, BTFIXUPCALL_NORM);
1467 BTFIXUPSET_CALL(flush_tlb_all, cypress_flush_tlb_all, BTFIXUPCALL_NORM);
1468 BTFIXUPSET_CALL(flush_tlb_mm, cypress_flush_tlb_mm, BTFIXUPCALL_NORM);
1469 BTFIXUPSET_CALL(flush_tlb_page, cypress_flush_tlb_page, BTFIXUPCALL_NORM);
1470 BTFIXUPSET_CALL(flush_tlb_range, cypress_flush_tlb_range, BTFIXUPCALL_NORM);
1473 BTFIXUPSET_CALL(__flush_page_to_ram, cypress_flush_page_to_ram, BTFIXUPCALL_NORM);
1474 BTFIXUPSET_CALL(flush_sig_insns, cypress_flush_sig_insns, BTFIXUPCALL_NOP);
1475 BTFIXUPSET_CALL(flush_page_for_dma, cypress_flush_page_for_dma, BTFIXUPCALL_NOP);
1477 poke_srmmu = poke_cypress;
1480 static void __init init_cypress_604(void)
1482 srmmu_name = "ROSS Cypress-604(UP)";
1483 srmmu_modtype = Cypress;
1484 init_cypress_common();
1487 static void __init init_cypress_605(unsigned long mrev)
1489 srmmu_name = "ROSS Cypress-605(MP)";
1491 srmmu_modtype = Cypress_vE;
1492 hwbug_bitmask |= HWBUG_COPYBACK_BROKEN;
1495 srmmu_modtype = Cypress_vD;
1496 hwbug_bitmask |= HWBUG_ASIFLUSH_BROKEN;
1498 srmmu_modtype = Cypress;
1501 init_cypress_common();
1504 static void __cpuinit poke_swift(void)
1508 /* Clear any crap from the cache or else... */
1509 swift_flush_cache_all();
1511 /* Enable I & D caches */
1512 mreg = srmmu_get_mmureg();
1513 mreg |= (SWIFT_IE | SWIFT_DE);
1515 * The Swift branch folding logic is completely broken. At
1516 * trap time, if things are just right, if can mistakenly
1517 * think that a trap is coming from kernel mode when in fact
1518 * it is coming from user mode (it mis-executes the branch in
1519 * the trap code). So you see things like crashme completely
1520 * hosing your machine which is completely unacceptable. Turn
1521 * this shit off... nice job Fujitsu.
1523 mreg &= ~(SWIFT_BF);
1524 srmmu_set_mmureg(mreg);
1527 #define SWIFT_MASKID_ADDR 0x10003018
1528 static void __init init_swift(void)
1530 unsigned long swift_rev;
1532 __asm__ __volatile__("lda [%1] %2, %0\n\t"
1533 "srl %0, 0x18, %0\n\t" :
1535 "r" (SWIFT_MASKID_ADDR), "i" (ASI_M_BYPASS));
1536 srmmu_name = "Fujitsu Swift";
1542 srmmu_modtype = Swift_lots_o_bugs;
1543 hwbug_bitmask |= (HWBUG_KERN_ACCBROKEN | HWBUG_KERN_CBITBROKEN);
1545 * Gee george, I wonder why Sun is so hush hush about
1546 * this hardware bug... really braindamage stuff going
1547 * on here. However I think we can find a way to avoid
1548 * all of the workaround overhead under Linux. Basically,
1549 * any page fault can cause kernel pages to become user
1550 * accessible (the mmu gets confused and clears some of
1551 * the ACC bits in kernel ptes). Aha, sounds pretty
1552 * horrible eh? But wait, after extensive testing it appears
1553 * that if you use pgd_t level large kernel pte's (like the
1554 * 4MB pages on the Pentium) the bug does not get tripped
1555 * at all. This avoids almost all of the major overhead.
1556 * Welcome to a world where your vendor tells you to,
1557 * "apply this kernel patch" instead of "sorry for the
1558 * broken hardware, send it back and we'll give you
1559 * properly functioning parts"
1564 srmmu_modtype = Swift_bad_c;
1565 hwbug_bitmask |= HWBUG_KERN_CBITBROKEN;
1567 * You see Sun allude to this hardware bug but never
1568 * admit things directly, they'll say things like,
1569 * "the Swift chip cache problems" or similar.
1573 srmmu_modtype = Swift_ok;
1577 BTFIXUPSET_CALL(flush_cache_all, swift_flush_cache_all, BTFIXUPCALL_NORM);
1578 BTFIXUPSET_CALL(flush_cache_mm, swift_flush_cache_mm, BTFIXUPCALL_NORM);
1579 BTFIXUPSET_CALL(flush_cache_page, swift_flush_cache_page, BTFIXUPCALL_NORM);
1580 BTFIXUPSET_CALL(flush_cache_range, swift_flush_cache_range, BTFIXUPCALL_NORM);
1583 BTFIXUPSET_CALL(flush_tlb_all, swift_flush_tlb_all, BTFIXUPCALL_NORM);
1584 BTFIXUPSET_CALL(flush_tlb_mm, swift_flush_tlb_mm, BTFIXUPCALL_NORM);
1585 BTFIXUPSET_CALL(flush_tlb_page, swift_flush_tlb_page, BTFIXUPCALL_NORM);
1586 BTFIXUPSET_CALL(flush_tlb_range, swift_flush_tlb_range, BTFIXUPCALL_NORM);
1588 BTFIXUPSET_CALL(__flush_page_to_ram, swift_flush_page_to_ram, BTFIXUPCALL_NORM);
1589 BTFIXUPSET_CALL(flush_sig_insns, swift_flush_sig_insns, BTFIXUPCALL_NORM);
1590 BTFIXUPSET_CALL(flush_page_for_dma, swift_flush_page_for_dma, BTFIXUPCALL_NORM);
1592 BTFIXUPSET_CALL(update_mmu_cache, swift_update_mmu_cache, BTFIXUPCALL_NORM);
1594 flush_page_for_dma_global = 0;
1597 * Are you now convinced that the Swift is one of the
1598 * biggest VLSI abortions of all time? Bravo Fujitsu!
1599 * Fujitsu, the !#?!%$'d up processor people. I bet if
1600 * you examined the microcode of the Swift you'd find
1601 * XXX's all over the place.
1603 poke_srmmu = poke_swift;
1606 static void turbosparc_flush_cache_all(void)
1608 flush_user_windows();
1609 turbosparc_idflash_clear();
1612 static void turbosparc_flush_cache_mm(struct mm_struct *mm)
1615 flush_user_windows();
1616 turbosparc_idflash_clear();
1620 static void turbosparc_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1622 FLUSH_BEGIN(vma->vm_mm)
1623 flush_user_windows();
1624 turbosparc_idflash_clear();
1628 static void turbosparc_flush_cache_page(struct vm_area_struct *vma, unsigned long page)
1630 FLUSH_BEGIN(vma->vm_mm)
1631 flush_user_windows();
1632 if (vma->vm_flags & VM_EXEC)
1633 turbosparc_flush_icache();
1634 turbosparc_flush_dcache();
1638 /* TurboSparc is copy-back, if we turn it on, but this does not work. */
1639 static void turbosparc_flush_page_to_ram(unsigned long page)
1641 #ifdef TURBOSPARC_WRITEBACK
1642 volatile unsigned long clear;
1644 if (srmmu_hwprobe(page))
1645 turbosparc_flush_page_cache(page);
1646 clear = srmmu_get_fstatus();
1650 static void turbosparc_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr)
1654 static void turbosparc_flush_page_for_dma(unsigned long page)
1656 turbosparc_flush_dcache();
1659 static void turbosparc_flush_tlb_all(void)
1661 srmmu_flush_whole_tlb();
1664 static void turbosparc_flush_tlb_mm(struct mm_struct *mm)
1667 srmmu_flush_whole_tlb();
1671 static void turbosparc_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1673 FLUSH_BEGIN(vma->vm_mm)
1674 srmmu_flush_whole_tlb();
1678 static void turbosparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
1680 FLUSH_BEGIN(vma->vm_mm)
1681 srmmu_flush_whole_tlb();
1686 static void __cpuinit poke_turbosparc(void)
1688 unsigned long mreg = srmmu_get_mmureg();
1689 unsigned long ccreg;
1691 /* Clear any crap from the cache or else... */
1692 turbosparc_flush_cache_all();
1693 mreg &= ~(TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* Temporarily disable I & D caches */
1694 mreg &= ~(TURBOSPARC_PCENABLE); /* Don't check parity */
1695 srmmu_set_mmureg(mreg);
1697 ccreg = turbosparc_get_ccreg();
1699 #ifdef TURBOSPARC_WRITEBACK
1700 ccreg |= (TURBOSPARC_SNENABLE); /* Do DVMA snooping in Dcache */
1701 ccreg &= ~(TURBOSPARC_uS2 | TURBOSPARC_WTENABLE);
1702 /* Write-back D-cache, emulate VLSI
1703 * abortion number three, not number one */
1705 /* For now let's play safe, optimize later */
1706 ccreg |= (TURBOSPARC_SNENABLE | TURBOSPARC_WTENABLE);
1707 /* Do DVMA snooping in Dcache, Write-thru D-cache */
1708 ccreg &= ~(TURBOSPARC_uS2);
1709 /* Emulate VLSI abortion number three, not number one */
1712 switch (ccreg & 7) {
1713 case 0: /* No SE cache */
1714 case 7: /* Test mode */
1717 ccreg |= (TURBOSPARC_SCENABLE);
1719 turbosparc_set_ccreg (ccreg);
1721 mreg |= (TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* I & D caches on */
1722 mreg |= (TURBOSPARC_ICSNOOP); /* Icache snooping on */
1723 srmmu_set_mmureg(mreg);
1726 static void __init init_turbosparc(void)
1728 srmmu_name = "Fujitsu TurboSparc";
1729 srmmu_modtype = TurboSparc;
1731 BTFIXUPSET_CALL(flush_cache_all, turbosparc_flush_cache_all, BTFIXUPCALL_NORM);
1732 BTFIXUPSET_CALL(flush_cache_mm, turbosparc_flush_cache_mm, BTFIXUPCALL_NORM);
1733 BTFIXUPSET_CALL(flush_cache_page, turbosparc_flush_cache_page, BTFIXUPCALL_NORM);
1734 BTFIXUPSET_CALL(flush_cache_range, turbosparc_flush_cache_range, BTFIXUPCALL_NORM);
1736 BTFIXUPSET_CALL(flush_tlb_all, turbosparc_flush_tlb_all, BTFIXUPCALL_NORM);
1737 BTFIXUPSET_CALL(flush_tlb_mm, turbosparc_flush_tlb_mm, BTFIXUPCALL_NORM);
1738 BTFIXUPSET_CALL(flush_tlb_page, turbosparc_flush_tlb_page, BTFIXUPCALL_NORM);
1739 BTFIXUPSET_CALL(flush_tlb_range, turbosparc_flush_tlb_range, BTFIXUPCALL_NORM);
1741 BTFIXUPSET_CALL(__flush_page_to_ram, turbosparc_flush_page_to_ram, BTFIXUPCALL_NORM);
1743 BTFIXUPSET_CALL(flush_sig_insns, turbosparc_flush_sig_insns, BTFIXUPCALL_NOP);
1744 BTFIXUPSET_CALL(flush_page_for_dma, turbosparc_flush_page_for_dma, BTFIXUPCALL_NORM);
1746 poke_srmmu = poke_turbosparc;
1749 static void __cpuinit poke_tsunami(void)
1751 unsigned long mreg = srmmu_get_mmureg();
1753 tsunami_flush_icache();
1754 tsunami_flush_dcache();
1755 mreg &= ~TSUNAMI_ITD;
1756 mreg |= (TSUNAMI_IENAB | TSUNAMI_DENAB);
1757 srmmu_set_mmureg(mreg);
1760 static void __init init_tsunami(void)
1763 * Tsunami's pretty sane, Sun and TI actually got it
1764 * somewhat right this time. Fujitsu should have
1765 * taken some lessons from them.
1768 srmmu_name = "TI Tsunami";
1769 srmmu_modtype = Tsunami;
1771 BTFIXUPSET_CALL(flush_cache_all, tsunami_flush_cache_all, BTFIXUPCALL_NORM);
1772 BTFIXUPSET_CALL(flush_cache_mm, tsunami_flush_cache_mm, BTFIXUPCALL_NORM);
1773 BTFIXUPSET_CALL(flush_cache_page, tsunami_flush_cache_page, BTFIXUPCALL_NORM);
1774 BTFIXUPSET_CALL(flush_cache_range, tsunami_flush_cache_range, BTFIXUPCALL_NORM);
1777 BTFIXUPSET_CALL(flush_tlb_all, tsunami_flush_tlb_all, BTFIXUPCALL_NORM);
1778 BTFIXUPSET_CALL(flush_tlb_mm, tsunami_flush_tlb_mm, BTFIXUPCALL_NORM);
1779 BTFIXUPSET_CALL(flush_tlb_page, tsunami_flush_tlb_page, BTFIXUPCALL_NORM);
1780 BTFIXUPSET_CALL(flush_tlb_range, tsunami_flush_tlb_range, BTFIXUPCALL_NORM);
1782 BTFIXUPSET_CALL(__flush_page_to_ram, tsunami_flush_page_to_ram, BTFIXUPCALL_NOP);
1783 BTFIXUPSET_CALL(flush_sig_insns, tsunami_flush_sig_insns, BTFIXUPCALL_NORM);
1784 BTFIXUPSET_CALL(flush_page_for_dma, tsunami_flush_page_for_dma, BTFIXUPCALL_NORM);
1786 poke_srmmu = poke_tsunami;
1788 tsunami_setup_blockops();
1791 static void __cpuinit poke_viking(void)
1793 unsigned long mreg = srmmu_get_mmureg();
1794 static int smp_catch;
1796 if(viking_mxcc_present) {
1797 unsigned long mxcc_control = mxcc_get_creg();
1799 mxcc_control |= (MXCC_CTL_ECE | MXCC_CTL_PRE | MXCC_CTL_MCE);
1800 mxcc_control &= ~(MXCC_CTL_RRC);
1801 mxcc_set_creg(mxcc_control);
1804 * We don't need memory parity checks.
1805 * XXX This is a mess, have to dig out later. ecd.
1806 viking_mxcc_turn_off_parity(&mreg, &mxcc_control);
1809 /* We do cache ptables on MXCC. */
1810 mreg |= VIKING_TCENABLE;
1812 unsigned long bpreg;
1814 mreg &= ~(VIKING_TCENABLE);
1816 /* Must disable mixed-cmd mode here for other cpu's. */
1817 bpreg = viking_get_bpreg();
1818 bpreg &= ~(VIKING_ACTION_MIX);
1819 viking_set_bpreg(bpreg);
1821 /* Just in case PROM does something funny. */
1826 mreg |= VIKING_SPENABLE;
1827 mreg |= (VIKING_ICENABLE | VIKING_DCENABLE);
1828 mreg |= VIKING_SBENABLE;
1829 mreg &= ~(VIKING_ACENABLE);
1830 srmmu_set_mmureg(mreg);
1833 static void __init init_viking(void)
1835 unsigned long mreg = srmmu_get_mmureg();
1837 /* Ahhh, the viking. SRMMU VLSI abortion number two... */
1838 if(mreg & VIKING_MMODE) {
1839 srmmu_name = "TI Viking";
1840 viking_mxcc_present = 0;
1844 * We need this to make sure old viking takes no hits
1845 * on it's cache for dma snoops to workaround the
1846 * "load from non-cacheable memory" interrupt bug.
1847 * This is only necessary because of the new way in
1848 * which we use the IOMMU.
1850 BTFIXUPSET_CALL(flush_page_for_dma, viking_flush_page, BTFIXUPCALL_NORM);
1852 flush_page_for_dma_global = 0;
1854 srmmu_name = "TI Viking/MXCC";
1855 viking_mxcc_present = 1;
1857 srmmu_cache_pagetables = 1;
1859 /* MXCC vikings lack the DMA snooping bug. */
1860 BTFIXUPSET_CALL(flush_page_for_dma, viking_flush_page_for_dma, BTFIXUPCALL_NOP);
1863 BTFIXUPSET_CALL(flush_cache_all, viking_flush_cache_all, BTFIXUPCALL_NORM);
1864 BTFIXUPSET_CALL(flush_cache_mm, viking_flush_cache_mm, BTFIXUPCALL_NORM);
1865 BTFIXUPSET_CALL(flush_cache_page, viking_flush_cache_page, BTFIXUPCALL_NORM);
1866 BTFIXUPSET_CALL(flush_cache_range, viking_flush_cache_range, BTFIXUPCALL_NORM);
1869 if (sparc_cpu_model == sun4d) {
1870 BTFIXUPSET_CALL(flush_tlb_all, sun4dsmp_flush_tlb_all, BTFIXUPCALL_NORM);
1871 BTFIXUPSET_CALL(flush_tlb_mm, sun4dsmp_flush_tlb_mm, BTFIXUPCALL_NORM);
1872 BTFIXUPSET_CALL(flush_tlb_page, sun4dsmp_flush_tlb_page, BTFIXUPCALL_NORM);
1873 BTFIXUPSET_CALL(flush_tlb_range, sun4dsmp_flush_tlb_range, BTFIXUPCALL_NORM);
1877 BTFIXUPSET_CALL(flush_tlb_all, viking_flush_tlb_all, BTFIXUPCALL_NORM);
1878 BTFIXUPSET_CALL(flush_tlb_mm, viking_flush_tlb_mm, BTFIXUPCALL_NORM);
1879 BTFIXUPSET_CALL(flush_tlb_page, viking_flush_tlb_page, BTFIXUPCALL_NORM);
1880 BTFIXUPSET_CALL(flush_tlb_range, viking_flush_tlb_range, BTFIXUPCALL_NORM);
1883 BTFIXUPSET_CALL(__flush_page_to_ram, viking_flush_page_to_ram, BTFIXUPCALL_NOP);
1884 BTFIXUPSET_CALL(flush_sig_insns, viking_flush_sig_insns, BTFIXUPCALL_NOP);
1886 poke_srmmu = poke_viking;
1889 #ifdef CONFIG_SPARC_LEON
1891 void __init poke_leonsparc(void)
1895 void __init init_leon(void)
1898 srmmu_name = "LEON";
1900 BTFIXUPSET_CALL(flush_cache_all, leon_flush_cache_all,
1902 BTFIXUPSET_CALL(flush_cache_mm, leon_flush_cache_all,
1904 BTFIXUPSET_CALL(flush_cache_page, leon_flush_pcache_all,
1906 BTFIXUPSET_CALL(flush_cache_range, leon_flush_cache_all,
1908 BTFIXUPSET_CALL(flush_page_for_dma, leon_flush_dcache_all,
1911 BTFIXUPSET_CALL(flush_tlb_all, leon_flush_tlb_all, BTFIXUPCALL_NORM);
1912 BTFIXUPSET_CALL(flush_tlb_mm, leon_flush_tlb_all, BTFIXUPCALL_NORM);
1913 BTFIXUPSET_CALL(flush_tlb_page, leon_flush_tlb_all, BTFIXUPCALL_NORM);
1914 BTFIXUPSET_CALL(flush_tlb_range, leon_flush_tlb_all, BTFIXUPCALL_NORM);
1916 BTFIXUPSET_CALL(__flush_page_to_ram, leon_flush_cache_all,
1918 BTFIXUPSET_CALL(flush_sig_insns, leon_flush_cache_all, BTFIXUPCALL_NOP);
1920 poke_srmmu = poke_leonsparc;
1922 srmmu_cache_pagetables = 0;
1924 leon_flush_during_switch = leon_flush_needed();
1928 /* Probe for the srmmu chip version. */
1929 static void __init get_srmmu_type(void)
1931 unsigned long mreg, psr;
1932 unsigned long mod_typ, mod_rev, psr_typ, psr_vers;
1934 srmmu_modtype = SRMMU_INVAL_MOD;
1937 mreg = srmmu_get_mmureg(); psr = get_psr();
1938 mod_typ = (mreg & 0xf0000000) >> 28;
1939 mod_rev = (mreg & 0x0f000000) >> 24;
1940 psr_typ = (psr >> 28) & 0xf;
1941 psr_vers = (psr >> 24) & 0xf;
1943 /* First, check for sparc-leon. */
1944 if (sparc_cpu_model == sparc_leon) {
1949 /* Second, check for HyperSparc or Cypress. */
1953 /* UP or MP Hypersparc */
1958 /* Uniprocessor Cypress */
1964 /* _REALLY OLD_ Cypress MP chips... */
1968 /* MP Cypress mmu/cache-controller */
1969 init_cypress_605(mod_rev);
1972 /* Some other Cypress revision, assume a 605. */
1973 init_cypress_605(mod_rev);
1980 * Now Fujitsu TurboSparc. It might happen that it is
1981 * in Swift emulation mode, so we will check later...
1983 if (psr_typ == 0 && psr_vers == 5) {
1988 /* Next check for Fujitsu Swift. */
1989 if(psr_typ == 0 && psr_vers == 4) {
1993 /* Look if it is not a TurboSparc emulating Swift... */
1994 cpunode = prom_getchild(prom_root_node);
1995 while((cpunode = prom_getsibling(cpunode)) != 0) {
1996 prom_getstring(cpunode, "device_type", node_str, sizeof(node_str));
1997 if(!strcmp(node_str, "cpu")) {
1998 if (!prom_getintdefault(cpunode, "psr-implementation", 1) &&
1999 prom_getintdefault(cpunode, "psr-version", 1) == 5) {
2011 /* Now the Viking family of srmmu. */
2014 ((psr_vers == 1) && (mod_typ == 0) && (mod_rev == 0)))) {
2019 /* Finally the Tsunami. */
2020 if(psr_typ == 4 && psr_vers == 1 && (mod_typ || mod_rev)) {
2029 extern unsigned long spwin_mmu_patchme, fwin_mmu_patchme,
2030 tsetup_mmu_patchme, rtrap_mmu_patchme;
2032 extern unsigned long spwin_srmmu_stackchk, srmmu_fwin_stackchk,
2033 tsetup_srmmu_stackchk, srmmu_rett_stackchk;
2036 /* Local cross-calls. */
2037 static void smp_flush_page_for_dma(unsigned long page)
2039 xc1((smpfunc_t) BTFIXUP_CALL(local_flush_page_for_dma), page);
2040 local_flush_page_for_dma(page);
2045 /* Load up routines and constants for sun4m and sun4d mmu */
2046 void __init load_mmu(void)
2048 extern void ld_mmu_iommu(void);
2049 extern void ld_mmu_iounit(void);
2050 extern void ___xchg32_sun4md(void);
2054 BTFIXUPSET_CALL(___xchg32, ___xchg32_sun4md, BTFIXUPCALL_SWAPG1G2);
2057 BTFIXUPSET_CALL(pgd_page_vaddr, srmmu_pgd_page, BTFIXUPCALL_NORM);
2059 BTFIXUPSET_CALL(pgd_set, srmmu_pgd_set, BTFIXUPCALL_NORM);
2060 BTFIXUPSET_CALL(pmd_set, srmmu_pmd_set, BTFIXUPCALL_NORM);
2061 BTFIXUPSET_CALL(pmd_populate, srmmu_pmd_populate, BTFIXUPCALL_NORM);
2063 BTFIXUPSET_INT(pte_modify_mask, SRMMU_CHG_MASK);
2064 BTFIXUPSET_CALL(pmd_offset, srmmu_pmd_offset, BTFIXUPCALL_NORM);
2065 BTFIXUPSET_CALL(pte_offset_kernel, srmmu_pte_offset, BTFIXUPCALL_NORM);
2067 BTFIXUPSET_CALL(free_pte_fast, srmmu_free_pte_fast, BTFIXUPCALL_NORM);
2068 BTFIXUPSET_CALL(pte_free, srmmu_pte_free, BTFIXUPCALL_NORM);
2069 BTFIXUPSET_CALL(pte_alloc_one_kernel, srmmu_pte_alloc_one_kernel, BTFIXUPCALL_NORM);
2070 BTFIXUPSET_CALL(pte_alloc_one, srmmu_pte_alloc_one, BTFIXUPCALL_NORM);
2071 BTFIXUPSET_CALL(free_pmd_fast, srmmu_pmd_free, BTFIXUPCALL_NORM);
2072 BTFIXUPSET_CALL(pmd_alloc_one, srmmu_pmd_alloc_one, BTFIXUPCALL_NORM);
2073 BTFIXUPSET_CALL(free_pgd_fast, srmmu_free_pgd_fast, BTFIXUPCALL_NORM);
2074 BTFIXUPSET_CALL(get_pgd_fast, srmmu_get_pgd_fast, BTFIXUPCALL_NORM);
2076 BTFIXUPSET_HALF(pte_filei, SRMMU_FILE);
2077 BTFIXUPSET_HALF(pte_wrprotecti, SRMMU_WRITE);
2078 BTFIXUPSET_HALF(pte_mkcleani, SRMMU_DIRTY);
2079 BTFIXUPSET_HALF(pte_mkoldi, SRMMU_REF);
2080 BTFIXUPSET_CALL(pte_mkwrite, srmmu_pte_mkwrite, BTFIXUPCALL_ORINT(SRMMU_WRITE));
2081 BTFIXUPSET_CALL(pte_mkdirty, srmmu_pte_mkdirty, BTFIXUPCALL_ORINT(SRMMU_DIRTY));
2082 BTFIXUPSET_CALL(pte_mkyoung, srmmu_pte_mkyoung, BTFIXUPCALL_ORINT(SRMMU_REF));
2083 BTFIXUPSET_CALL(update_mmu_cache, srmmu_update_mmu_cache, BTFIXUPCALL_NOP);
2084 BTFIXUPSET_CALL(destroy_context, srmmu_destroy_context, BTFIXUPCALL_NORM);
2086 BTFIXUPSET_CALL(sparc_mapiorange, srmmu_mapiorange, BTFIXUPCALL_NORM);
2087 BTFIXUPSET_CALL(sparc_unmapiorange, srmmu_unmapiorange, BTFIXUPCALL_NORM);
2089 BTFIXUPSET_CALL(__swp_type, srmmu_swp_type, BTFIXUPCALL_NORM);
2090 BTFIXUPSET_CALL(__swp_offset, srmmu_swp_offset, BTFIXUPCALL_NORM);
2091 BTFIXUPSET_CALL(__swp_entry, srmmu_swp_entry, BTFIXUPCALL_NORM);
2093 BTFIXUPSET_CALL(mmu_info, srmmu_mmu_info, BTFIXUPCALL_NORM);
2098 /* El switcheroo... */
2100 BTFIXUPCOPY_CALL(local_flush_cache_all, flush_cache_all);
2101 BTFIXUPCOPY_CALL(local_flush_cache_mm, flush_cache_mm);
2102 BTFIXUPCOPY_CALL(local_flush_cache_range, flush_cache_range);
2103 BTFIXUPCOPY_CALL(local_flush_cache_page, flush_cache_page);
2104 BTFIXUPCOPY_CALL(local_flush_tlb_all, flush_tlb_all);
2105 BTFIXUPCOPY_CALL(local_flush_tlb_mm, flush_tlb_mm);
2106 BTFIXUPCOPY_CALL(local_flush_tlb_range, flush_tlb_range);
2107 BTFIXUPCOPY_CALL(local_flush_tlb_page, flush_tlb_page);
2108 BTFIXUPCOPY_CALL(local_flush_page_to_ram, __flush_page_to_ram);
2109 BTFIXUPCOPY_CALL(local_flush_sig_insns, flush_sig_insns);
2110 BTFIXUPCOPY_CALL(local_flush_page_for_dma, flush_page_for_dma);
2112 BTFIXUPSET_CALL(flush_cache_all, smp_flush_cache_all, BTFIXUPCALL_NORM);
2113 BTFIXUPSET_CALL(flush_cache_mm, smp_flush_cache_mm, BTFIXUPCALL_NORM);
2114 BTFIXUPSET_CALL(flush_cache_range, smp_flush_cache_range, BTFIXUPCALL_NORM);
2115 BTFIXUPSET_CALL(flush_cache_page, smp_flush_cache_page, BTFIXUPCALL_NORM);
2116 if (sparc_cpu_model != sun4d &&
2117 sparc_cpu_model != sparc_leon) {
2118 BTFIXUPSET_CALL(flush_tlb_all, smp_flush_tlb_all, BTFIXUPCALL_NORM);
2119 BTFIXUPSET_CALL(flush_tlb_mm, smp_flush_tlb_mm, BTFIXUPCALL_NORM);
2120 BTFIXUPSET_CALL(flush_tlb_range, smp_flush_tlb_range, BTFIXUPCALL_NORM);
2121 BTFIXUPSET_CALL(flush_tlb_page, smp_flush_tlb_page, BTFIXUPCALL_NORM);
2123 BTFIXUPSET_CALL(__flush_page_to_ram, smp_flush_page_to_ram, BTFIXUPCALL_NORM);
2124 BTFIXUPSET_CALL(flush_sig_insns, smp_flush_sig_insns, BTFIXUPCALL_NORM);
2125 BTFIXUPSET_CALL(flush_page_for_dma, smp_flush_page_for_dma, BTFIXUPCALL_NORM);
2127 if (poke_srmmu == poke_viking) {
2128 /* Avoid unnecessary cross calls. */
2129 BTFIXUPCOPY_CALL(flush_cache_all, local_flush_cache_all);
2130 BTFIXUPCOPY_CALL(flush_cache_mm, local_flush_cache_mm);
2131 BTFIXUPCOPY_CALL(flush_cache_range, local_flush_cache_range);
2132 BTFIXUPCOPY_CALL(flush_cache_page, local_flush_cache_page);
2133 BTFIXUPCOPY_CALL(__flush_page_to_ram, local_flush_page_to_ram);
2134 BTFIXUPCOPY_CALL(flush_sig_insns, local_flush_sig_insns);
2135 BTFIXUPCOPY_CALL(flush_page_for_dma, local_flush_page_for_dma);
2139 if (sparc_cpu_model == sun4d)
2144 if (sparc_cpu_model == sun4d)
2146 else if (sparc_cpu_model == sparc_leon)