arch/mips/mm/c-r4k.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
   7  * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org)
   8  * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
   9  */
  10 #include <linux/cpu_pm.h>
  11 #include <linux/hardirq.h>
  12 #include <linux/init.h>
  13 #include <linux/highmem.h>
  14 #include <linux/kernel.h>
  15 #include <linux/linkage.h>
  16 #include <linux/preempt.h>
  17 #include <linux/sched.h>
  18 #include <linux/smp.h>
  19 #include <linux/mm.h>
  20 #include <linux/module.h>
  21 #include <linux/bitops.h>
  22
  23 #include <asm/bcache.h>
  24 #include <asm/bootinfo.h>
  25 #include <asm/cache.h>
  26 #include <asm/cacheops.h>
  27 #include <asm/cpu.h>
  28 #include <asm/cpu-features.h>
  29 #include <asm/cpu-type.h>
  30 #include <asm/io.h>
  31 #include <asm/page.h>
  32 #include <asm/pgtable.h>
  33 #include <asm/r4kcache.h>
  34 #include <asm/sections.h>
  35 #include <asm/mmu_context.h>
  36 #include <asm/war.h>
  37 #include <asm/cacheflush.h> /* for run_uncached() */
  38 #include <asm/traps.h>
  39 #include <asm/dma-coherence.h>
  40
  41 /*
  42  * Special Variant of smp_call_function for use by cache functions:
  43  *
  44  *  o No return value
  45  *  o collapses to normal function call on UP kernels
  46  *  o collapses to normal function call on systems with a single shared
  47  *    primary cache.
  48  *  o doesn't disable interrupts on the local CPU
  49  */
  50 static inline void r4k_on_each_cpu(void (*func) (void *info), void *info)
  51 {
  52         preempt_disable();
  53
  54 #ifndef CONFIG_MIPS_MT_SMP
  55         smp_call_function(func, info, 1);
  56 #endif
  57         func(info);
  58         preempt_enable();
  59 }
  60
  61 #if defined(CONFIG_MIPS_CMP) || defined(CONFIG_MIPS_CPS)
  62 #define cpu_has_safe_index_cacheops 0
  63 #else
  64 #define cpu_has_safe_index_cacheops 1
  65 #endif
  66
  67 /*
  68  * Must die.
  69  */
  70 static unsigned long icache_size __read_mostly;
  71 static unsigned long dcache_size __read_mostly;
  72 static unsigned long scache_size __read_mostly;
  73
  74 /*
  75  * Dummy cache handling routines for machines without boardcaches
  76  */
  77 static void cache_noop(void) {}
  78
  79 static struct bcache_ops no_sc_ops = {
  80         .bc_enable = (void *)cache_noop,
  81         .bc_disable = (void *)cache_noop,
  82         .bc_wback_inv = (void *)cache_noop,
  83         .bc_inv = (void *)cache_noop
  84 };
  85
  86 struct bcache_ops *bcops = &no_sc_ops;
  87
  88 #define cpu_is_r4600_v1_x()     ((read_c0_prid() & 0xfffffff0) == 0x00002010)
  89 #define cpu_is_r4600_v2_x()     ((read_c0_prid() & 0xfffffff0) == 0x00002020)
  90
  91 #define R4600_HIT_CACHEOP_WAR_IMPL                                      \
  92 do {                                                                    \
  93         if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())            \
  94                 *(volatile unsigned long *)CKSEG1;                      \
  95         if (R4600_V1_HIT_CACHEOP_WAR)                                   \
  96                 __asm__ __volatile__("nop;nop;nop;nop");                \
  97 } while (0)
  98
  99 static void (*r4k_blast_dcache_page)(unsigned long addr);
 100
 101 static inline void r4k_blast_dcache_page_dc32(unsigned long addr)
 102 {
 103         R4600_HIT_CACHEOP_WAR_IMPL;
 104         blast_dcache32_page(addr);
 105 }
 106
 107 static inline void r4k_blast_dcache_page_dc64(unsigned long addr)
 108 {
 109         blast_dcache64_page(addr);
 110 }
 111
 112 static void r4k_blast_dcache_page_setup(void)
 113 {
 114         unsigned long  dc_lsize = cpu_dcache_line_size();
 115
 116         if (dc_lsize == 0)
 117                 r4k_blast_dcache_page = (void *)cache_noop;
 118         else if (dc_lsize == 16)
 119                 r4k_blast_dcache_page = blast_dcache16_page;
 120         else if (dc_lsize == 32)
 121                 r4k_blast_dcache_page = r4k_blast_dcache_page_dc32;
 122         else if (dc_lsize == 64)
 123                 r4k_blast_dcache_page = r4k_blast_dcache_page_dc64;
 124 }
 125
 126 #ifndef CONFIG_EVA
 127 #define r4k_blast_dcache_user_page  r4k_blast_dcache_page
 128 #else
 129
 130 static void (*r4k_blast_dcache_user_page)(unsigned long addr);
 131
 132 static void r4k_blast_dcache_user_page_setup(void)
 133 {
 134         unsigned long  dc_lsize = cpu_dcache_line_size();
 135
 136         if (dc_lsize == 0)
 137                 r4k_blast_dcache_user_page = (void *)cache_noop;
 138         else if (dc_lsize == 16)
 139                 r4k_blast_dcache_user_page = blast_dcache16_user_page;
 140         else if (dc_lsize == 32)
 141                 r4k_blast_dcache_user_page = blast_dcache32_user_page;
 142         else if (dc_lsize == 64)
 143                 r4k_blast_dcache_user_page = blast_dcache64_user_page;
 144 }
 145
 146 #endif
 147
 148 static void (* r4k_blast_dcache_page_indexed)(unsigned long addr);
 149
 150 static void r4k_blast_dcache_page_indexed_setup(void)
 151 {
 152         unsigned long dc_lsize = cpu_dcache_line_size();
 153
 154         if (dc_lsize == 0)
 155                 r4k_blast_dcache_page_indexed = (void *)cache_noop;
 156         else if (dc_lsize == 16)
 157                 r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed;
 158         else if (dc_lsize == 32)
 159                 r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed;
 160         else if (dc_lsize == 64)
 161                 r4k_blast_dcache_page_indexed = blast_dcache64_page_indexed;
 162 }
 163
 164 void (* r4k_blast_dcache)(void);
 165 EXPORT_SYMBOL(r4k_blast_dcache);
 166
 167 static void r4k_blast_dcache_setup(void)
 168 {
 169         unsigned long dc_lsize = cpu_dcache_line_size();
 170
 171         if (dc_lsize == 0)
 172                 r4k_blast_dcache = (void *)cache_noop;
 173         else if (dc_lsize == 16)
 174                 r4k_blast_dcache = blast_dcache16;
 175         else if (dc_lsize == 32)
 176                 r4k_blast_dcache = blast_dcache32;
 177         else if (dc_lsize == 64)
 178                 r4k_blast_dcache = blast_dcache64;
 179 }
 180
 181 /* force code alignment (used for TX49XX_ICACHE_INDEX_INV_WAR) */
 182 #define JUMP_TO_ALIGN(order) \
 183         __asm__ __volatile__( \
 184                 "b\t1f\n\t" \
 185                 ".align\t" #order "\n\t" \
 186                 "1:\n\t" \
 187                 )
 188 #define CACHE32_UNROLL32_ALIGN  JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */
 189 #define CACHE32_UNROLL32_ALIGN2 JUMP_TO_ALIGN(11)
 190
 191 static inline void blast_r4600_v1_icache32(void)
 192 {
 193         unsigned long flags;
 194
 195         local_irq_save(flags);
 196         blast_icache32();
 197         local_irq_restore(flags);
 198 }
 199
 200 static inline void tx49_blast_icache32(void)
 201 {
 202         unsigned long start = INDEX_BASE;
 203         unsigned long end = start + current_cpu_data.icache.waysize;
 204         unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
 205         unsigned long ws_end = current_cpu_data.icache.ways <<
 206                                current_cpu_data.icache.waybit;
 207         unsigned long ws, addr;
 208
 209         CACHE32_UNROLL32_ALIGN2;
 210         /* I'm in even chunk.  blast odd chunks */
 211         for (ws = 0; ws < ws_end; ws += ws_inc)
 212                 for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
 213                         cache32_unroll32(addr|ws, Index_Invalidate_I);
 214         CACHE32_UNROLL32_ALIGN;
 215         /* I'm in odd chunk.  blast even chunks */
 216         for (ws = 0; ws < ws_end; ws += ws_inc)
 217                 for (addr = start; addr < end; addr += 0x400 * 2)
 218                         cache32_unroll32(addr|ws, Index_Invalidate_I);
 219 }
 220
 221 static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page)
 222 {
 223         unsigned long flags;
 224
 225         local_irq_save(flags);
 226         blast_icache32_page_indexed(page);
 227         local_irq_restore(flags);
 228 }
 229
 230 static inline void tx49_blast_icache32_page_indexed(unsigned long page)
 231 {
 232         unsigned long indexmask = current_cpu_data.icache.waysize - 1;
 233         unsigned long start = INDEX_BASE + (page & indexmask);
 234         unsigned long end = start + PAGE_SIZE;
 235         unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
 236         unsigned long ws_end = current_cpu_data.icache.ways <<
 237                                current_cpu_data.icache.waybit;
 238         unsigned long ws, addr;
 239
 240         CACHE32_UNROLL32_ALIGN2;
 241         /* I'm in even chunk.  blast odd chunks */
 242         for (ws = 0; ws < ws_end; ws += ws_inc)
 243                 for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
 244                         cache32_unroll32(addr|ws, Index_Invalidate_I);
 245         CACHE32_UNROLL32_ALIGN;
 246         /* I'm in odd chunk.  blast even chunks */
 247         for (ws = 0; ws < ws_end; ws += ws_inc)
 248                 for (addr = start; addr < end; addr += 0x400 * 2)
 249                         cache32_unroll32(addr|ws, Index_Invalidate_I);
 250 }
 251
 252 static void (* r4k_blast_icache_page)(unsigned long addr);
 253
 254 static void r4k_blast_icache_page_setup(void)
 255 {
 256         unsigned long ic_lsize = cpu_icache_line_size();
 257
 258         if (ic_lsize == 0)
 259                 r4k_blast_icache_page = (void *)cache_noop;
 260         else if (ic_lsize == 16)
 261                 r4k_blast_icache_page = blast_icache16_page;
 262         else if (ic_lsize == 32 && current_cpu_type() == CPU_LOONGSON2)
 263                 r4k_blast_icache_page = loongson2_blast_icache32_page;
 264         else if (ic_lsize == 32)
 265                 r4k_blast_icache_page = blast_icache32_page;
 266         else if (ic_lsize == 64)
 267                 r4k_blast_icache_page = blast_icache64_page;
 268 }
 269
 270 #ifndef CONFIG_EVA
 271 #define r4k_blast_icache_user_page  r4k_blast_icache_page
 272 #else
 273
 274 static void (*r4k_blast_icache_user_page)(unsigned long addr);
 275
 276 static void __cpuinit r4k_blast_icache_user_page_setup(void)
 277 {
 278         unsigned long ic_lsize = cpu_icache_line_size();
 279
 280         if (ic_lsize == 0)
 281                 r4k_blast_icache_user_page = (void *)cache_noop;
 282         else if (ic_lsize == 16)
 283                 r4k_blast_icache_user_page = blast_icache16_user_page;
 284         else if (ic_lsize == 32)
 285                 r4k_blast_icache_user_page = blast_icache32_user_page;
 286         else if (ic_lsize == 64)
 287                 r4k_blast_icache_user_page = blast_icache64_user_page;
 288 }
 289
 290 #endif
 291
 292 static void (* r4k_blast_icache_page_indexed)(unsigned long addr);
 293
 294 static void r4k_blast_icache_page_indexed_setup(void)
 295 {
 296         unsigned long ic_lsize = cpu_icache_line_size();
 297
 298         if (ic_lsize == 0)
 299                 r4k_blast_icache_page_indexed = (void *)cache_noop;
 300         else if (ic_lsize == 16)
 301                 r4k_blast_icache_page_indexed = blast_icache16_page_indexed;
 302         else if (ic_lsize == 32) {
 303                 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
 304                         r4k_blast_icache_page_indexed =
 305                                 blast_icache32_r4600_v1_page_indexed;
 306                 else if (TX49XX_ICACHE_INDEX_INV_WAR)
 307                         r4k_blast_icache_page_indexed =
 308                                 tx49_blast_icache32_page_indexed;
 309                 else if (current_cpu_type() == CPU_LOONGSON2)
 310                         r4k_blast_icache_page_indexed =
 311                                 loongson2_blast_icache32_page_indexed;
 312                 else
 313                         r4k_blast_icache_page_indexed =
 314                                 blast_icache32_page_indexed;
 315         } else if (ic_lsize == 64)
 316                 r4k_blast_icache_page_indexed = blast_icache64_page_indexed;
 317 }
 318
 319 void (* r4k_blast_icache)(void);
 320 EXPORT_SYMBOL(r4k_blast_icache);
 321
 322 static void r4k_blast_icache_setup(void)
 323 {
 324         unsigned long ic_lsize = cpu_icache_line_size();
 325
 326         if (ic_lsize == 0)
 327                 r4k_blast_icache = (void *)cache_noop;
 328         else if (ic_lsize == 16)
 329                 r4k_blast_icache = blast_icache16;
 330         else if (ic_lsize == 32) {
 331                 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
 332                         r4k_blast_icache = blast_r4600_v1_icache32;
 333                 else if (TX49XX_ICACHE_INDEX_INV_WAR)
 334                         r4k_blast_icache = tx49_blast_icache32;
 335                 else if (current_cpu_type() == CPU_LOONGSON2)
 336                         r4k_blast_icache = loongson2_blast_icache32;
 337                 else
 338                         r4k_blast_icache = blast_icache32;
 339         } else if (ic_lsize == 64)
 340                 r4k_blast_icache = blast_icache64;
 341 }
 342
 343 static void (* r4k_blast_scache_page)(unsigned long addr);
 344
 345 static void r4k_blast_scache_page_setup(void)
 346 {
 347         unsigned long sc_lsize = cpu_scache_line_size();
 348
 349         if (scache_size == 0)
 350                 r4k_blast_scache_page = (void *)cache_noop;
 351         else if (sc_lsize == 16)
 352                 r4k_blast_scache_page = blast_scache16_page;
 353         else if (sc_lsize == 32)
 354                 r4k_blast_scache_page = blast_scache32_page;
 355         else if (sc_lsize == 64)
 356                 r4k_blast_scache_page = blast_scache64_page;
 357         else if (sc_lsize == 128)
 358                 r4k_blast_scache_page = blast_scache128_page;
 359 }
 360
 361 static void (* r4k_blast_scache_page_indexed)(unsigned long addr);
 362
 363 static void r4k_blast_scache_page_indexed_setup(void)
 364 {
 365         unsigned long sc_lsize = cpu_scache_line_size();
 366
 367         if (scache_size == 0)
 368                 r4k_blast_scache_page_indexed = (void *)cache_noop;
 369         else if (sc_lsize == 16)
 370                 r4k_blast_scache_page_indexed = blast_scache16_page_indexed;
 371         else if (sc_lsize == 32)
 372                 r4k_blast_scache_page_indexed = blast_scache32_page_indexed;
 373         else if (sc_lsize == 64)
 374                 r4k_blast_scache_page_indexed = blast_scache64_page_indexed;
 375         else if (sc_lsize == 128)
 376                 r4k_blast_scache_page_indexed = blast_scache128_page_indexed;
 377 }
 378
 379 static void (* r4k_blast_scache)(void);
 380
 381 static void r4k_blast_scache_setup(void)
 382 {
 383         unsigned long sc_lsize = cpu_scache_line_size();
 384
 385         if (scache_size == 0)
 386                 r4k_blast_scache = (void *)cache_noop;
 387         else if (sc_lsize == 16)
 388                 r4k_blast_scache = blast_scache16;
 389         else if (sc_lsize == 32)
 390                 r4k_blast_scache = blast_scache32;
 391         else if (sc_lsize == 64)
 392                 r4k_blast_scache = blast_scache64;
 393         else if (sc_lsize == 128)
 394                 r4k_blast_scache = blast_scache128;
 395 }
 396
 397 static inline void local_r4k___flush_cache_all(void * args)
 398 {
 399         switch (current_cpu_type()) {
 400         case CPU_LOONGSON2:
 401         case CPU_LOONGSON3:
 402         case CPU_R4000SC:
 403         case CPU_R4000MC:
 404         case CPU_R4400SC:
 405         case CPU_R4400MC:
 406         case CPU_R10000:
 407         case CPU_R12000:
 408         case CPU_R14000:
 409                 /*
 410                  * These caches are inclusive caches, that is, if something
 411                  * is not cached in the S-cache, we know it also won't be
 412                  * in one of the primary caches.
 413                  */
 414                 r4k_blast_scache();
 415                 break;
 416
 417         default:
 418                 r4k_blast_dcache();
 419                 r4k_blast_icache();
 420                 break;
 421         }
 422 }
 423
 424 static void r4k___flush_cache_all(void)
 425 {
 426         r4k_on_each_cpu(local_r4k___flush_cache_all, NULL);
 427 }
 428
 429 static inline int has_valid_asid(const struct mm_struct *mm)
 430 {
 431 #ifdef CONFIG_MIPS_MT_SMP
 432         int i;
 433
 434         for_each_online_cpu(i)
 435                 if (cpu_context(i, mm))
 436                         return 1;
 437
 438         return 0;
 439 #else
 440         return cpu_context(smp_processor_id(), mm);
 441 #endif
 442 }
 443
 444 static void r4k__flush_cache_vmap(void)
 445 {
 446         r4k_blast_dcache();
 447 }
 448
 449 static void r4k__flush_cache_vunmap(void)
 450 {
 451         r4k_blast_dcache();
 452 }
 453
 454 static inline void local_r4k_flush_cache_range(void * args)
 455 {
 456         struct vm_area_struct *vma = args;
 457         int exec = vma->vm_flags & VM_EXEC;
 458
 459         if (!(has_valid_asid(vma->vm_mm)))
 460                 return;
 461
 462         r4k_blast_dcache();
 463         if (exec)
 464                 r4k_blast_icache();
 465 }
 466
 467 static void r4k_flush_cache_range(struct vm_area_struct *vma,
 468         unsigned long start, unsigned long end)
 469 {
 470         int exec = vma->vm_flags & VM_EXEC;
 471
 472         if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc))
 473                 r4k_on_each_cpu(local_r4k_flush_cache_range, vma);
 474 }
 475
 476 static inline void local_r4k_flush_cache_mm(void * args)
 477 {
 478         struct mm_struct *mm = args;
 479
 480         if (!has_valid_asid(mm))
 481                 return;
 482
 483         /*
 484          * Kludge alert.  For obscure reasons R4000SC and R4400SC go nuts if we
 485          * only flush the primary caches but R10000 and R12000 behave sane ...
 486          * R4000SC and R4400SC indexed S-cache ops also invalidate primary
 487          * caches, so we can bail out early.
 488          */
 489         if (current_cpu_type() == CPU_R4000SC ||
 490             current_cpu_type() == CPU_R4000MC ||
 491             current_cpu_type() == CPU_R4400SC ||
 492             current_cpu_type() == CPU_R4400MC) {
 493                 r4k_blast_scache();
 494                 return;
 495         }
 496
 497         r4k_blast_dcache();
 498 }
 499
 500 static void r4k_flush_cache_mm(struct mm_struct *mm)
 501 {
 502         if (!cpu_has_dc_aliases)
 503                 return;
 504
 505         r4k_on_each_cpu(local_r4k_flush_cache_mm, mm);
 506 }
 507
 508 struct flush_cache_page_args {
 509         struct vm_area_struct *vma;
 510         unsigned long addr;
 511         unsigned long pfn;
 512 };
 513
 514 static inline void local_r4k_flush_cache_page(void *args)
 515 {
 516         struct flush_cache_page_args *fcp_args = args;
 517         struct vm_area_struct *vma = fcp_args->vma;
 518         unsigned long addr = fcp_args->addr;
 519         struct page *page = pfn_to_page(fcp_args->pfn);
 520         int exec = vma->vm_flags & VM_EXEC;
 521         struct mm_struct *mm = vma->vm_mm;
 522         int map_coherent = 0;
 523         pgd_t *pgdp;
 524         pud_t *pudp;
 525         pmd_t *pmdp;
 526         pte_t *ptep;
 527         void *vaddr;
 528
 529         /*
 530          * If ownes no valid ASID yet, cannot possibly have gotten
 531          * this page into the cache.
 532          */
 533         if (!has_valid_asid(mm))
 534                 return;
 535
 536         addr &= PAGE_MASK;
 537         pgdp = pgd_offset(mm, addr);
 538         pudp = pud_offset(pgdp, addr);
 539         pmdp = pmd_offset(pudp, addr);
 540         ptep = pte_offset(pmdp, addr);
 541
 542         /*
 543          * If the page isn't marked valid, the page cannot possibly be
 544          * in the cache.
 545          */
 546         if (!(pte_present(*ptep)))
 547                 return;
 548
 549         if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID))
 550                 vaddr = NULL;
 551         else {
 552                 /*
 553                  * Use kmap_coherent or kmap_atomic to do flushes for
 554                  * another ASID than the current one.
 555                  */
 556                 map_coherent = (cpu_has_dc_aliases &&
 557                                 page_mapped(page) && !Page_dcache_dirty(page));
 558                 if (map_coherent)
 559                         vaddr = kmap_coherent(page, addr);
 560                 else
 561                         vaddr = kmap_atomic(page);
 562                 addr = (unsigned long)vaddr;
 563         }
 564
 565         if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) {
 566                 vaddr ? r4k_blast_dcache_page(addr) :
 567                         r4k_blast_dcache_user_page(addr);
 568                 if (exec && !cpu_icache_snoops_remote_store)
 569                         r4k_blast_scache_page(addr);
 570         }
 571         if (exec) {
 572                 if (vaddr && cpu_has_vtag_icache && mm == current->active_mm) {
 573                         int cpu = smp_processor_id();
 574
 575                         if (cpu_context(cpu, mm) != 0)
 576                                 drop_mmu_context(mm, cpu);
 577                 } else
 578                         vaddr ? r4k_blast_icache_page(addr) :
 579                                 r4k_blast_icache_user_page(addr);
 580         }
 581
 582         if (vaddr) {
 583                 if (map_coherent)
 584                         kunmap_coherent();
 585                 else
 586                         kunmap_atomic(vaddr);
 587         }
 588 }
 589
 590 static void r4k_flush_cache_page(struct vm_area_struct *vma,
 591         unsigned long addr, unsigned long pfn)
 592 {
 593         struct flush_cache_page_args args;
 594
 595         args.vma = vma;
 596         args.addr = addr;
 597         args.pfn = pfn;
 598
 599         r4k_on_each_cpu(local_r4k_flush_cache_page, &args);
 600 }
 601
 602 static inline void local_r4k_flush_data_cache_page(void * addr)
 603 {
 604         r4k_blast_dcache_page((unsigned long) addr);
 605 }
 606
 607 static void r4k_flush_data_cache_page(unsigned long addr)
 608 {
 609         if (in_atomic())
 610                 local_r4k_flush_data_cache_page((void *)addr);
 611         else
 612                 r4k_on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr);
 613 }
 614
 615 struct flush_icache_range_args {
 616         unsigned long start;
 617         unsigned long end;
 618 };
 619
 620 static inline void local_r4k_flush_icache_range(unsigned long start, unsigned long end)
 621 {
 622         if (!cpu_has_ic_fills_f_dc) {
 623                 if (end - start >= dcache_size) {
 624                         r4k_blast_dcache();
 625                 } else {
 626                         R4600_HIT_CACHEOP_WAR_IMPL;
 627                         protected_blast_dcache_range(start, end);
 628                 }
 629         }
 630
 631         if (end - start > icache_size)
 632                 r4k_blast_icache();
 633         else {
 634                 switch (boot_cpu_type()) {
 635                 case CPU_LOONGSON2:
 636                         protected_loongson2_blast_icache_range(start, end);
 637                         break;
 638
 639                 default:
 640                         protected_blast_icache_range(start, end);
 641                         break;
 642                 }
 643         }
 644 #ifdef CONFIG_EVA
 645         /*
 646          * Due to all possible segment mappings, there might cache aliases
 647          * caused by the bootloader being in non-EVA mode, and the CPU switching
 648          * to EVA during early kernel init. It's best to flush the scache
 649          * to avoid having secondary cores fetching stale data and lead to
 650          * kernel crashes.
 651          */
 652         bc_wback_inv(start, (end - start));
 653         __sync();
 654 #endif
 655 }
 656
 657 static inline void local_r4k_flush_icache_range_ipi(void *args)
 658 {
 659         struct flush_icache_range_args *fir_args = args;
 660         unsigned long start = fir_args->start;
 661         unsigned long end = fir_args->end;
 662
 663         local_r4k_flush_icache_range(start, end);
 664 }
 665
 666 static void r4k_flush_icache_range(unsigned long start, unsigned long end)
 667 {
 668         struct flush_icache_range_args args;
 669
 670         args.start = start;
 671         args.end = end;
 672
 673         r4k_on_each_cpu(local_r4k_flush_icache_range_ipi, &args);
 674         instruction_hazard();
 675 }
 676
 677 #if defined(CONFIG_DMA_NONCOHERENT) || defined(CONFIG_DMA_MAYBE_COHERENT)
 678
 679 static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size)
 680 {
 681         /* Catch bad driver code */
 682         BUG_ON(size == 0);
 683
 684         preempt_disable();
 685         if (cpu_has_inclusive_pcaches) {
 686                 if (size >= scache_size)
 687                         r4k_blast_scache();
 688                 else
 689                         blast_scache_range(addr, addr + size);
 690                 preempt_enable();
 691                 __sync();
 692                 return;
 693         }
 694
 695         /*
 696          * Either no secondary cache or the available caches don't have the
 697          * subset property so we have to flush the primary caches
 698          * explicitly
 699          */
 700         if (cpu_has_safe_index_cacheops && size >= dcache_size) {
 701                 r4k_blast_dcache();
 702         } else {
 703                 R4600_HIT_CACHEOP_WAR_IMPL;
 704                 blast_dcache_range(addr, addr + size);
 705         }
 706         preempt_enable();
 707
 708         bc_wback_inv(addr, size);
 709         __sync();
 710 }
 711
 712 static void r4k_dma_cache_inv(unsigned long addr, unsigned long size)
 713 {
 714         /* Catch bad driver code */
 715         BUG_ON(size == 0);
 716
 717         preempt_disable();
 718         if (cpu_has_inclusive_pcaches) {
 719                 if (size >= scache_size)
 720                         r4k_blast_scache();
 721                 else {
 722                         /*
 723                          * There is no clearly documented alignment requirement
 724                          * for the cache instruction on MIPS processors and
 725                          * some processors, among them the RM5200 and RM7000
 726                          * QED processors will throw an address error for cache
 727                          * hit ops with insufficient alignment.  Solved by
 728                          * aligning the address to cache line size.
 729                          */
 730                         blast_inv_scache_range(addr, addr + size);
 731                 }
 732                 preempt_enable();
 733                 __sync();
 734                 return;
 735         }
 736
 737         if (cpu_has_safe_index_cacheops && size >= dcache_size) {
 738                 r4k_blast_dcache();
 739         } else {
 740                 R4600_HIT_CACHEOP_WAR_IMPL;
 741                 blast_inv_dcache_range(addr, addr + size);
 742         }
 743         preempt_enable();
 744
 745         bc_inv(addr, size);
 746         __sync();
 747 }
 748 #endif /* CONFIG_DMA_NONCOHERENT || CONFIG_DMA_MAYBE_COHERENT */
 749
 750 /*
 751  * While we're protected against bad userland addresses we don't care
 752  * very much about what happens in that case.  Usually a segmentation
 753  * fault will dump the process later on anyway ...
 754  */
 755 static void local_r4k_flush_cache_sigtramp(void * arg)
 756 {
 757         unsigned long ic_lsize = cpu_icache_line_size();
 758         unsigned long dc_lsize = cpu_dcache_line_size();
 759         unsigned long sc_lsize = cpu_scache_line_size();
 760         unsigned long addr = (unsigned long) arg;
 761
 762         R4600_HIT_CACHEOP_WAR_IMPL;
 763         if (dc_lsize)
 764                 protected_writeback_dcache_line(addr & ~(dc_lsize - 1));
 765         if (!cpu_icache_snoops_remote_store && scache_size)
 766                 protected_writeback_scache_line(addr & ~(sc_lsize - 1));
 767         if (ic_lsize)
 768                 protected_flush_icache_line(addr & ~(ic_lsize - 1));
 769         if (MIPS4K_ICACHE_REFILL_WAR) {
 770                 __asm__ __volatile__ (
 771                         ".set push\n\t"
 772                         ".set noat\n\t"
 773                         ".set mips3\n\t"
 774 #ifdef CONFIG_32BIT
 775                         "la     $at,1f\n\t"
 776 #endif
 777 #ifdef CONFIG_64BIT
 778                         "dla    $at,1f\n\t"
 779 #endif
 780                         "cache  %0,($at)\n\t"
 781                         "nop; nop; nop\n"
 782                         "1:\n\t"
 783                         ".set pop"
 784                         :
 785                         : "i" (Hit_Invalidate_I));
 786         }
 787         if (MIPS_CACHE_SYNC_WAR)
 788                 __asm__ __volatile__ ("sync");
 789 }
 790
 791 static void r4k_flush_cache_sigtramp(unsigned long addr)
 792 {
 793         r4k_on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr);
 794 }
 795
 796 static void r4k_flush_icache_all(void)
 797 {
 798         if (cpu_has_vtag_icache)
 799                 r4k_blast_icache();
 800 }
 801
 802 struct flush_kernel_vmap_range_args {
 803         unsigned long   vaddr;
 804         int             size;
 805 };
 806
 807 static inline void local_r4k_flush_kernel_vmap_range(void *args)
 808 {
 809         struct flush_kernel_vmap_range_args *vmra = args;
 810         unsigned long vaddr = vmra->vaddr;
 811         int size = vmra->size;
 812
 813         /*
 814          * Aliases only affect the primary caches so don't bother with
 815          * S-caches or T-caches.
 816          */
 817         if (cpu_has_safe_index_cacheops && size >= dcache_size)
 818                 r4k_blast_dcache();
 819         else {
 820                 R4600_HIT_CACHEOP_WAR_IMPL;
 821                 blast_dcache_range(vaddr, vaddr + size);
 822         }
 823 }
 824
 825 static void r4k_flush_kernel_vmap_range(unsigned long vaddr, int size)
 826 {
 827         struct flush_kernel_vmap_range_args args;
 828
 829         args.vaddr = (unsigned long) vaddr;
 830         args.size = size;
 831
 832         r4k_on_each_cpu(local_r4k_flush_kernel_vmap_range, &args);
 833 }
 834
 835 static inline void rm7k_erratum31(void)
 836 {
 837         const unsigned long ic_lsize = 32;
 838         unsigned long addr;
 839
 840         /* RM7000 erratum #31. The icache is screwed at startup. */
 841         write_c0_taglo(0);
 842         write_c0_taghi(0);
 843
 844         for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) {
 845                 __asm__ __volatile__ (
 846                         ".set push\n\t"
 847                         ".set noreorder\n\t"
 848                         ".set mips3\n\t"
 849                         "cache\t%1, 0(%0)\n\t"
 850                         "cache\t%1, 0x1000(%0)\n\t"
 851                         "cache\t%1, 0x2000(%0)\n\t"
 852                         "cache\t%1, 0x3000(%0)\n\t"
 853                         "cache\t%2, 0(%0)\n\t"
 854                         "cache\t%2, 0x1000(%0)\n\t"
 855                         "cache\t%2, 0x2000(%0)\n\t"
 856                         "cache\t%2, 0x3000(%0)\n\t"
 857                         "cache\t%1, 0(%0)\n\t"
 858                         "cache\t%1, 0x1000(%0)\n\t"
 859                         "cache\t%1, 0x2000(%0)\n\t"
 860                         "cache\t%1, 0x3000(%0)\n\t"
 861                         ".set pop\n"
 862                         :
 863                         : "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill));
 864         }
 865 }
 866
 867 static inline void alias_74k_erratum(struct cpuinfo_mips *c)
 868 {
 869         unsigned int imp = c->processor_id & PRID_IMP_MASK;
 870         unsigned int rev = c->processor_id & PRID_REV_MASK;
 871
 872         /*
 873          * Early versions of the 74K do not update the cache tags on a
 874          * vtag miss/ptag hit which can occur in the case of KSEG0/KUSEG
 875          * aliases. In this case it is better to treat the cache as always
 876          * having aliases.
 877          */
 878         switch (imp) {
 879         case PRID_IMP_74K:
 880                 if (rev <= PRID_REV_ENCODE_332(2, 4, 0))
 881                         c->dcache.flags |= MIPS_CACHE_VTAG;
 882                 if (rev == PRID_REV_ENCODE_332(2, 4, 0))
 883                         write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND);
 884                 break;
 885         case PRID_IMP_1074K:
 886                 if (rev <= PRID_REV_ENCODE_332(1, 1, 0)) {
 887                         c->dcache.flags |= MIPS_CACHE_VTAG;
 888                         write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND);
 889                 }
 890                 break;
 891         default:
 892                 BUG();
 893         }
 894 }
 895
 896 static char *way_string[] = { NULL, "direct mapped", "2-way",
 897         "3-way", "4-way", "5-way", "6-way", "7-way", "8-way"
 898 };
 899
 900 static void probe_pcache(void)
 901 {
 902         struct cpuinfo_mips *c = &current_cpu_data;
 903         unsigned int config = read_c0_config();
 904         unsigned int prid = read_c0_prid();
 905         unsigned long config1;
 906         unsigned int lsize;
 907
 908         switch (current_cpu_type()) {
 909         case CPU_R4600:                 /* QED style two way caches? */
 910         case CPU_R4700:
 911         case CPU_R5000:
 912         case CPU_NEVADA:
 913                 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
 914                 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
 915                 c->icache.ways = 2;
 916                 c->icache.waybit = __ffs(icache_size/2);
 917
 918                 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
 919                 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
 920                 c->dcache.ways = 2;
 921                 c->dcache.waybit= __ffs(dcache_size/2);
 922
 923                 c->options |= MIPS_CPU_CACHE_CDEX_P;
 924                 break;
 925
 926         case CPU_R5432:
 927         case CPU_R5500:
 928                 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
 929                 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
 930                 c->icache.ways = 2;
 931                 c->icache.waybit= 0;
 932
 933                 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
 934                 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
 935                 c->dcache.ways = 2;
 936                 c->dcache.waybit = 0;
 937
 938                 c->options |= MIPS_CPU_CACHE_CDEX_P | MIPS_CPU_PREFETCH;
 939                 break;
 940
 941         case CPU_TX49XX:
 942                 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
 943                 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
 944                 c->icache.ways = 4;
 945                 c->icache.waybit= 0;
 946
 947                 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
 948                 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
 949                 c->dcache.ways = 4;
 950                 c->dcache.waybit = 0;
 951
 952                 c->options |= MIPS_CPU_CACHE_CDEX_P;
 953                 c->options |= MIPS_CPU_PREFETCH;
 954                 break;
 955
 956         case CPU_R4000PC:
 957         case CPU_R4000SC:
 958         case CPU_R4000MC:
 959         case CPU_R4400PC:
 960         case CPU_R4400SC:
 961         case CPU_R4400MC:
 962         case CPU_R4300:
 963                 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
 964                 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
 965                 c->icache.ways = 1;
 966                 c->icache.waybit = 0;   /* doesn't matter */
 967
 968                 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
 969                 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
 970                 c->dcache.ways = 1;
 971                 c->dcache.waybit = 0;   /* does not matter */
 972
 973                 c->options |= MIPS_CPU_CACHE_CDEX_P;
 974                 break;
 975
 976         case CPU_R10000:
 977         case CPU_R12000:
 978         case CPU_R14000:
 979                 icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29));
 980                 c->icache.linesz = 64;
 981                 c->icache.ways = 2;
 982                 c->icache.waybit = 0;
 983
 984                 dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26));
 985                 c->dcache.linesz = 32;
 986                 c->dcache.ways = 2;
 987                 c->dcache.waybit = 0;
 988
 989                 c->options |= MIPS_CPU_PREFETCH;
 990                 break;
 991
 992         case CPU_VR4133:
 993                 write_c0_config(config & ~VR41_CONF_P4K);
 994         case CPU_VR4131:
 995                 /* Workaround for cache instruction bug of VR4131 */
 996                 if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U ||
 997                     c->processor_id == 0x0c82U) {
 998                         config |= 0x00400000U;
 999                         if (c->processor_id == 0x0c80U)
1000                                 config |= VR41_CONF_BP;
1001                         write_c0_config(config);
1002                 } else
1003                         c->options |= MIPS_CPU_CACHE_CDEX_P;
1004
1005                 icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
1006                 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1007                 c->icache.ways = 2;
1008                 c->icache.waybit = __ffs(icache_size/2);
1009
1010                 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
1011                 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1012                 c->dcache.ways = 2;
1013                 c->dcache.waybit = __ffs(dcache_size/2);
1014                 break;
1015
1016         case CPU_VR41XX:
1017         case CPU_VR4111:
1018         case CPU_VR4121:
1019         case CPU_VR4122:
1020         case CPU_VR4181:
1021         case CPU_VR4181A:
1022                 icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
1023                 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1024                 c->icache.ways = 1;
1025                 c->icache.waybit = 0;   /* doesn't matter */
1026
1027                 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
1028                 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1029                 c->dcache.ways = 1;
1030                 c->dcache.waybit = 0;   /* does not matter */
1031
1032                 c->options |= MIPS_CPU_CACHE_CDEX_P;
1033                 break;
1034
1035         case CPU_RM7000:
1036                 rm7k_erratum31();
1037
1038                 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
1039                 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1040                 c->icache.ways = 4;
1041                 c->icache.waybit = __ffs(icache_size / c->icache.ways);
1042
1043                 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
1044                 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1045                 c->dcache.ways = 4;
1046                 c->dcache.waybit = __ffs(dcache_size / c->dcache.ways);
1047
1048                 c->options |= MIPS_CPU_CACHE_CDEX_P;
1049                 c->options |= MIPS_CPU_PREFETCH;
1050                 break;
1051
1052         case CPU_LOONGSON2:
1053                 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
1054                 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1055                 if (prid & 0x3)
1056                         c->icache.ways = 4;
1057                 else
1058                         c->icache.ways = 2;
1059                 c->icache.waybit = 0;
1060
1061                 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
1062                 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1063                 if (prid & 0x3)
1064                         c->dcache.ways = 4;
1065                 else
1066                         c->dcache.ways = 2;
1067                 c->dcache.waybit = 0;
1068                 break;
1069
1070         case CPU_LOONGSON3:
1071                 config1 = read_c0_config1();
1072                 lsize = (config1 >> 19) & 7;
1073                 if (lsize)
1074                         c->icache.linesz = 2 << lsize;
1075                 else
1076                         c->icache.linesz = 0;
1077                 c->icache.sets = 64 << ((config1 >> 22) & 7);
1078                 c->icache.ways = 1 + ((config1 >> 16) & 7);
1079                 icache_size = c->icache.sets *
1080                                           c->icache.ways *
1081                                           c->icache.linesz;
1082                 c->icache.waybit = 0;
1083
1084                 lsize = (config1 >> 10) & 7;
1085                 if (lsize)
1086                         c->dcache.linesz = 2 << lsize;
1087                 else
1088                         c->dcache.linesz = 0;
1089                 c->dcache.sets = 64 << ((config1 >> 13) & 7);
1090                 c->dcache.ways = 1 + ((config1 >> 7) & 7);
1091                 dcache_size = c->dcache.sets *
1092                                           c->dcache.ways *
1093                                           c->dcache.linesz;
1094                 c->dcache.waybit = 0;
1095                 break;
1096
1097         default:
1098                 if (!(config & MIPS_CONF_M))
1099                         panic("Don't know how to probe P-caches on this cpu.");
1100
1101                 /*
1102                  * So we seem to be a MIPS32 or MIPS64 CPU
1103                  * So let's probe the I-cache ...
1104                  */
1105                 config1 = read_c0_config1();
1106
1107                 lsize = (config1 >> 19) & 7;
1108
1109                 /* IL == 7 is reserved */
1110                 if (lsize == 7)
1111                         panic("Invalid icache line size");
1112
1113                 c->icache.linesz = lsize ? 2 << lsize : 0;
1114
1115                 c->icache.sets = 32 << (((config1 >> 22) + 1) & 7);
1116                 c->icache.ways = 1 + ((config1 >> 16) & 7);
1117
1118                 icache_size = c->icache.sets *
1119                               c->icache.ways *
1120                               c->icache.linesz;
1121                 c->icache.waybit = __ffs(icache_size/c->icache.ways);
1122
1123                 if (config & 0x8)               /* VI bit */
1124                         c->icache.flags |= MIPS_CACHE_VTAG;
1125
1126                 /*
1127                  * Now probe the MIPS32 / MIPS64 data cache.
1128                  */
1129                 c->dcache.flags = 0;
1130
1131                 lsize = (config1 >> 10) & 7;
1132
1133                 /* DL == 7 is reserved */
1134                 if (lsize == 7)
1135                         panic("Invalid dcache line size");
1136
1137                 c->dcache.linesz = lsize ? 2 << lsize : 0;
1138
1139                 c->dcache.sets = 32 << (((config1 >> 13) + 1) & 7);
1140                 c->dcache.ways = 1 + ((config1 >> 7) & 7);
1141
1142                 dcache_size = c->dcache.sets *
1143                               c->dcache.ways *
1144                               c->dcache.linesz;
1145                 c->dcache.waybit = __ffs(dcache_size/c->dcache.ways);
1146
1147                 c->options |= MIPS_CPU_PREFETCH;
1148                 break;
1149         }
1150
1151         /*
1152          * Processor configuration sanity check for the R4000SC erratum
1153          * #5.  With page sizes larger than 32kB there is no possibility
1154          * to get a VCE exception anymore so we don't care about this
1155          * misconfiguration.  The case is rather theoretical anyway;
1156          * presumably no vendor is shipping his hardware in the "bad"
1157          * configuration.
1158          */
1159         if ((prid & PRID_IMP_MASK) == PRID_IMP_R4000 &&
1160             (prid & PRID_REV_MASK) < PRID_REV_R4400 &&
1161             !(config & CONF_SC) && c->icache.linesz != 16 &&
1162             PAGE_SIZE <= 0x8000)
1163                 panic("Improper R4000SC processor configuration detected");
1164
1165         /* compute a couple of other cache variables */
1166         c->icache.waysize = icache_size / c->icache.ways;
1167         c->dcache.waysize = dcache_size / c->dcache.ways;
1168
1169         c->icache.sets = c->icache.linesz ?
1170                 icache_size / (c->icache.linesz * c->icache.ways) : 0;
1171         c->dcache.sets = c->dcache.linesz ?
1172                 dcache_size / (c->dcache.linesz * c->dcache.ways) : 0;
1173
1174         /*
1175          * R10000 and R12000 P-caches are odd in a positive way.  They're 32kB
1176          * 2-way virtually indexed so normally would suffer from aliases.  So
1177          * normally they'd suffer from aliases but magic in the hardware deals
1178          * with that for us so we don't need to take care ourselves.
1179          */
1180         switch (current_cpu_type()) {
1181         case CPU_20KC:
1182         case CPU_25KF:
1183         case CPU_SB1:
1184         case CPU_SB1A:
1185         case CPU_XLR:
1186                 c->dcache.flags |= MIPS_CACHE_PINDEX;
1187                 break;
1188
1189         case CPU_R10000:
1190         case CPU_R12000:
1191         case CPU_R14000:
1192                 break;
1193
1194         case CPU_M14KC:
1195         case CPU_M14KEC:
1196         case CPU_24K:
1197         case CPU_34K:
1198         case CPU_74K:
1199         case CPU_1004K:
1200         case CPU_1074K:
1201         case CPU_INTERAPTIV:
1202         case CPU_P5600:
1203         case CPU_PROAPTIV:
1204         case CPU_M5150:
1205                 if ((c->cputype == CPU_74K) || (c->cputype == CPU_1074K))
1206                         alias_74k_erratum(c);
1207                 if (!(read_c0_config7() & MIPS_CONF7_IAR) &&
1208                     (c->icache.waysize > PAGE_SIZE))
1209                         c->icache.flags |= MIPS_CACHE_ALIASES;
1210                 if (read_c0_config7() & MIPS_CONF7_AR) {
1211                         /*
1212                          * Effectively physically indexed dcache,
1213                          * thus no virtual aliases.
1214                         */
1215                         c->dcache.flags |= MIPS_CACHE_PINDEX;
1216                         break;
1217                 }
1218         default:
1219                 if (c->dcache.waysize > PAGE_SIZE)
1220                         c->dcache.flags |= MIPS_CACHE_ALIASES;
1221         }
1222
1223         switch (current_cpu_type()) {
1224         case CPU_20KC:
1225                 /*
1226                  * Some older 20Kc chips doesn't have the 'VI' bit in
1227                  * the config register.
1228                  */
1229                 c->icache.flags |= MIPS_CACHE_VTAG;
1230                 break;
1231
1232         case CPU_ALCHEMY:
1233                 c->icache.flags |= MIPS_CACHE_IC_F_DC;
1234                 break;
1235
1236         case CPU_LOONGSON2:
1237                 /*
1238                  * LOONGSON2 has 4 way icache, but when using indexed cache op,
1239                  * one op will act on all 4 ways
1240                  */
1241                 c->icache.ways = 1;
1242         }
1243
1244         printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n",
1245                icache_size >> 10,
1246                c->icache.flags & MIPS_CACHE_VTAG ? "VIVT" : "VIPT",
1247                way_string[c->icache.ways], c->icache.linesz);
1248
1249         printk("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n",
1250                dcache_size >> 10, way_string[c->dcache.ways],
1251                (c->dcache.flags & MIPS_CACHE_PINDEX) ? "PIPT" : "VIPT",
1252                (c->dcache.flags & MIPS_CACHE_ALIASES) ?
1253                         "cache aliases" : "no aliases",
1254                c->dcache.linesz);
1255 }
1256
1257 /*
1258  * If you even _breathe_ on this function, look at the gcc output and make sure
1259  * it does not pop things on and off the stack for the cache sizing loop that
1260  * executes in KSEG1 space or else you will crash and burn badly.  You have
1261  * been warned.
1262  */
1263 static int probe_scache(void)
1264 {
1265         unsigned long flags, addr, begin, end, pow2;
1266         unsigned int config = read_c0_config();
1267         struct cpuinfo_mips *c = &current_cpu_data;
1268
1269         if (config & CONF_SC)
1270                 return 0;
1271
1272         begin = (unsigned long) &_stext;
1273         begin &= ~((4 * 1024 * 1024) - 1);
1274         end = begin + (4 * 1024 * 1024);
1275
1276         /*
1277          * This is such a bitch, you'd think they would make it easy to do
1278          * this.  Away you daemons of stupidity!
1279          */
1280         local_irq_save(flags);
1281
1282         /* Fill each size-multiple cache line with a valid tag. */
1283         pow2 = (64 * 1024);
1284         for (addr = begin; addr < end; addr = (begin + pow2)) {
1285                 unsigned long *p = (unsigned long *) addr;
1286                 __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */
1287                 pow2 <<= 1;
1288         }
1289
1290         /* Load first line with zero (therefore invalid) tag. */
1291         write_c0_taglo(0);
1292         write_c0_taghi(0);
1293         __asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */
1294         cache_op(Index_Store_Tag_I, begin);
1295         cache_op(Index_Store_Tag_D, begin);
1296         cache_op(Index_Store_Tag_SD, begin);
1297
1298         /* Now search for the wrap around point. */
1299         pow2 = (128 * 1024);
1300         for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) {
1301                 cache_op(Index_Load_Tag_SD, addr);
1302                 __asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */
1303                 if (!read_c0_taglo())
1304                         break;
1305                 pow2 <<= 1;
1306         }
1307         local_irq_restore(flags);
1308         addr -= begin;
1309
1310         scache_size = addr;
1311         c->scache.linesz = 16 << ((config & R4K_CONF_SB) >> 22);
1312         c->scache.ways = 1;
1313         c->dcache.waybit = 0;           /* does not matter */
1314
1315         return 1;
1316 }
1317
1318 static void __init loongson2_sc_init(void)
1319 {
1320         struct cpuinfo_mips *c = &current_cpu_data;
1321
1322         scache_size = 512*1024;
1323         c->scache.linesz = 32;
1324         c->scache.ways = 4;
1325         c->scache.waybit = 0;
1326         c->scache.waysize = scache_size / (c->scache.ways);
1327         c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways);
1328         pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
1329                scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
1330
1331         c->options |= MIPS_CPU_INCLUSIVE_CACHES;
1332 }
1333
1334 static void __init loongson3_sc_init(void)
1335 {
1336         struct cpuinfo_mips *c = &current_cpu_data;
1337         unsigned int config2, lsize;
1338
1339         config2 = read_c0_config2();
1340         lsize = (config2 >> 4) & 15;
1341         if (lsize)
1342                 c->scache.linesz = 2 << lsize;
1343         else
1344                 c->scache.linesz = 0;
1345         c->scache.sets = 64 << ((config2 >> 8) & 15);
1346         c->scache.ways = 1 + (config2 & 15);
1347
1348         scache_size = c->scache.sets *
1349                                   c->scache.ways *
1350                                   c->scache.linesz;
1351         /* Loongson-3 has 4 cores, 1MB scache for each. scaches are shared */
1352         scache_size *= 4;
1353         c->scache.waybit = 0;
1354         pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
1355                scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
1356         if (scache_size)
1357                 c->options |= MIPS_CPU_INCLUSIVE_CACHES;
1358         return;
1359 }
1360
1361 extern int r5k_sc_init(void);
1362 extern int rm7k_sc_init(void);
1363 extern int mips_sc_init(void);
1364
1365 static void setup_scache(void)
1366 {
1367         struct cpuinfo_mips *c = &current_cpu_data;
1368         unsigned int config = read_c0_config();
1369         int sc_present = 0;
1370
1371         /*
1372          * Do the probing thing on R4000SC and R4400SC processors.  Other
1373          * processors don't have a S-cache that would be relevant to the
1374          * Linux memory management.
1375          */
1376         switch (current_cpu_type()) {
1377         case CPU_R4000SC:
1378         case CPU_R4000MC:
1379         case CPU_R4400SC:
1380         case CPU_R4400MC:
1381                 sc_present = run_uncached(probe_scache);
1382                 if (sc_present)
1383                         c->options |= MIPS_CPU_CACHE_CDEX_S;
1384                 break;
1385
1386         case CPU_R10000:
1387         case CPU_R12000:
1388         case CPU_R14000:
1389                 scache_size = 0x80000 << ((config & R10K_CONF_SS) >> 16);
1390                 c->scache.linesz = 64 << ((config >> 13) & 1);
1391                 c->scache.ways = 2;
1392                 c->scache.waybit= 0;
1393                 sc_present = 1;
1394                 break;
1395
1396         case CPU_R5000:
1397         case CPU_NEVADA:
1398 #ifdef CONFIG_R5000_CPU_SCACHE
1399                 r5k_sc_init();
1400 #endif
1401                 return;
1402
1403         case CPU_RM7000:
1404 #ifdef CONFIG_RM7000_CPU_SCACHE
1405                 rm7k_sc_init();
1406 #endif
1407                 return;
1408
1409         case CPU_LOONGSON2:
1410                 loongson2_sc_init();
1411                 return;
1412
1413         case CPU_LOONGSON3:
1414                 loongson3_sc_init();
1415                 return;
1416
1417         case CPU_XLP:
1418                 /* don't need to worry about L2, fully coherent */
1419                 return;
1420
1421         default:
1422                 if (c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M32R2 |
1423                                     MIPS_CPU_ISA_M64R1 | MIPS_CPU_ISA_M64R2)) {
1424 #ifdef CONFIG_MIPS_CPU_SCACHE
1425                         if (mips_sc_init ()) {
1426                                 scache_size = c->scache.ways * c->scache.sets * c->scache.linesz;
1427                                 printk("MIPS secondary cache %ldkB, %s, linesize %d bytes.\n",
1428                                        scache_size >> 10,
1429                                        way_string[c->scache.ways], c->scache.linesz);
1430                         }
1431 #else
1432                         if (!(c->scache.flags & MIPS_CACHE_NOT_PRESENT))
1433                                 panic("Dunno how to handle MIPS32 / MIPS64 second level cache");
1434 #endif
1435                         return;
1436                 }
1437                 sc_present = 0;
1438         }
1439
1440         if (!sc_present)
1441                 return;
1442
1443         /* compute a couple of other cache variables */
1444         c->scache.waysize = scache_size / c->scache.ways;
1445
1446         c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways);
1447
1448         printk("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
1449                scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
1450
1451         c->options |= MIPS_CPU_INCLUSIVE_CACHES;
1452 }
1453
1454 void au1x00_fixup_config_od(void)
1455 {
1456         /*
1457          * c0_config.od (bit 19) was write only (and read as 0)
1458          * on the early revisions of Alchemy SOCs.  It disables the bus
1459          * transaction overlapping and needs to be set to fix various errata.
1460          */
1461         switch (read_c0_prid()) {
1462         case 0x00030100: /* Au1000 DA */
1463         case 0x00030201: /* Au1000 HA */
1464         case 0x00030202: /* Au1000 HB */
1465         case 0x01030200: /* Au1500 AB */
1466         /*
1467          * Au1100 errata actually keeps silence about this bit, so we set it
1468          * just in case for those revisions that require it to be set according
1469          * to the (now gone) cpu table.
1470          */
1471         case 0x02030200: /* Au1100 AB */
1472         case 0x02030201: /* Au1100 BA */
1473         case 0x02030202: /* Au1100 BC */
1474                 set_c0_config(1 << 19);
1475                 break;
1476         }
1477 }
1478
1479 /* CP0 hazard avoidance. */
1480 #define NXP_BARRIER()                                                   \
1481          __asm__ __volatile__(                                          \
1482         ".set noreorder\n\t"                                            \
1483         "nop; nop; nop; nop; nop; nop;\n\t"                             \
1484         ".set reorder\n\t")
1485
1486 static void nxp_pr4450_fixup_config(void)
1487 {
1488         unsigned long config0;
1489
1490         config0 = read_c0_config();
1491
1492         /* clear all three cache coherency fields */
1493         config0 &= ~(0x7 | (7 << 25) | (7 << 28));
1494         config0 |= (((_page_cachable_default >> _CACHE_SHIFT) <<  0) |
1495                     ((_page_cachable_default >> _CACHE_SHIFT) << 25) |
1496                     ((_page_cachable_default >> _CACHE_SHIFT) << 28));
1497         write_c0_config(config0);
1498         NXP_BARRIER();
1499 }
1500
1501 static int cca = -1;
1502
1503 static int __init cca_setup(char *str)
1504 {
1505         get_option(&str, &cca);
1506
1507         return 0;
1508 }
1509
1510 early_param("cca", cca_setup);
1511
1512 static void coherency_setup(void)
1513 {
1514         if (cca < 0 || cca > 7)
1515                 cca = read_c0_config() & CONF_CM_CMASK;
1516         _page_cachable_default = cca << _CACHE_SHIFT;
1517
1518         pr_debug("Using cache attribute %d\n", cca);
1519         change_c0_config(CONF_CM_CMASK, cca);
1520
1521         /*
1522          * c0_status.cu=0 specifies that updates by the sc instruction use
1523          * the coherency mode specified by the TLB; 1 means cachable
1524          * coherent update on write will be used.  Not all processors have
1525          * this bit and; some wire it to zero, others like Toshiba had the
1526          * silly idea of putting something else there ...
1527          */
1528         switch (current_cpu_type()) {
1529         case CPU_R4000PC:
1530         case CPU_R4000SC:
1531         case CPU_R4000MC:
1532         case CPU_R4400PC:
1533         case CPU_R4400SC:
1534         case CPU_R4400MC:
1535                 clear_c0_config(CONF_CU);
1536                 break;
1537         /*
1538          * We need to catch the early Alchemy SOCs with
1539          * the write-only co_config.od bit and set it back to one on:
1540          * Au1000 rev DA, HA, HB;  Au1100 AB, BA, BC, Au1500 AB
1541          */
1542         case CPU_ALCHEMY:
1543                 au1x00_fixup_config_od();
1544                 break;
1545
1546         case PRID_IMP_PR4450:
1547                 nxp_pr4450_fixup_config();
1548                 break;
1549         }
1550 }
1551
1552 static void r4k_cache_error_setup(void)
1553 {
1554         extern char __weak except_vec2_generic;
1555         extern char __weak except_vec2_sb1;
1556
1557         switch (current_cpu_type()) {
1558         case CPU_SB1:
1559         case CPU_SB1A:
1560                 set_uncached_handler(0x100, &except_vec2_sb1, 0x80);
1561                 break;
1562
1563         default:
1564                 set_uncached_handler(0x100, &except_vec2_generic, 0x80);
1565                 break;
1566         }
1567 }
1568
1569 void r4k_cache_init(void)
1570 {
1571         extern void build_clear_page(void);
1572         extern void build_copy_page(void);
1573         struct cpuinfo_mips *c = &current_cpu_data;
1574
1575         probe_pcache();
1576         setup_scache();
1577
1578         r4k_blast_dcache_page_setup();
1579         r4k_blast_dcache_page_indexed_setup();
1580         r4k_blast_dcache_setup();
1581         r4k_blast_icache_page_setup();
1582         r4k_blast_icache_page_indexed_setup();
1583         r4k_blast_icache_setup();
1584         r4k_blast_scache_page_setup();
1585         r4k_blast_scache_page_indexed_setup();
1586         r4k_blast_scache_setup();
1587 #ifdef CONFIG_EVA
1588         r4k_blast_dcache_user_page_setup();
1589         r4k_blast_icache_user_page_setup();
1590 #endif
1591
1592         /*
1593          * Some MIPS32 and MIPS64 processors have physically indexed caches.
1594          * This code supports virtually indexed processors and will be
1595          * unnecessarily inefficient on physically indexed processors.
1596          */
1597         if (c->dcache.linesz)
1598                 shm_align_mask = max_t( unsigned long,
1599                                         c->dcache.sets * c->dcache.linesz - 1,
1600                                         PAGE_SIZE - 1);
1601         else
1602                 shm_align_mask = PAGE_SIZE-1;
1603
1604         __flush_cache_vmap      = r4k__flush_cache_vmap;
1605         __flush_cache_vunmap    = r4k__flush_cache_vunmap;
1606
1607         flush_cache_all         = cache_noop;
1608         __flush_cache_all       = r4k___flush_cache_all;
1609         flush_cache_mm          = r4k_flush_cache_mm;
1610         flush_cache_page        = r4k_flush_cache_page;
1611         flush_cache_range       = r4k_flush_cache_range;
1612
1613         __flush_kernel_vmap_range = r4k_flush_kernel_vmap_range;
1614
1615         flush_cache_sigtramp    = r4k_flush_cache_sigtramp;
1616         flush_icache_all        = r4k_flush_icache_all;
1617         local_flush_data_cache_page     = local_r4k_flush_data_cache_page;
1618         flush_data_cache_page   = r4k_flush_data_cache_page;
1619         flush_icache_range      = r4k_flush_icache_range;
1620         local_flush_icache_range        = local_r4k_flush_icache_range;
1621
1622 #if defined(CONFIG_DMA_NONCOHERENT) || defined(CONFIG_DMA_MAYBE_COHERENT)
1623         if (coherentio) {
1624                 _dma_cache_wback_inv    = (void *)cache_noop;
1625                 _dma_cache_wback        = (void *)cache_noop;
1626                 _dma_cache_inv          = (void *)cache_noop;
1627         } else {
1628                 _dma_cache_wback_inv    = r4k_dma_cache_wback_inv;
1629                 _dma_cache_wback        = r4k_dma_cache_wback_inv;
1630                 _dma_cache_inv          = r4k_dma_cache_inv;
1631         }
1632 #endif
1633
1634         build_clear_page();
1635         build_copy_page();
1636
1637         /*
1638          * We want to run CMP kernels on core with and without coherent
1639          * caches. Therefore, do not use CONFIG_MIPS_CMP to decide whether
1640          * or not to flush caches.
1641          */
1642         local_r4k___flush_cache_all(NULL);
1643
1644         coherency_setup();
1645         board_cache_error_setup = r4k_cache_error_setup;
1646 }
1647
1648 static int r4k_cache_pm_notifier(struct notifier_block *self, unsigned long cmd,
1649                                void *v)
1650 {
1651         switch (cmd) {
1652         case CPU_PM_ENTER_FAILED:
1653         case CPU_PM_EXIT:
1654                 coherency_setup();
1655                 break;
1656         }
1657
1658         return NOTIFY_OK;
1659 }
1660
1661 static struct notifier_block r4k_cache_pm_notifier_block = {
1662         .notifier_call = r4k_cache_pm_notifier,
1663 };
1664
1665 int __init r4k_cache_init_pm(void)
1666 {
1667         return cpu_pm_register_notifier(&r4k_cache_pm_notifier_block);
1668 }
1669 arch_initcall(r4k_cache_init_pm);