arch/mips/mm/c-r4k.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
   7  * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org)
   8  * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
   9  */
  10 #include <linux/config.h>
  11 #include <linux/init.h>
  12 #include <linux/kernel.h>
  13 #include <linux/sched.h>
  14 #include <linux/mm.h>
  15 #include <linux/bitops.h>
  16
  17 #include <asm/bcache.h>
  18 #include <asm/bootinfo.h>
  19 #include <asm/cache.h>
  20 #include <asm/cacheops.h>
  21 #include <asm/cpu.h>
  22 #include <asm/cpu-features.h>
  23 #include <asm/io.h>
  24 #include <asm/page.h>
  25 #include <asm/pgtable.h>
  26 #include <asm/r4kcache.h>
  27 #include <asm/system.h>
  28 #include <asm/mmu_context.h>
  29 #include <asm/war.h>
  30 #include <asm/cacheflush.h> /* for run_uncached() */
  31
  32
  33 /*
  34  * Special Variant of smp_call_function for use by cache functions:
  35  *
  36  *  o No return value
  37  *  o collapses to normal function call on UP kernels
  38  *  o collapses to normal function call on systems with a single shared
  39  *    primary cache.
  40  */
  41 static inline void r4k_on_each_cpu(void (*func) (void *info), void *info,
  42                                    int retry, int wait)
  43 {
  44         preempt_disable();
  45
  46 #if !defined(CONFIG_MIPS_MT_SMP) && !defined(CONFIG_MIPS_MT_SMTC)
  47         smp_call_function(func, info, retry, wait);
  48 #endif
  49         func(info);
  50         preempt_enable();
  51 }
  52
  53 /*
  54  * Must die.
  55  */
  56 static unsigned long icache_size __read_mostly;
  57 static unsigned long dcache_size __read_mostly;
  58 static unsigned long scache_size __read_mostly;
  59
  60 /*
  61  * Dummy cache handling routines for machines without boardcaches
  62  */
  63 static void no_sc_noop(void) {}
  64
  65 static struct bcache_ops no_sc_ops = {
  66         .bc_enable = (void *)no_sc_noop,
  67         .bc_disable = (void *)no_sc_noop,
  68         .bc_wback_inv = (void *)no_sc_noop,
  69         .bc_inv = (void *)no_sc_noop
  70 };
  71
  72 struct bcache_ops *bcops = &no_sc_ops;
  73
  74 #define cpu_is_r4600_v1_x()     ((read_c0_prid() & 0xfffffff0) == 0x00002010)
  75 #define cpu_is_r4600_v2_x()     ((read_c0_prid() & 0xfffffff0) == 0x00002020)
  76
  77 #define R4600_HIT_CACHEOP_WAR_IMPL                                      \
  78 do {                                                                    \
  79         if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())            \
  80                 *(volatile unsigned long *)CKSEG1;                      \
  81         if (R4600_V1_HIT_CACHEOP_WAR)                                   \
  82                 __asm__ __volatile__("nop;nop;nop;nop");                \
  83 } while (0)
  84
  85 static void (*r4k_blast_dcache_page)(unsigned long addr);
  86
  87 static inline void r4k_blast_dcache_page_dc32(unsigned long addr)
  88 {
  89         R4600_HIT_CACHEOP_WAR_IMPL;
  90         blast_dcache32_page(addr);
  91 }
  92
  93 static inline void r4k_blast_dcache_page_setup(void)
  94 {
  95         unsigned long  dc_lsize = cpu_dcache_line_size();
  96
  97         if (dc_lsize == 16)
  98                 r4k_blast_dcache_page = blast_dcache16_page;
  99         else if (dc_lsize == 32)
 100                 r4k_blast_dcache_page = r4k_blast_dcache_page_dc32;
 101 }
 102
 103 static void (* r4k_blast_dcache_page_indexed)(unsigned long addr);
 104
 105 static inline void r4k_blast_dcache_page_indexed_setup(void)
 106 {
 107         unsigned long dc_lsize = cpu_dcache_line_size();
 108
 109         if (dc_lsize == 16)
 110                 r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed;
 111         else if (dc_lsize == 32)
 112                 r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed;
 113 }
 114
 115 static void (* r4k_blast_dcache)(void);
 116
 117 static inline void r4k_blast_dcache_setup(void)
 118 {
 119         unsigned long dc_lsize = cpu_dcache_line_size();
 120
 121         if (dc_lsize == 16)
 122                 r4k_blast_dcache = blast_dcache16;
 123         else if (dc_lsize == 32)
 124                 r4k_blast_dcache = blast_dcache32;
 125 }
 126
 127 /* force code alignment (used for TX49XX_ICACHE_INDEX_INV_WAR) */
 128 #define JUMP_TO_ALIGN(order) \
 129         __asm__ __volatile__( \
 130                 "b\t1f\n\t" \
 131                 ".align\t" #order "\n\t" \
 132                 "1:\n\t" \
 133                 )
 134 #define CACHE32_UNROLL32_ALIGN  JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */
 135 #define CACHE32_UNROLL32_ALIGN2 JUMP_TO_ALIGN(11)
 136
 137 static inline void blast_r4600_v1_icache32(void)
 138 {
 139         unsigned long flags;
 140
 141         local_irq_save(flags);
 142         blast_icache32();
 143         local_irq_restore(flags);
 144 }
 145
 146 static inline void tx49_blast_icache32(void)
 147 {
 148         unsigned long start = INDEX_BASE;
 149         unsigned long end = start + current_cpu_data.icache.waysize;
 150         unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
 151         unsigned long ws_end = current_cpu_data.icache.ways <<
 152                                current_cpu_data.icache.waybit;
 153         unsigned long ws, addr;
 154
 155         CACHE32_UNROLL32_ALIGN2;
 156         /* I'm in even chunk.  blast odd chunks */
 157         for (ws = 0; ws < ws_end; ws += ws_inc)
 158                 for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
 159                         cache32_unroll32(addr|ws,Index_Invalidate_I);
 160         CACHE32_UNROLL32_ALIGN;
 161         /* I'm in odd chunk.  blast even chunks */
 162         for (ws = 0; ws < ws_end; ws += ws_inc)
 163                 for (addr = start; addr < end; addr += 0x400 * 2)
 164                         cache32_unroll32(addr|ws,Index_Invalidate_I);
 165 }
 166
 167 static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page)
 168 {
 169         unsigned long flags;
 170
 171         local_irq_save(flags);
 172         blast_icache32_page_indexed(page);
 173         local_irq_restore(flags);
 174 }
 175
 176 static inline void tx49_blast_icache32_page_indexed(unsigned long page)
 177 {
 178         unsigned long indexmask = current_cpu_data.icache.waysize - 1;
 179         unsigned long start = INDEX_BASE + (page & indexmask);
 180         unsigned long end = start + PAGE_SIZE;
 181         unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
 182         unsigned long ws_end = current_cpu_data.icache.ways <<
 183                                current_cpu_data.icache.waybit;
 184         unsigned long ws, addr;
 185
 186         CACHE32_UNROLL32_ALIGN2;
 187         /* I'm in even chunk.  blast odd chunks */
 188         for (ws = 0; ws < ws_end; ws += ws_inc)
 189                 for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
 190                         cache32_unroll32(addr|ws,Index_Invalidate_I);
 191         CACHE32_UNROLL32_ALIGN;
 192         /* I'm in odd chunk.  blast even chunks */
 193         for (ws = 0; ws < ws_end; ws += ws_inc)
 194                 for (addr = start; addr < end; addr += 0x400 * 2)
 195                         cache32_unroll32(addr|ws,Index_Invalidate_I);
 196 }
 197
 198 static void (* r4k_blast_icache_page)(unsigned long addr);
 199
 200 static inline void r4k_blast_icache_page_setup(void)
 201 {
 202         unsigned long ic_lsize = cpu_icache_line_size();
 203
 204         if (ic_lsize == 16)
 205                 r4k_blast_icache_page = blast_icache16_page;
 206         else if (ic_lsize == 32)
 207                 r4k_blast_icache_page = blast_icache32_page;
 208         else if (ic_lsize == 64)
 209                 r4k_blast_icache_page = blast_icache64_page;
 210 }
 211
 212
 213 static void (* r4k_blast_icache_page_indexed)(unsigned long addr);
 214
 215 static inline void r4k_blast_icache_page_indexed_setup(void)
 216 {
 217         unsigned long ic_lsize = cpu_icache_line_size();
 218
 219         if (ic_lsize == 16)
 220                 r4k_blast_icache_page_indexed = blast_icache16_page_indexed;
 221         else if (ic_lsize == 32) {
 222                 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
 223                         r4k_blast_icache_page_indexed =
 224                                 blast_icache32_r4600_v1_page_indexed;
 225                 else if (TX49XX_ICACHE_INDEX_INV_WAR)
 226                         r4k_blast_icache_page_indexed =
 227                                 tx49_blast_icache32_page_indexed;
 228                 else
 229                         r4k_blast_icache_page_indexed =
 230                                 blast_icache32_page_indexed;
 231         } else if (ic_lsize == 64)
 232                 r4k_blast_icache_page_indexed = blast_icache64_page_indexed;
 233 }
 234
 235 static void (* r4k_blast_icache)(void);
 236
 237 static inline void r4k_blast_icache_setup(void)
 238 {
 239         unsigned long ic_lsize = cpu_icache_line_size();
 240
 241         if (ic_lsize == 16)
 242                 r4k_blast_icache = blast_icache16;
 243         else if (ic_lsize == 32) {
 244                 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
 245                         r4k_blast_icache = blast_r4600_v1_icache32;
 246                 else if (TX49XX_ICACHE_INDEX_INV_WAR)
 247                         r4k_blast_icache = tx49_blast_icache32;
 248                 else
 249                         r4k_blast_icache = blast_icache32;
 250         } else if (ic_lsize == 64)
 251                 r4k_blast_icache = blast_icache64;
 252 }
 253
 254 static void (* r4k_blast_scache_page)(unsigned long addr);
 255
 256 static inline void r4k_blast_scache_page_setup(void)
 257 {
 258         unsigned long sc_lsize = cpu_scache_line_size();
 259
 260         if (scache_size == 0)
 261                 r4k_blast_scache_page = (void *)no_sc_noop;
 262         else if (sc_lsize == 16)
 263                 r4k_blast_scache_page = blast_scache16_page;
 264         else if (sc_lsize == 32)
 265                 r4k_blast_scache_page = blast_scache32_page;
 266         else if (sc_lsize == 64)
 267                 r4k_blast_scache_page = blast_scache64_page;
 268         else if (sc_lsize == 128)
 269                 r4k_blast_scache_page = blast_scache128_page;
 270 }
 271
 272 static void (* r4k_blast_scache_page_indexed)(unsigned long addr);
 273
 274 static inline void r4k_blast_scache_page_indexed_setup(void)
 275 {
 276         unsigned long sc_lsize = cpu_scache_line_size();
 277
 278         if (scache_size == 0)
 279                 r4k_blast_scache_page_indexed = (void *)no_sc_noop;
 280         else if (sc_lsize == 16)
 281                 r4k_blast_scache_page_indexed = blast_scache16_page_indexed;
 282         else if (sc_lsize == 32)
 283                 r4k_blast_scache_page_indexed = blast_scache32_page_indexed;
 284         else if (sc_lsize == 64)
 285                 r4k_blast_scache_page_indexed = blast_scache64_page_indexed;
 286         else if (sc_lsize == 128)
 287                 r4k_blast_scache_page_indexed = blast_scache128_page_indexed;
 288 }
 289
 290 static void (* r4k_blast_scache)(void);
 291
 292 static inline void r4k_blast_scache_setup(void)
 293 {
 294         unsigned long sc_lsize = cpu_scache_line_size();
 295
 296         if (scache_size == 0)
 297                 r4k_blast_scache = (void *)no_sc_noop;
 298         else if (sc_lsize == 16)
 299                 r4k_blast_scache = blast_scache16;
 300         else if (sc_lsize == 32)
 301                 r4k_blast_scache = blast_scache32;
 302         else if (sc_lsize == 64)
 303                 r4k_blast_scache = blast_scache64;
 304         else if (sc_lsize == 128)
 305                 r4k_blast_scache = blast_scache128;
 306 }
 307
 308 /*
 309  * This is former mm's flush_cache_all() which really should be
 310  * flush_cache_vunmap these days ...
 311  */
 312 static inline void local_r4k_flush_cache_all(void * args)
 313 {
 314         r4k_blast_dcache();
 315         r4k_blast_icache();
 316 }
 317
 318 static void r4k_flush_cache_all(void)
 319 {
 320         if (!cpu_has_dc_aliases)
 321                 return;
 322
 323         r4k_on_each_cpu(local_r4k_flush_cache_all, NULL, 1, 1);
 324 }
 325
 326 static inline void local_r4k___flush_cache_all(void * args)
 327 {
 328         r4k_blast_dcache();
 329         r4k_blast_icache();
 330
 331         switch (current_cpu_data.cputype) {
 332         case CPU_R4000SC:
 333         case CPU_R4000MC:
 334         case CPU_R4400SC:
 335         case CPU_R4400MC:
 336         case CPU_R10000:
 337         case CPU_R12000:
 338                 r4k_blast_scache();
 339         }
 340 }
 341
 342 static void r4k___flush_cache_all(void)
 343 {
 344         r4k_on_each_cpu(local_r4k___flush_cache_all, NULL, 1, 1);
 345 }
 346
 347 static inline void local_r4k_flush_cache_range(void * args)
 348 {
 349         struct vm_area_struct *vma = args;
 350         int exec;
 351
 352         if (!(cpu_context(smp_processor_id(), vma->vm_mm)))
 353                 return;
 354
 355         exec = vma->vm_flags & VM_EXEC;
 356         if (cpu_has_dc_aliases || exec)
 357                 r4k_blast_dcache();
 358         if (exec)
 359                 r4k_blast_icache();
 360 }
 361
 362 static void r4k_flush_cache_range(struct vm_area_struct *vma,
 363         unsigned long start, unsigned long end)
 364 {
 365         r4k_on_each_cpu(local_r4k_flush_cache_range, vma, 1, 1);
 366 }
 367
 368 static inline void local_r4k_flush_cache_mm(void * args)
 369 {
 370         struct mm_struct *mm = args;
 371
 372         if (!cpu_context(smp_processor_id(), mm))
 373                 return;
 374
 375         r4k_blast_dcache();
 376         r4k_blast_icache();
 377
 378         /*
 379          * Kludge alert.  For obscure reasons R4000SC and R4400SC go nuts if we
 380          * only flush the primary caches but R10000 and R12000 behave sane ...
 381          */
 382         if (current_cpu_data.cputype == CPU_R4000SC ||
 383             current_cpu_data.cputype == CPU_R4000MC ||
 384             current_cpu_data.cputype == CPU_R4400SC ||
 385             current_cpu_data.cputype == CPU_R4400MC)
 386                 r4k_blast_scache();
 387 }
 388
 389 static void r4k_flush_cache_mm(struct mm_struct *mm)
 390 {
 391         if (!cpu_has_dc_aliases)
 392                 return;
 393
 394         r4k_on_each_cpu(local_r4k_flush_cache_mm, mm, 1, 1);
 395 }
 396
 397 struct flush_cache_page_args {
 398         struct vm_area_struct *vma;
 399         unsigned long addr;
 400         unsigned long pfn;
 401 };
 402
 403 static inline void local_r4k_flush_cache_page(void *args)
 404 {
 405         struct flush_cache_page_args *fcp_args = args;
 406         struct vm_area_struct *vma = fcp_args->vma;
 407         unsigned long addr = fcp_args->addr;
 408         unsigned long paddr = fcp_args->pfn << PAGE_SHIFT;
 409         int exec = vma->vm_flags & VM_EXEC;
 410         struct mm_struct *mm = vma->vm_mm;
 411         pgd_t *pgdp;
 412         pud_t *pudp;
 413         pmd_t *pmdp;
 414         pte_t *ptep;
 415
 416         /*
 417          * If ownes no valid ASID yet, cannot possibly have gotten
 418          * this page into the cache.
 419          */
 420         if (cpu_context(smp_processor_id(), mm) == 0)
 421                 return;
 422
 423         addr &= PAGE_MASK;
 424         pgdp = pgd_offset(mm, addr);
 425         pudp = pud_offset(pgdp, addr);
 426         pmdp = pmd_offset(pudp, addr);
 427         ptep = pte_offset(pmdp, addr);
 428
 429         /*
 430          * If the page isn't marked valid, the page cannot possibly be
 431          * in the cache.
 432          */
 433         if (!(pte_val(*ptep) & _PAGE_PRESENT))
 434                 return;
 435
 436         /*
 437          * Doing flushes for another ASID than the current one is
 438          * too difficult since stupid R4k caches do a TLB translation
 439          * for every cache flush operation.  So we do indexed flushes
 440          * in that case, which doesn't overly flush the cache too much.
 441          */
 442         if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) {
 443                 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) {
 444                         r4k_blast_dcache_page(addr);
 445                         if (exec && !cpu_icache_snoops_remote_store)
 446                                 r4k_blast_scache_page(addr);
 447                 }
 448                 if (exec)
 449                         r4k_blast_icache_page(addr);
 450
 451                 return;
 452         }
 453
 454         /*
 455          * Do indexed flush, too much work to get the (possible) TLB refills
 456          * to work correctly.
 457          */
 458         if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) {
 459                 r4k_blast_dcache_page_indexed(cpu_has_pindexed_dcache ?
 460                                               paddr : addr);
 461                 if (exec && !cpu_icache_snoops_remote_store) {
 462                         r4k_blast_scache_page_indexed(paddr);
 463                 }
 464         }
 465         if (exec) {
 466                 if (cpu_has_vtag_icache) {
 467                         int cpu = smp_processor_id();
 468
 469                         if (cpu_context(cpu, mm) != 0)
 470                                 drop_mmu_context(mm, cpu);
 471                 } else
 472                         r4k_blast_icache_page_indexed(addr);
 473         }
 474 }
 475
 476 static void r4k_flush_cache_page(struct vm_area_struct *vma,
 477         unsigned long addr, unsigned long pfn)
 478 {
 479         struct flush_cache_page_args args;
 480
 481         args.vma = vma;
 482         args.addr = addr;
 483         args.pfn = pfn;
 484
 485         r4k_on_each_cpu(local_r4k_flush_cache_page, &args, 1, 1);
 486 }
 487
 488 static inline void local_r4k_flush_data_cache_page(void * addr)
 489 {
 490         r4k_blast_dcache_page((unsigned long) addr);
 491 }
 492
 493 static void r4k_flush_data_cache_page(unsigned long addr)
 494 {
 495         r4k_on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr, 1, 1);
 496 }
 497
 498 struct flush_icache_range_args {
 499         unsigned long start;
 500         unsigned long end;
 501 };
 502
 503 static inline void local_r4k_flush_icache_range(void *args)
 504 {
 505         struct flush_icache_range_args *fir_args = args;
 506         unsigned long start = fir_args->start;
 507         unsigned long end = fir_args->end;
 508
 509         if (!cpu_has_ic_fills_f_dc) {
 510                 if (end - start > dcache_size) {
 511                         r4k_blast_dcache();
 512                 } else {
 513                         R4600_HIT_CACHEOP_WAR_IMPL;
 514                         protected_blast_dcache_range(start, end);
 515                 }
 516
 517                 if (!cpu_icache_snoops_remote_store && scache_size) {
 518                         if (end - start > scache_size)
 519                                 r4k_blast_scache();
 520                         else
 521                                 protected_blast_scache_range(start, end);
 522                 }
 523         }
 524
 525         if (end - start > icache_size)
 526                 r4k_blast_icache();
 527         else
 528                 protected_blast_icache_range(start, end);
 529 }
 530
 531 static void r4k_flush_icache_range(unsigned long start, unsigned long end)
 532 {
 533         struct flush_icache_range_args args;
 534
 535         args.start = start;
 536         args.end = end;
 537
 538         r4k_on_each_cpu(local_r4k_flush_icache_range, &args, 1, 1);
 539         instruction_hazard();
 540 }
 541
 542 /*
 543  * Ok, this seriously sucks.  We use them to flush a user page but don't
 544  * know the virtual address, so we have to blast away the whole icache
 545  * which is significantly more expensive than the real thing.  Otoh we at
 546  * least know the kernel address of the page so we can flush it
 547  * selectivly.
 548  */
 549
 550 struct flush_icache_page_args {
 551         struct vm_area_struct *vma;
 552         struct page *page;
 553 };
 554
 555 static inline void local_r4k_flush_icache_page(void *args)
 556 {
 557         struct flush_icache_page_args *fip_args = args;
 558         struct vm_area_struct *vma = fip_args->vma;
 559         struct page *page = fip_args->page;
 560
 561         /*
 562          * Tricky ...  Because we don't know the virtual address we've got the
 563          * choice of either invalidating the entire primary and secondary
 564          * caches or invalidating the secondary caches also.  With the subset
 565          * enforcment on R4000SC, R4400SC, R10000 and R12000 invalidating the
 566          * secondary cache will result in any entries in the primary caches
 567          * also getting invalidated which hopefully is a bit more economical.
 568          */
 569         if (cpu_has_subset_pcaches) {
 570                 unsigned long addr = (unsigned long) page_address(page);
 571
 572                 r4k_blast_scache_page(addr);
 573                 ClearPageDcacheDirty(page);
 574
 575                 return;
 576         }
 577
 578         if (!cpu_has_ic_fills_f_dc) {
 579                 unsigned long addr = (unsigned long) page_address(page);
 580                 r4k_blast_dcache_page(addr);
 581                 if (!cpu_icache_snoops_remote_store)
 582                         r4k_blast_scache_page(addr);
 583                 ClearPageDcacheDirty(page);
 584         }
 585
 586         /*
 587          * We're not sure of the virtual address(es) involved here, so
 588          * we have to flush the entire I-cache.
 589          */
 590         if (cpu_has_vtag_icache) {
 591                 int cpu = smp_processor_id();
 592
 593                 if (cpu_context(cpu, vma->vm_mm) != 0)
 594                         drop_mmu_context(vma->vm_mm, cpu);
 595         } else
 596                 r4k_blast_icache();
 597 }
 598
 599 static void r4k_flush_icache_page(struct vm_area_struct *vma,
 600         struct page *page)
 601 {
 602         struct flush_icache_page_args args;
 603
 604         /*
 605          * If there's no context yet, or the page isn't executable, no I-cache
 606          * flush is needed.
 607          */
 608         if (!(vma->vm_flags & VM_EXEC))
 609                 return;
 610
 611         args.vma = vma;
 612         args.page = page;
 613
 614         r4k_on_each_cpu(local_r4k_flush_icache_page, &args, 1, 1);
 615 }
 616
 617
 618 #ifdef CONFIG_DMA_NONCOHERENT
 619
 620 static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size)
 621 {
 622         /* Catch bad driver code */
 623         BUG_ON(size == 0);
 624
 625         if (cpu_has_subset_pcaches) {
 626                 if (size >= scache_size)
 627                         r4k_blast_scache();
 628                 else
 629                         blast_scache_range(addr, addr + size);
 630                 return;
 631         }
 632
 633         /*
 634          * Either no secondary cache or the available caches don't have the
 635          * subset property so we have to flush the primary caches
 636          * explicitly
 637          */
 638         if (size >= dcache_size) {
 639                 r4k_blast_dcache();
 640         } else {
 641                 R4600_HIT_CACHEOP_WAR_IMPL;
 642                 blast_dcache_range(addr, addr + size);
 643         }
 644
 645         bc_wback_inv(addr, size);
 646 }
 647
 648 static void r4k_dma_cache_inv(unsigned long addr, unsigned long size)
 649 {
 650         /* Catch bad driver code */
 651         BUG_ON(size == 0);
 652
 653         if (cpu_has_subset_pcaches) {
 654                 if (size >= scache_size)
 655                         r4k_blast_scache();
 656                 else
 657                         blast_scache_range(addr, addr + size);
 658                 return;
 659         }
 660
 661         if (size >= dcache_size) {
 662                 r4k_blast_dcache();
 663         } else {
 664                 R4600_HIT_CACHEOP_WAR_IMPL;
 665                 blast_dcache_range(addr, addr + size);
 666         }
 667
 668         bc_inv(addr, size);
 669 }
 670 #endif /* CONFIG_DMA_NONCOHERENT */
 671
 672 /*
 673  * While we're protected against bad userland addresses we don't care
 674  * very much about what happens in that case.  Usually a segmentation
 675  * fault will dump the process later on anyway ...
 676  */
 677 static void local_r4k_flush_cache_sigtramp(void * arg)
 678 {
 679         unsigned long ic_lsize = cpu_icache_line_size();
 680         unsigned long dc_lsize = cpu_dcache_line_size();
 681         unsigned long sc_lsize = cpu_scache_line_size();
 682         unsigned long addr = (unsigned long) arg;
 683
 684         R4600_HIT_CACHEOP_WAR_IMPL;
 685         protected_writeback_dcache_line(addr & ~(dc_lsize - 1));
 686         if (!cpu_icache_snoops_remote_store && scache_size)
 687                 protected_writeback_scache_line(addr & ~(sc_lsize - 1));
 688         protected_flush_icache_line(addr & ~(ic_lsize - 1));
 689         if (MIPS4K_ICACHE_REFILL_WAR) {
 690                 __asm__ __volatile__ (
 691                         ".set push\n\t"
 692                         ".set noat\n\t"
 693                         ".set mips3\n\t"
 694 #ifdef CONFIG_32BIT
 695                         "la     $at,1f\n\t"
 696 #endif
 697 #ifdef CONFIG_64BIT
 698                         "dla    $at,1f\n\t"
 699 #endif
 700                         "cache  %0,($at)\n\t"
 701                         "nop; nop; nop\n"
 702                         "1:\n\t"
 703                         ".set pop"
 704                         :
 705                         : "i" (Hit_Invalidate_I));
 706         }
 707         if (MIPS_CACHE_SYNC_WAR)
 708                 __asm__ __volatile__ ("sync");
 709 }
 710
 711 static void r4k_flush_cache_sigtramp(unsigned long addr)
 712 {
 713         r4k_on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr, 1, 1);
 714 }
 715
 716 static void r4k_flush_icache_all(void)
 717 {
 718         if (cpu_has_vtag_icache)
 719                 r4k_blast_icache();
 720 }
 721
 722 static inline void rm7k_erratum31(void)
 723 {
 724         const unsigned long ic_lsize = 32;
 725         unsigned long addr;
 726
 727         /* RM7000 erratum #31. The icache is screwed at startup. */
 728         write_c0_taglo(0);
 729         write_c0_taghi(0);
 730
 731         for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) {
 732                 __asm__ __volatile__ (
 733                         ".set push\n\t"
 734                         ".set noreorder\n\t"
 735                         ".set mips3\n\t"
 736                         "cache\t%1, 0(%0)\n\t"
 737                         "cache\t%1, 0x1000(%0)\n\t"
 738                         "cache\t%1, 0x2000(%0)\n\t"
 739                         "cache\t%1, 0x3000(%0)\n\t"
 740                         "cache\t%2, 0(%0)\n\t"
 741                         "cache\t%2, 0x1000(%0)\n\t"
 742                         "cache\t%2, 0x2000(%0)\n\t"
 743                         "cache\t%2, 0x3000(%0)\n\t"
 744                         "cache\t%1, 0(%0)\n\t"
 745                         "cache\t%1, 0x1000(%0)\n\t"
 746                         "cache\t%1, 0x2000(%0)\n\t"
 747                         "cache\t%1, 0x3000(%0)\n\t"
 748                         ".set pop\n"
 749                         :
 750                         : "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill));
 751         }
 752 }
 753
 754 static char *way_string[] __initdata = { NULL, "direct mapped", "2-way",
 755         "3-way", "4-way", "5-way", "6-way", "7-way", "8-way"
 756 };
 757
 758 static void __init probe_pcache(void)
 759 {
 760         struct cpuinfo_mips *c = &current_cpu_data;
 761         unsigned int config = read_c0_config();
 762         unsigned int prid = read_c0_prid();
 763         unsigned long config1;
 764         unsigned int lsize;
 765
 766         switch (c->cputype) {
 767         case CPU_R4600:                 /* QED style two way caches? */
 768         case CPU_R4700:
 769         case CPU_R5000:
 770         case CPU_NEVADA:
 771                 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
 772                 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
 773                 c->icache.ways = 2;
 774                 c->icache.waybit = __ffs(icache_size/2);
 775
 776                 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
 777                 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
 778                 c->dcache.ways = 2;
 779                 c->dcache.waybit= __ffs(dcache_size/2);
 780
 781                 c->options |= MIPS_CPU_CACHE_CDEX_P;
 782                 break;
 783
 784         case CPU_R5432:
 785         case CPU_R5500:
 786                 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
 787                 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
 788                 c->icache.ways = 2;
 789                 c->icache.waybit= 0;
 790
 791                 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
 792                 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
 793                 c->dcache.ways = 2;
 794                 c->dcache.waybit = 0;
 795
 796                 c->options |= MIPS_CPU_CACHE_CDEX_P;
 797                 break;
 798
 799         case CPU_TX49XX:
 800                 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
 801                 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
 802                 c->icache.ways = 4;
 803                 c->icache.waybit= 0;
 804
 805                 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
 806                 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
 807                 c->dcache.ways = 4;
 808                 c->dcache.waybit = 0;
 809
 810                 c->options |= MIPS_CPU_CACHE_CDEX_P;
 811                 c->options |= MIPS_CPU_PREFETCH;
 812                 break;
 813
 814         case CPU_R4000PC:
 815         case CPU_R4000SC:
 816         case CPU_R4000MC:
 817         case CPU_R4400PC:
 818         case CPU_R4400SC:
 819         case CPU_R4400MC:
 820         case CPU_R4300:
 821                 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
 822                 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
 823                 c->icache.ways = 1;
 824                 c->icache.waybit = 0;   /* doesn't matter */
 825
 826                 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
 827                 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
 828                 c->dcache.ways = 1;
 829                 c->dcache.waybit = 0;   /* does not matter */
 830
 831                 c->options |= MIPS_CPU_CACHE_CDEX_P;
 832                 break;
 833
 834         case CPU_R10000:
 835         case CPU_R12000:
 836                 icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29));
 837                 c->icache.linesz = 64;
 838                 c->icache.ways = 2;
 839                 c->icache.waybit = 0;
 840
 841                 dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26));
 842                 c->dcache.linesz = 32;
 843                 c->dcache.ways = 2;
 844                 c->dcache.waybit = 0;
 845
 846                 c->options |= MIPS_CPU_PREFETCH;
 847                 break;
 848
 849         case CPU_VR4133:
 850                 write_c0_config(config & ~CONF_EB);
 851         case CPU_VR4131:
 852                 /* Workaround for cache instruction bug of VR4131 */
 853                 if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U ||
 854                     c->processor_id == 0x0c82U) {
 855                         config &= ~0x00000030U;
 856                         config |= 0x00410000U;
 857                         write_c0_config(config);
 858                 }
 859                 icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
 860                 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
 861                 c->icache.ways = 2;
 862                 c->icache.waybit = __ffs(icache_size/2);
 863
 864                 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
 865                 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
 866                 c->dcache.ways = 2;
 867                 c->dcache.waybit = __ffs(dcache_size/2);
 868
 869                 c->options |= MIPS_CPU_CACHE_CDEX_P;
 870                 break;
 871
 872         case CPU_VR41XX:
 873         case CPU_VR4111:
 874         case CPU_VR4121:
 875         case CPU_VR4122:
 876         case CPU_VR4181:
 877         case CPU_VR4181A:
 878                 icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
 879                 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
 880                 c->icache.ways = 1;
 881                 c->icache.waybit = 0;   /* doesn't matter */
 882
 883                 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
 884                 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
 885                 c->dcache.ways = 1;
 886                 c->dcache.waybit = 0;   /* does not matter */
 887
 888                 c->options |= MIPS_CPU_CACHE_CDEX_P;
 889                 break;
 890
 891         case CPU_RM7000:
 892                 rm7k_erratum31();
 893
 894         case CPU_RM9000:
 895                 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
 896                 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
 897                 c->icache.ways = 4;
 898                 c->icache.waybit = __ffs(icache_size / c->icache.ways);
 899
 900                 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
 901                 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
 902                 c->dcache.ways = 4;
 903                 c->dcache.waybit = __ffs(dcache_size / c->dcache.ways);
 904
 905 #if !defined(CONFIG_SMP) || !defined(RM9000_CDEX_SMP_WAR)
 906                 c->options |= MIPS_CPU_CACHE_CDEX_P;
 907 #endif
 908                 c->options |= MIPS_CPU_PREFETCH;
 909                 break;
 910
 911         default:
 912                 if (!(config & MIPS_CONF_M))
 913                         panic("Don't know how to probe P-caches on this cpu.");
 914
 915                 /*
 916                  * So we seem to be a MIPS32 or MIPS64 CPU
 917                  * So let's probe the I-cache ...
 918                  */
 919                 config1 = read_c0_config1();
 920
 921                 if ((lsize = ((config1 >> 19) & 7)))
 922                         c->icache.linesz = 2 << lsize;
 923                 else
 924                         c->icache.linesz = lsize;
 925                 c->icache.sets = 64 << ((config1 >> 22) & 7);
 926                 c->icache.ways = 1 + ((config1 >> 16) & 7);
 927
 928                 icache_size = c->icache.sets *
 929                               c->icache.ways *
 930                               c->icache.linesz;
 931                 c->icache.waybit = __ffs(icache_size/c->icache.ways);
 932
 933                 if (config & 0x8)               /* VI bit */
 934                         c->icache.flags |= MIPS_CACHE_VTAG;
 935
 936                 /*
 937                  * Now probe the MIPS32 / MIPS64 data cache.
 938                  */
 939                 c->dcache.flags = 0;
 940
 941                 if ((lsize = ((config1 >> 10) & 7)))
 942                         c->dcache.linesz = 2 << lsize;
 943                 else
 944                         c->dcache.linesz= lsize;
 945                 c->dcache.sets = 64 << ((config1 >> 13) & 7);
 946                 c->dcache.ways = 1 + ((config1 >> 7) & 7);
 947
 948                 dcache_size = c->dcache.sets *
 949                               c->dcache.ways *
 950                               c->dcache.linesz;
 951                 c->dcache.waybit = __ffs(dcache_size/c->dcache.ways);
 952
 953                 c->options |= MIPS_CPU_PREFETCH;
 954                 break;
 955         }
 956
 957         /*
 958          * Processor configuration sanity check for the R4000SC erratum
 959          * #5.  With page sizes larger than 32kB there is no possibility
 960          * to get a VCE exception anymore so we don't care about this
 961          * misconfiguration.  The case is rather theoretical anyway;
 962          * presumably no vendor is shipping his hardware in the "bad"
 963          * configuration.
 964          */
 965         if ((prid & 0xff00) == PRID_IMP_R4000 && (prid & 0xff) < 0x40 &&
 966             !(config & CONF_SC) && c->icache.linesz != 16 &&
 967             PAGE_SIZE <= 0x8000)
 968                 panic("Improper R4000SC processor configuration detected");
 969
 970         /* compute a couple of other cache variables */
 971         c->icache.waysize = icache_size / c->icache.ways;
 972         c->dcache.waysize = dcache_size / c->dcache.ways;
 973
 974         c->icache.sets = icache_size / (c->icache.linesz * c->icache.ways);
 975         c->dcache.sets = dcache_size / (c->dcache.linesz * c->dcache.ways);
 976
 977         /*
 978          * R10000 and R12000 P-caches are odd in a positive way.  They're 32kB
 979          * 2-way virtually indexed so normally would suffer from aliases.  So
 980          * normally they'd suffer from aliases but magic in the hardware deals
 981          * with that for us so we don't need to take care ourselves.
 982          */
 983         switch (c->cputype) {
 984         case CPU_20KC:
 985         case CPU_25KF:
 986                 c->dcache.flags |= MIPS_CACHE_PINDEX;
 987         case CPU_R10000:
 988         case CPU_R12000:
 989         case CPU_SB1:
 990                 break;
 991         case CPU_24K:
 992         case CPU_34K:
 993                 if (!(read_c0_config7() & (1 << 16)))
 994         default:
 995                         if (c->dcache.waysize > PAGE_SIZE)
 996                                 c->dcache.flags |= MIPS_CACHE_ALIASES;
 997         }
 998
 999         switch (c->cputype) {
1000         case CPU_20KC:
1001                 /*
1002                  * Some older 20Kc chips doesn't have the 'VI' bit in
1003                  * the config register.
1004                  */
1005                 c->icache.flags |= MIPS_CACHE_VTAG;
1006                 break;
1007
1008         case CPU_AU1000:
1009         case CPU_AU1500:
1010         case CPU_AU1100:
1011         case CPU_AU1550:
1012         case CPU_AU1200:
1013                 c->icache.flags |= MIPS_CACHE_IC_F_DC;
1014                 break;
1015         }
1016
1017         printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n",
1018                icache_size >> 10,
1019                cpu_has_vtag_icache ? "virtually tagged" : "physically tagged",
1020                way_string[c->icache.ways], c->icache.linesz);
1021
1022         printk("Primary data cache %ldkB, %s, linesize %d bytes.\n",
1023                dcache_size >> 10, way_string[c->dcache.ways], c->dcache.linesz);
1024 }
1025
1026 /*
1027  * If you even _breathe_ on this function, look at the gcc output and make sure
1028  * it does not pop things on and off the stack for the cache sizing loop that
1029  * executes in KSEG1 space or else you will crash and burn badly.  You have
1030  * been warned.
1031  */
1032 static int __init probe_scache(void)
1033 {
1034         extern unsigned long stext;
1035         unsigned long flags, addr, begin, end, pow2;
1036         unsigned int config = read_c0_config();
1037         struct cpuinfo_mips *c = &current_cpu_data;
1038         int tmp;
1039
1040         if (config & CONF_SC)
1041                 return 0;
1042
1043         begin = (unsigned long) &stext;
1044         begin &= ~((4 * 1024 * 1024) - 1);
1045         end = begin + (4 * 1024 * 1024);
1046
1047         /*
1048          * This is such a bitch, you'd think they would make it easy to do
1049          * this.  Away you daemons of stupidity!
1050          */
1051         local_irq_save(flags);
1052
1053         /* Fill each size-multiple cache line with a valid tag. */
1054         pow2 = (64 * 1024);
1055         for (addr = begin; addr < end; addr = (begin + pow2)) {
1056                 unsigned long *p = (unsigned long *) addr;
1057                 __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */
1058                 pow2 <<= 1;
1059         }
1060
1061         /* Load first line with zero (therefore invalid) tag. */
1062         write_c0_taglo(0);
1063         write_c0_taghi(0);
1064         __asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */
1065         cache_op(Index_Store_Tag_I, begin);
1066         cache_op(Index_Store_Tag_D, begin);
1067         cache_op(Index_Store_Tag_SD, begin);
1068
1069         /* Now search for the wrap around point. */
1070         pow2 = (128 * 1024);
1071         tmp = 0;
1072         for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) {
1073                 cache_op(Index_Load_Tag_SD, addr);
1074                 __asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */
1075                 if (!read_c0_taglo())
1076                         break;
1077                 pow2 <<= 1;
1078         }
1079         local_irq_restore(flags);
1080         addr -= begin;
1081
1082         scache_size = addr;
1083         c->scache.linesz = 16 << ((config & R4K_CONF_SB) >> 22);
1084         c->scache.ways = 1;
1085         c->dcache.waybit = 0;           /* does not matter */
1086
1087         return 1;
1088 }
1089
1090 extern int r5k_sc_init(void);
1091 extern int rm7k_sc_init(void);
1092
1093 static void __init setup_scache(void)
1094 {
1095         struct cpuinfo_mips *c = &current_cpu_data;
1096         unsigned int config = read_c0_config();
1097         int sc_present = 0;
1098
1099         /*
1100          * Do the probing thing on R4000SC and R4400SC processors.  Other
1101          * processors don't have a S-cache that would be relevant to the
1102          * Linux memory managment.
1103          */
1104         switch (c->cputype) {
1105         case CPU_R4000SC:
1106         case CPU_R4000MC:
1107         case CPU_R4400SC:
1108         case CPU_R4400MC:
1109                 sc_present = run_uncached(probe_scache);
1110                 if (sc_present)
1111                         c->options |= MIPS_CPU_CACHE_CDEX_S;
1112                 break;
1113
1114         case CPU_R10000:
1115         case CPU_R12000:
1116                 scache_size = 0x80000 << ((config & R10K_CONF_SS) >> 16);
1117                 c->scache.linesz = 64 << ((config >> 13) & 1);
1118                 c->scache.ways = 2;
1119                 c->scache.waybit= 0;
1120                 sc_present = 1;
1121                 break;
1122
1123         case CPU_R5000:
1124         case CPU_NEVADA:
1125 #ifdef CONFIG_R5000_CPU_SCACHE
1126                 r5k_sc_init();
1127 #endif
1128                 return;
1129
1130         case CPU_RM7000:
1131         case CPU_RM9000:
1132 #ifdef CONFIG_RM7000_CPU_SCACHE
1133                 rm7k_sc_init();
1134 #endif
1135                 return;
1136
1137         default:
1138                 sc_present = 0;
1139         }
1140
1141         if (!sc_present)
1142                 return;
1143
1144         if ((c->isa_level == MIPS_CPU_ISA_M32R1 ||
1145              c->isa_level == MIPS_CPU_ISA_M64R1) &&
1146             !(c->scache.flags & MIPS_CACHE_NOT_PRESENT))
1147                 panic("Dunno how to handle MIPS32 / MIPS64 second level cache");
1148
1149         /* compute a couple of other cache variables */
1150         c->scache.waysize = scache_size / c->scache.ways;
1151
1152         c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways);
1153
1154         printk("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
1155                scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
1156
1157         c->options |= MIPS_CPU_SUBSET_CACHES;
1158 }
1159
1160 static inline void coherency_setup(void)
1161 {
1162         change_c0_config(CONF_CM_CMASK, CONF_CM_DEFAULT);
1163
1164         /*
1165          * c0_status.cu=0 specifies that updates by the sc instruction use
1166          * the coherency mode specified by the TLB; 1 means cachable
1167          * coherent update on write will be used.  Not all processors have
1168          * this bit and; some wire it to zero, others like Toshiba had the
1169          * silly idea of putting something else there ...
1170          */
1171         switch (current_cpu_data.cputype) {
1172         case CPU_R4000PC:
1173         case CPU_R4000SC:
1174         case CPU_R4000MC:
1175         case CPU_R4400PC:
1176         case CPU_R4400SC:
1177         case CPU_R4400MC:
1178                 clear_c0_config(CONF_CU);
1179                 break;
1180         }
1181 }
1182
1183 void __init r4k_cache_init(void)
1184 {
1185         extern void build_clear_page(void);
1186         extern void build_copy_page(void);
1187         extern char except_vec2_generic;
1188         struct cpuinfo_mips *c = &current_cpu_data;
1189
1190         /* Default cache error handler for R4000 and R5000 family */
1191         set_uncached_handler (0x100, &except_vec2_generic, 0x80);
1192
1193         probe_pcache();
1194         setup_scache();
1195
1196         r4k_blast_dcache_page_setup();
1197         r4k_blast_dcache_page_indexed_setup();
1198         r4k_blast_dcache_setup();
1199         r4k_blast_icache_page_setup();
1200         r4k_blast_icache_page_indexed_setup();
1201         r4k_blast_icache_setup();
1202         r4k_blast_scache_page_setup();
1203         r4k_blast_scache_page_indexed_setup();
1204         r4k_blast_scache_setup();
1205
1206         /*
1207          * Some MIPS32 and MIPS64 processors have physically indexed caches.
1208          * This code supports virtually indexed processors and will be
1209          * unnecessarily inefficient on physically indexed processors.
1210          */
1211         shm_align_mask = max_t( unsigned long,
1212                                 c->dcache.sets * c->dcache.linesz - 1,
1213                                 PAGE_SIZE - 1);
1214
1215         flush_cache_all         = r4k_flush_cache_all;
1216         __flush_cache_all       = r4k___flush_cache_all;
1217         flush_cache_mm          = r4k_flush_cache_mm;
1218         flush_cache_page        = r4k_flush_cache_page;
1219         flush_icache_page       = r4k_flush_icache_page;
1220         flush_cache_range       = r4k_flush_cache_range;
1221
1222         flush_cache_sigtramp    = r4k_flush_cache_sigtramp;
1223         flush_icache_all        = r4k_flush_icache_all;
1224         local_flush_data_cache_page     = local_r4k_flush_data_cache_page;
1225         flush_data_cache_page   = r4k_flush_data_cache_page;
1226         flush_icache_range      = r4k_flush_icache_range;
1227
1228 #ifdef CONFIG_DMA_NONCOHERENT
1229         _dma_cache_wback_inv    = r4k_dma_cache_wback_inv;
1230         _dma_cache_wback        = r4k_dma_cache_wback_inv;
1231         _dma_cache_inv          = r4k_dma_cache_inv;
1232 #endif
1233
1234         build_clear_page();
1235         build_copy_page();
1236         local_r4k___flush_cache_all(NULL);
1237         coherency_setup();
1238 }