arch/arm/mm/cache-l2x0.c

   1 /*
   2  * arch/arm/mm/cache-l2x0.c - L210/L220 cache controller support
   3  *
   4  * Copyright (C) 2007 ARM Limited
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  18  */
  19 #include <linux/cpu.h>
  20 #include <linux/err.h>
  21 #include <linux/init.h>
  22 #include <linux/smp.h>
  23 #include <linux/spinlock.h>
  24 #include <linux/io.h>
  25 #include <linux/of.h>
  26 #include <linux/of_address.h>
  27
  28 #include <asm/cacheflush.h>
  29 #include <asm/cp15.h>
  30 #include <asm/cputype.h>
  31 #include <asm/hardware/cache-l2x0.h>
  32 #include "cache-tauros3.h"
  33 #include "cache-aurora-l2.h"
  34
  35 struct l2c_init_data {
  36         const char *type;
  37         unsigned way_size_0;
  38         unsigned num_lock;
  39         void (*of_parse)(const struct device_node *, u32 *, u32 *);
  40         void (*enable)(void __iomem *, u32, unsigned);
  41         void (*fixup)(void __iomem *, u32, struct outer_cache_fns *);
  42         void (*save)(void __iomem *);
  43         struct outer_cache_fns outer_cache;
  44 };
  45
  46 #define CACHE_LINE_SIZE         32
  47
  48 static void __iomem *l2x0_base;
  49 static DEFINE_RAW_SPINLOCK(l2x0_lock);
  50 static u32 l2x0_way_mask;       /* Bitmask of active ways */
  51 static u32 l2x0_size;
  52 static unsigned long sync_reg_offset = L2X0_CACHE_SYNC;
  53
  54 struct l2x0_regs l2x0_saved_regs;
  55
  56 /*
  57  * Common code for all cache controllers.
  58  */
  59 static inline void l2c_wait_mask(void __iomem *reg, unsigned long mask)
  60 {
  61         /* wait for cache operation by line or way to complete */
  62         while (readl_relaxed(reg) & mask)
  63                 cpu_relax();
  64 }
  65
  66 /*
  67  * By default, we write directly to secure registers.  Platforms must
  68  * override this if they are running non-secure.
  69  */
  70 static void l2c_write_sec(unsigned long val, void __iomem *base, unsigned reg)
  71 {
  72         if (val == readl_relaxed(base + reg))
  73                 return;
  74         if (outer_cache.write_sec)
  75                 outer_cache.write_sec(val, reg);
  76         else
  77                 writel_relaxed(val, base + reg);
  78 }
  79
  80 /*
  81  * This should only be called when we have a requirement that the
  82  * register be written due to a work-around, as platforms running
  83  * in non-secure mode may not be able to access this register.
  84  */
  85 static inline void l2c_set_debug(void __iomem *base, unsigned long val)
  86 {
  87         l2c_write_sec(val, base, L2X0_DEBUG_CTRL);
  88 }
  89
  90 static void __l2c_op_way(void __iomem *reg)
  91 {
  92         writel_relaxed(l2x0_way_mask, reg);
  93         l2c_wait_mask(reg, l2x0_way_mask);
  94 }
  95
  96 static inline void l2c_unlock(void __iomem *base, unsigned num)
  97 {
  98         unsigned i;
  99
 100         for (i = 0; i < num; i++) {
 101                 writel_relaxed(0, base + L2X0_LOCKDOWN_WAY_D_BASE +
 102                                i * L2X0_LOCKDOWN_STRIDE);
 103                 writel_relaxed(0, base + L2X0_LOCKDOWN_WAY_I_BASE +
 104                                i * L2X0_LOCKDOWN_STRIDE);
 105         }
 106 }
 107
 108 /*
 109  * Enable the L2 cache controller.  This function must only be
 110  * called when the cache controller is known to be disabled.
 111  */
 112 static void l2c_enable(void __iomem *base, u32 aux, unsigned num_lock)
 113 {
 114         unsigned long flags;
 115
 116         l2c_write_sec(aux, base, L2X0_AUX_CTRL);
 117
 118         l2c_unlock(base, num_lock);
 119
 120         local_irq_save(flags);
 121         __l2c_op_way(base + L2X0_INV_WAY);
 122         writel_relaxed(0, base + sync_reg_offset);
 123         l2c_wait_mask(base + sync_reg_offset, 1);
 124         local_irq_restore(flags);
 125
 126         l2c_write_sec(L2X0_CTRL_EN, base, L2X0_CTRL);
 127 }
 128
 129 static void l2c_disable(void)
 130 {
 131         void __iomem *base = l2x0_base;
 132
 133         outer_cache.flush_all();
 134         l2c_write_sec(0, base, L2X0_CTRL);
 135         dsb(st);
 136 }
 137
 138 #ifdef CONFIG_CACHE_PL310
 139 static inline void cache_wait(void __iomem *reg, unsigned long mask)
 140 {
 141         /* cache operations by line are atomic on PL310 */
 142 }
 143 #else
 144 #define cache_wait      l2c_wait_mask
 145 #endif
 146
 147 static inline void cache_sync(void)
 148 {
 149         void __iomem *base = l2x0_base;
 150
 151         writel_relaxed(0, base + sync_reg_offset);
 152         cache_wait(base + L2X0_CACHE_SYNC, 1);
 153 }
 154
 155 #if defined(CONFIG_PL310_ERRATA_588369) || defined(CONFIG_PL310_ERRATA_727915)
 156 static inline void debug_writel(unsigned long val)
 157 {
 158         l2c_set_debug(l2x0_base, val);
 159 }
 160 #else
 161 /* Optimised out for non-errata case */
 162 static inline void debug_writel(unsigned long val)
 163 {
 164 }
 165 #endif
 166
 167 static void l2x0_cache_sync(void)
 168 {
 169         unsigned long flags;
 170
 171         raw_spin_lock_irqsave(&l2x0_lock, flags);
 172         cache_sync();
 173         raw_spin_unlock_irqrestore(&l2x0_lock, flags);
 174 }
 175
 176 static void __l2x0_flush_all(void)
 177 {
 178         debug_writel(0x03);
 179         __l2c_op_way(l2x0_base + L2X0_CLEAN_INV_WAY);
 180         cache_sync();
 181         debug_writel(0x00);
 182 }
 183
 184 static void l2x0_flush_all(void)
 185 {
 186         unsigned long flags;
 187
 188         /* clean all ways */
 189         raw_spin_lock_irqsave(&l2x0_lock, flags);
 190         __l2x0_flush_all();
 191         raw_spin_unlock_irqrestore(&l2x0_lock, flags);
 192 }
 193
 194 static void l2x0_disable(void)
 195 {
 196         unsigned long flags;
 197
 198         raw_spin_lock_irqsave(&l2x0_lock, flags);
 199         __l2x0_flush_all();
 200         l2c_write_sec(0, l2x0_base, L2X0_CTRL);
 201         dsb(st);
 202         raw_spin_unlock_irqrestore(&l2x0_lock, flags);
 203 }
 204
 205 static void l2c_save(void __iomem *base)
 206 {
 207         l2x0_saved_regs.aux_ctrl = readl_relaxed(l2x0_base + L2X0_AUX_CTRL);
 208 }
 209
 210 /*
 211  * L2C-210 specific code.
 212  *
 213  * The L2C-2x0 PA, set/way and sync operations are atomic, but we must
 214  * ensure that no background operation is running.  The way operations
 215  * are all background tasks.
 216  *
 217  * While a background operation is in progress, any new operation is
 218  * ignored (unspecified whether this causes an error.)  Thankfully, not
 219  * used on SMP.
 220  *
 221  * Never has a different sync register other than L2X0_CACHE_SYNC, but
 222  * we use sync_reg_offset here so we can share some of this with L2C-310.
 223  */
 224 static void __l2c210_cache_sync(void __iomem *base)
 225 {
 226         writel_relaxed(0, base + sync_reg_offset);
 227 }
 228
 229 static void __l2c210_op_pa_range(void __iomem *reg, unsigned long start,
 230         unsigned long end)
 231 {
 232         while (start < end) {
 233                 writel_relaxed(start, reg);
 234                 start += CACHE_LINE_SIZE;
 235         }
 236 }
 237
 238 static void l2c210_inv_range(unsigned long start, unsigned long end)
 239 {
 240         void __iomem *base = l2x0_base;
 241
 242         if (start & (CACHE_LINE_SIZE - 1)) {
 243                 start &= ~(CACHE_LINE_SIZE - 1);
 244                 writel_relaxed(start, base + L2X0_CLEAN_INV_LINE_PA);
 245                 start += CACHE_LINE_SIZE;
 246         }
 247
 248         if (end & (CACHE_LINE_SIZE - 1)) {
 249                 end &= ~(CACHE_LINE_SIZE - 1);
 250                 writel_relaxed(end, base + L2X0_CLEAN_INV_LINE_PA);
 251         }
 252
 253         __l2c210_op_pa_range(base + L2X0_INV_LINE_PA, start, end);
 254         __l2c210_cache_sync(base);
 255 }
 256
 257 static void l2c210_clean_range(unsigned long start, unsigned long end)
 258 {
 259         void __iomem *base = l2x0_base;
 260
 261         start &= ~(CACHE_LINE_SIZE - 1);
 262         __l2c210_op_pa_range(base + L2X0_CLEAN_LINE_PA, start, end);
 263         __l2c210_cache_sync(base);
 264 }
 265
 266 static void l2c210_flush_range(unsigned long start, unsigned long end)
 267 {
 268         void __iomem *base = l2x0_base;
 269
 270         start &= ~(CACHE_LINE_SIZE - 1);
 271         __l2c210_op_pa_range(base + L2X0_CLEAN_INV_LINE_PA, start, end);
 272         __l2c210_cache_sync(base);
 273 }
 274
 275 static void l2c210_flush_all(void)
 276 {
 277         void __iomem *base = l2x0_base;
 278
 279         BUG_ON(!irqs_disabled());
 280
 281         __l2c_op_way(base + L2X0_CLEAN_INV_WAY);
 282         __l2c210_cache_sync(base);
 283 }
 284
 285 static void l2c210_sync(void)
 286 {
 287         __l2c210_cache_sync(l2x0_base);
 288 }
 289
 290 static void l2c210_resume(void)
 291 {
 292         void __iomem *base = l2x0_base;
 293
 294         if (!(readl_relaxed(base + L2X0_CTRL) & L2X0_CTRL_EN))
 295                 l2c_enable(base, l2x0_saved_regs.aux_ctrl, 1);
 296 }
 297
 298 static const struct l2c_init_data l2c210_data __initconst = {
 299         .type = "L2C-210",
 300         .way_size_0 = SZ_8K,
 301         .num_lock = 1,
 302         .enable = l2c_enable,
 303         .save = l2c_save,
 304         .outer_cache = {
 305                 .inv_range = l2c210_inv_range,
 306                 .clean_range = l2c210_clean_range,
 307                 .flush_range = l2c210_flush_range,
 308                 .flush_all = l2c210_flush_all,
 309                 .disable = l2c_disable,
 310                 .sync = l2c210_sync,
 311                 .resume = l2c210_resume,
 312         },
 313 };
 314
 315 /*
 316  * L2C-220 specific code.
 317  *
 318  * All operations are background operations: they have to be waited for.
 319  * Conflicting requests generate a slave error (which will cause an
 320  * imprecise abort.)  Never uses sync_reg_offset, so we hard-code the
 321  * sync register here.
 322  *
 323  * However, we can re-use the l2c210_resume call.
 324  */
 325 static inline void __l2c220_cache_sync(void __iomem *base)
 326 {
 327         writel_relaxed(0, base + L2X0_CACHE_SYNC);
 328         l2c_wait_mask(base + L2X0_CACHE_SYNC, 1);
 329 }
 330
 331 static void l2c220_op_way(void __iomem *base, unsigned reg)
 332 {
 333         unsigned long flags;
 334
 335         raw_spin_lock_irqsave(&l2x0_lock, flags);
 336         __l2c_op_way(base + reg);
 337         __l2c220_cache_sync(base);
 338         raw_spin_unlock_irqrestore(&l2x0_lock, flags);
 339 }
 340
 341 static unsigned long l2c220_op_pa_range(void __iomem *reg, unsigned long start,
 342         unsigned long end, unsigned long flags)
 343 {
 344         raw_spinlock_t *lock = &l2x0_lock;
 345
 346         while (start < end) {
 347                 unsigned long blk_end = start + min(end - start, 4096UL);
 348
 349                 while (start < blk_end) {
 350                         l2c_wait_mask(reg, 1);
 351                         writel_relaxed(start, reg);
 352                         start += CACHE_LINE_SIZE;
 353                 }
 354
 355                 if (blk_end < end) {
 356                         raw_spin_unlock_irqrestore(lock, flags);
 357                         raw_spin_lock_irqsave(lock, flags);
 358                 }
 359         }
 360
 361         return flags;
 362 }
 363
 364 static void l2c220_inv_range(unsigned long start, unsigned long end)
 365 {
 366         void __iomem *base = l2x0_base;
 367         unsigned long flags;
 368
 369         raw_spin_lock_irqsave(&l2x0_lock, flags);
 370         if ((start | end) & (CACHE_LINE_SIZE - 1)) {
 371                 if (start & (CACHE_LINE_SIZE - 1)) {
 372                         start &= ~(CACHE_LINE_SIZE - 1);
 373                         writel_relaxed(start, base + L2X0_CLEAN_INV_LINE_PA);
 374                         start += CACHE_LINE_SIZE;
 375                 }
 376
 377                 if (end & (CACHE_LINE_SIZE - 1)) {
 378                         end &= ~(CACHE_LINE_SIZE - 1);
 379                         l2c_wait_mask(base + L2X0_CLEAN_INV_LINE_PA, 1);
 380                         writel_relaxed(end, base + L2X0_CLEAN_INV_LINE_PA);
 381                 }
 382         }
 383
 384         flags = l2c220_op_pa_range(base + L2X0_INV_LINE_PA,
 385                                    start, end, flags);
 386         l2c_wait_mask(base + L2X0_INV_LINE_PA, 1);
 387         __l2c220_cache_sync(base);
 388         raw_spin_unlock_irqrestore(&l2x0_lock, flags);
 389 }
 390
 391 static void l2c220_clean_range(unsigned long start, unsigned long end)
 392 {
 393         void __iomem *base = l2x0_base;
 394         unsigned long flags;
 395
 396         start &= ~(CACHE_LINE_SIZE - 1);
 397         if ((end - start) >= l2x0_size) {
 398                 l2c220_op_way(base, L2X0_CLEAN_WAY);
 399                 return;
 400         }
 401
 402         raw_spin_lock_irqsave(&l2x0_lock, flags);
 403         flags = l2c220_op_pa_range(base + L2X0_CLEAN_LINE_PA,
 404                                    start, end, flags);
 405         l2c_wait_mask(base + L2X0_CLEAN_INV_LINE_PA, 1);
 406         __l2c220_cache_sync(base);
 407         raw_spin_unlock_irqrestore(&l2x0_lock, flags);
 408 }
 409
 410 static void l2c220_flush_range(unsigned long start, unsigned long end)
 411 {
 412         void __iomem *base = l2x0_base;
 413         unsigned long flags;
 414
 415         start &= ~(CACHE_LINE_SIZE - 1);
 416         if ((end - start) >= l2x0_size) {
 417                 l2c220_op_way(base, L2X0_CLEAN_INV_WAY);
 418                 return;
 419         }
 420
 421         raw_spin_lock_irqsave(&l2x0_lock, flags);
 422         flags = l2c220_op_pa_range(base + L2X0_CLEAN_INV_LINE_PA,
 423                                    start, end, flags);
 424         l2c_wait_mask(base + L2X0_CLEAN_INV_LINE_PA, 1);
 425         __l2c220_cache_sync(base);
 426         raw_spin_unlock_irqrestore(&l2x0_lock, flags);
 427 }
 428
 429 static void l2c220_flush_all(void)
 430 {
 431         l2c220_op_way(l2x0_base, L2X0_CLEAN_INV_WAY);
 432 }
 433
 434 static void l2c220_sync(void)
 435 {
 436         unsigned long flags;
 437
 438         raw_spin_lock_irqsave(&l2x0_lock, flags);
 439         __l2c220_cache_sync(l2x0_base);
 440         raw_spin_unlock_irqrestore(&l2x0_lock, flags);
 441 }
 442
 443 static void l2c220_enable(void __iomem *base, u32 aux, unsigned num_lock)
 444 {
 445         /*
 446          * Always enable non-secure access to the lockdown registers -
 447          * we write to them as part of the L2C enable sequence so they
 448          * need to be accessible.
 449          */
 450         aux |= L220_AUX_CTRL_NS_LOCKDOWN;
 451
 452         l2c_enable(base, aux, num_lock);
 453 }
 454
 455 static const struct l2c_init_data l2c220_data = {
 456         .type = "L2C-220",
 457         .way_size_0 = SZ_8K,
 458         .num_lock = 1,
 459         .enable = l2c220_enable,
 460         .save = l2c_save,
 461         .outer_cache = {
 462                 .inv_range = l2c220_inv_range,
 463                 .clean_range = l2c220_clean_range,
 464                 .flush_range = l2c220_flush_range,
 465                 .flush_all = l2c220_flush_all,
 466                 .disable = l2c_disable,
 467                 .sync = l2c220_sync,
 468                 .resume = l2c210_resume,
 469         },
 470 };
 471
 472 /*
 473  * L2C-310 specific code.
 474  *
 475  * Very similar to L2C-210, the PA, set/way and sync operations are atomic,
 476  * and the way operations are all background tasks.  However, issuing an
 477  * operation while a background operation is in progress results in a
 478  * SLVERR response.  We can reuse:
 479  *
 480  *  __l2c210_cache_sync (using sync_reg_offset)
 481  *  l2c210_sync
 482  *  l2c210_inv_range (if 588369 is not applicable)
 483  *  l2c210_clean_range
 484  *  l2c210_flush_range (if 588369 is not applicable)
 485  *  l2c210_flush_all (if 727915 is not applicable)
 486  *
 487  * Errata:
 488  * 588369: PL310 R0P0->R1P0, fixed R2P0.
 489  *      Affects: all clean+invalidate operations
 490  *      clean and invalidate skips the invalidate step, so we need to issue
 491  *      separate operations.  We also require the above debug workaround
 492  *      enclosing this code fragment on affected parts.  On unaffected parts,
 493  *      we must not use this workaround without the debug register writes
 494  *      to avoid exposing a problem similar to 727915.
 495  *
 496  * 727915: PL310 R2P0->R3P0, fixed R3P1.
 497  *      Affects: clean+invalidate by way
 498  *      clean and invalidate by way runs in the background, and a store can
 499  *      hit the line between the clean operation and invalidate operation,
 500  *      resulting in the store being lost.
 501  *
 502  * 752271: PL310 R3P0->R3P1-50REL0, fixed R3P2.
 503  *      Affects: 8x64-bit (double fill) line fetches
 504  *      double fill line fetches can fail to cause dirty data to be evicted
 505  *      from the cache before the new data overwrites the second line.
 506  *
 507  * 753970: PL310 R3P0, fixed R3P1.
 508  *      Affects: sync
 509  *      prevents merging writes after the sync operation, until another L2C
 510  *      operation is performed (or a number of other conditions.)
 511  *
 512  * 769419: PL310 R0P0->R3P1, fixed R3P2.
 513  *      Affects: store buffer
 514  *      store buffer is not automatically drained.
 515  */
 516 static void l2c310_inv_range_erratum(unsigned long start, unsigned long end)
 517 {
 518         void __iomem *base = l2x0_base;
 519
 520         if ((start | end) & (CACHE_LINE_SIZE - 1)) {
 521                 unsigned long flags;
 522
 523                 /* Erratum 588369 for both clean+invalidate operations */
 524                 raw_spin_lock_irqsave(&l2x0_lock, flags);
 525                 l2c_set_debug(base, 0x03);
 526
 527                 if (start & (CACHE_LINE_SIZE - 1)) {
 528                         start &= ~(CACHE_LINE_SIZE - 1);
 529                         writel_relaxed(start, base + L2X0_CLEAN_LINE_PA);
 530                         writel_relaxed(start, base + L2X0_INV_LINE_PA);
 531                         start += CACHE_LINE_SIZE;
 532                 }
 533
 534                 if (end & (CACHE_LINE_SIZE - 1)) {
 535                         end &= ~(CACHE_LINE_SIZE - 1);
 536                         writel_relaxed(end, base + L2X0_CLEAN_LINE_PA);
 537                         writel_relaxed(end, base + L2X0_INV_LINE_PA);
 538                 }
 539
 540                 l2c_set_debug(base, 0x00);
 541                 raw_spin_unlock_irqrestore(&l2x0_lock, flags);
 542         }
 543
 544         __l2c210_op_pa_range(base + L2X0_INV_LINE_PA, start, end);
 545         __l2c210_cache_sync(base);
 546 }
 547
 548 static void l2c310_flush_range_erratum(unsigned long start, unsigned long end)
 549 {
 550         raw_spinlock_t *lock = &l2x0_lock;
 551         unsigned long flags;
 552         void __iomem *base = l2x0_base;
 553
 554         raw_spin_lock_irqsave(lock, flags);
 555         while (start < end) {
 556                 unsigned long blk_end = start + min(end - start, 4096UL);
 557
 558                 l2c_set_debug(base, 0x03);
 559                 while (start < blk_end) {
 560                         writel_relaxed(start, base + L2X0_CLEAN_LINE_PA);
 561                         writel_relaxed(start, base + L2X0_INV_LINE_PA);
 562                         start += CACHE_LINE_SIZE;
 563                 }
 564                 l2c_set_debug(base, 0x00);
 565
 566                 if (blk_end < end) {
 567                         raw_spin_unlock_irqrestore(lock, flags);
 568                         raw_spin_lock_irqsave(lock, flags);
 569                 }
 570         }
 571         raw_spin_unlock_irqrestore(lock, flags);
 572         __l2c210_cache_sync(base);
 573 }
 574
 575 static void l2c310_flush_all_erratum(void)
 576 {
 577         void __iomem *base = l2x0_base;
 578         unsigned long flags;
 579
 580         raw_spin_lock_irqsave(&l2x0_lock, flags);
 581         l2c_set_debug(base, 0x03);
 582         __l2c_op_way(base + L2X0_CLEAN_INV_WAY);
 583         l2c_set_debug(base, 0x00);
 584         __l2c210_cache_sync(base);
 585         raw_spin_unlock_irqrestore(&l2x0_lock, flags);
 586 }
 587
 588 static void __init l2c310_save(void __iomem *base)
 589 {
 590         unsigned revision;
 591
 592         l2c_save(base);
 593
 594         l2x0_saved_regs.tag_latency = readl_relaxed(base +
 595                 L310_TAG_LATENCY_CTRL);
 596         l2x0_saved_regs.data_latency = readl_relaxed(base +
 597                 L310_DATA_LATENCY_CTRL);
 598         l2x0_saved_regs.filter_end = readl_relaxed(base +
 599                 L310_ADDR_FILTER_END);
 600         l2x0_saved_regs.filter_start = readl_relaxed(base +
 601                 L310_ADDR_FILTER_START);
 602
 603         revision = readl_relaxed(base + L2X0_CACHE_ID) &
 604                         L2X0_CACHE_ID_RTL_MASK;
 605
 606         /* From r2p0, there is Prefetch offset/control register */
 607         if (revision >= L310_CACHE_ID_RTL_R2P0)
 608                 l2x0_saved_regs.prefetch_ctrl = readl_relaxed(base +
 609                                                         L310_PREFETCH_CTRL);
 610
 611         /* From r3p0, there is Power control register */
 612         if (revision >= L310_CACHE_ID_RTL_R3P0)
 613                 l2x0_saved_regs.pwr_ctrl = readl_relaxed(base +
 614                                                         L310_POWER_CTRL);
 615 }
 616
 617 static void l2c310_resume(void)
 618 {
 619         void __iomem *base = l2x0_base;
 620
 621         if (!(readl_relaxed(base + L2X0_CTRL) & L2X0_CTRL_EN)) {
 622                 unsigned revision;
 623
 624                 /* restore pl310 setup */
 625                 writel_relaxed(l2x0_saved_regs.tag_latency,
 626                                base + L310_TAG_LATENCY_CTRL);
 627                 writel_relaxed(l2x0_saved_regs.data_latency,
 628                                base + L310_DATA_LATENCY_CTRL);
 629                 writel_relaxed(l2x0_saved_regs.filter_end,
 630                                base + L310_ADDR_FILTER_END);
 631                 writel_relaxed(l2x0_saved_regs.filter_start,
 632                                base + L310_ADDR_FILTER_START);
 633
 634                 revision = readl_relaxed(base + L2X0_CACHE_ID) &
 635                                 L2X0_CACHE_ID_RTL_MASK;
 636
 637                 if (revision >= L310_CACHE_ID_RTL_R2P0)
 638                         l2c_write_sec(l2x0_saved_regs.prefetch_ctrl, base,
 639                                       L310_PREFETCH_CTRL);
 640                 if (revision >= L310_CACHE_ID_RTL_R3P0)
 641                         l2c_write_sec(l2x0_saved_regs.pwr_ctrl, base,
 642                                       L310_POWER_CTRL);
 643
 644                 l2c_enable(base, l2x0_saved_regs.aux_ctrl, 8);
 645
 646                 /* Re-enable full-line-of-zeros for Cortex-A9 */
 647                 if (l2x0_saved_regs.aux_ctrl & L310_AUX_CTRL_FULL_LINE_ZERO)
 648                         set_auxcr(get_auxcr() | BIT(3) | BIT(2) | BIT(1));
 649         }
 650 }
 651
 652 static int l2c310_cpu_enable_flz(struct notifier_block *nb, unsigned long act, void *data)
 653 {
 654         switch (act & ~CPU_TASKS_FROZEN) {
 655         case CPU_STARTING:
 656                 set_auxcr(get_auxcr() | BIT(3) | BIT(2) | BIT(1));
 657                 break;
 658         case CPU_DYING:
 659                 set_auxcr(get_auxcr() & ~(BIT(3) | BIT(2) | BIT(1)));
 660                 break;
 661         }
 662         return NOTIFY_OK;
 663 }
 664
 665 static void __init l2c310_enable(void __iomem *base, u32 aux, unsigned num_lock)
 666 {
 667         unsigned rev = readl_relaxed(base + L2X0_CACHE_ID) & L2X0_CACHE_ID_RTL_MASK;
 668         bool cortex_a9 = read_cpuid_part_number() == ARM_CPU_PART_CORTEX_A9;
 669
 670         if (rev >= L310_CACHE_ID_RTL_R2P0) {
 671                 if (cortex_a9) {
 672                         aux |= L310_AUX_CTRL_EARLY_BRESP;
 673                         pr_info("L2C-310 enabling early BRESP for Cortex-A9\n");
 674                 } else if (aux & L310_AUX_CTRL_EARLY_BRESP) {
 675                         pr_warn("L2C-310 early BRESP only supported with Cortex-A9\n");
 676                         aux &= ~L310_AUX_CTRL_EARLY_BRESP;
 677                 }
 678         }
 679
 680         if (cortex_a9) {
 681                 u32 aux_cur = readl_relaxed(base + L2X0_AUX_CTRL);
 682                 u32 acr = get_auxcr();
 683
 684                 pr_debug("Cortex-A9 ACR=0x%08x\n", acr);
 685
 686                 if (acr & BIT(3) && !(aux_cur & L310_AUX_CTRL_FULL_LINE_ZERO))
 687                         pr_err("L2C-310: full line of zeros enabled in Cortex-A9 but not L2C-310 - invalid\n");
 688
 689                 if (aux & L310_AUX_CTRL_FULL_LINE_ZERO && !(acr & BIT(3)))
 690                         pr_err("L2C-310: enabling full line of zeros but not enabled in Cortex-A9\n");
 691
 692                 if (!(aux & L310_AUX_CTRL_FULL_LINE_ZERO) && !outer_cache.write_sec) {
 693                         aux |= L310_AUX_CTRL_FULL_LINE_ZERO;
 694                         pr_info("L2C-310 full line of zeros enabled for Cortex-A9\n");
 695                 }
 696         } else if (aux & (L310_AUX_CTRL_FULL_LINE_ZERO | L310_AUX_CTRL_EARLY_BRESP)) {
 697                 pr_err("L2C-310: disabling Cortex-A9 specific feature bits\n");
 698                 aux &= ~(L310_AUX_CTRL_FULL_LINE_ZERO | L310_AUX_CTRL_EARLY_BRESP);
 699         }
 700
 701         if (aux & (L310_AUX_CTRL_DATA_PREFETCH | L310_AUX_CTRL_INSTR_PREFETCH)) {
 702                 u32 prefetch = readl_relaxed(base + L310_PREFETCH_CTRL);
 703
 704                 pr_info("L2C-310 %s%s prefetch enabled, offset %u lines\n",
 705                         aux & L310_AUX_CTRL_INSTR_PREFETCH ? "I" : "",
 706                         aux & L310_AUX_CTRL_DATA_PREFETCH ? "D" : "",
 707                         1 + (prefetch & L310_PREFETCH_CTRL_OFFSET_MASK));
 708         }
 709
 710         /* r3p0 or later has power control register */
 711         if (rev >= L310_CACHE_ID_RTL_R3P0) {
 712                 u32 power_ctrl;
 713
 714                 l2c_write_sec(L310_DYNAMIC_CLK_GATING_EN | L310_STNDBY_MODE_EN,
 715                               base, L310_POWER_CTRL);
 716                 power_ctrl = readl_relaxed(base + L310_POWER_CTRL);
 717                 pr_info("L2C-310 dynamic clock gating %sabled, standby mode %sabled\n",
 718                         power_ctrl & L310_DYNAMIC_CLK_GATING_EN ? "en" : "dis",
 719                         power_ctrl & L310_STNDBY_MODE_EN ? "en" : "dis");
 720         }
 721
 722         /*
 723          * Always enable non-secure access to the lockdown registers -
 724          * we write to them as part of the L2C enable sequence so they
 725          * need to be accessible.
 726          */
 727         aux |= L310_AUX_CTRL_NS_LOCKDOWN;
 728
 729         l2c_enable(base, aux, num_lock);
 730
 731         if (aux & L310_AUX_CTRL_FULL_LINE_ZERO) {
 732                 set_auxcr(get_auxcr() | BIT(3) | BIT(2) | BIT(1));
 733                 cpu_notifier(l2c310_cpu_enable_flz, 0);
 734         }
 735 }
 736
 737 static void __init l2c310_fixup(void __iomem *base, u32 cache_id,
 738         struct outer_cache_fns *fns)
 739 {
 740         unsigned revision = cache_id & L2X0_CACHE_ID_RTL_MASK;
 741         const char *errata[8];
 742         unsigned n = 0;
 743
 744         if (IS_ENABLED(CONFIG_PL310_ERRATA_588369) &&
 745             revision < L310_CACHE_ID_RTL_R2P0 &&
 746             /* For bcm compatibility */
 747             fns->inv_range == l2c210_inv_range) {
 748                 fns->inv_range = l2c310_inv_range_erratum;
 749                 fns->flush_range = l2c310_flush_range_erratum;
 750                 errata[n++] = "588369";
 751         }
 752
 753         if (IS_ENABLED(CONFIG_PL310_ERRATA_727915) &&
 754             revision >= L310_CACHE_ID_RTL_R2P0 &&
 755             revision < L310_CACHE_ID_RTL_R3P1) {
 756                 fns->flush_all = l2c310_flush_all_erratum;
 757                 errata[n++] = "727915";
 758         }
 759
 760         if (revision >= L310_CACHE_ID_RTL_R3P0 &&
 761             revision < L310_CACHE_ID_RTL_R3P2) {
 762                 u32 val = readl_relaxed(base + L310_PREFETCH_CTRL);
 763                 /* I don't think bit23 is required here... but iMX6 does so */
 764                 if (val & (BIT(30) | BIT(23))) {
 765                         val &= ~(BIT(30) | BIT(23));
 766                         l2c_write_sec(val, base, L310_PREFETCH_CTRL);
 767                         errata[n++] = "752271";
 768                 }
 769         }
 770
 771         if (IS_ENABLED(CONFIG_PL310_ERRATA_753970) &&
 772             revision == L310_CACHE_ID_RTL_R3P0) {
 773                 sync_reg_offset = L2X0_DUMMY_REG;
 774                 errata[n++] = "753970";
 775         }
 776
 777         if (IS_ENABLED(CONFIG_PL310_ERRATA_769419))
 778                 errata[n++] = "769419";
 779
 780         if (n) {
 781                 unsigned i;
 782
 783                 pr_info("L2C-310 errat%s", n > 1 ? "a" : "um");
 784                 for (i = 0; i < n; i++)
 785                         pr_cont(" %s", errata[i]);
 786                 pr_cont(" enabled\n");
 787         }
 788 }
 789
 790 static void l2c310_disable(void)
 791 {
 792         /*
 793          * If full-line-of-zeros is enabled, we must first disable it in the
 794          * Cortex-A9 auxiliary control register before disabling the L2 cache.
 795          */
 796         if (l2x0_saved_regs.aux_ctrl & L310_AUX_CTRL_FULL_LINE_ZERO)
 797                 set_auxcr(get_auxcr() & ~(BIT(3) | BIT(2) | BIT(1)));
 798
 799         l2c_disable();
 800 }
 801
 802 static const struct l2c_init_data l2c310_init_fns __initconst = {
 803         .type = "L2C-310",
 804         .way_size_0 = SZ_8K,
 805         .num_lock = 8,
 806         .enable = l2c310_enable,
 807         .fixup = l2c310_fixup,
 808         .save = l2c310_save,
 809         .outer_cache = {
 810                 .inv_range = l2c210_inv_range,
 811                 .clean_range = l2c210_clean_range,
 812                 .flush_range = l2c210_flush_range,
 813                 .flush_all = l2c210_flush_all,
 814                 .disable = l2c310_disable,
 815                 .sync = l2c210_sync,
 816                 .resume = l2c310_resume,
 817         },
 818 };
 819
 820 static void __init __l2c_init(const struct l2c_init_data *data,
 821         u32 aux_val, u32 aux_mask, u32 cache_id)
 822 {
 823         struct outer_cache_fns fns;
 824         unsigned way_size_bits, ways;
 825         u32 aux, old_aux;
 826
 827         /*
 828          * Sanity check the aux values.  aux_mask is the bits we preserve
 829          * from reading the hardware register, and aux_val is the bits we
 830          * set.
 831          */
 832         if (aux_val & aux_mask)
 833                 pr_alert("L2C: platform provided aux values permit register corruption.\n");
 834
 835         old_aux = aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL);
 836         aux &= aux_mask;
 837         aux |= aux_val;
 838
 839         if (old_aux != aux)
 840                 pr_warn("L2C: DT/platform modifies aux control register: 0x%08x -> 0x%08x\n",
 841                         old_aux, aux);
 842
 843         /* Determine the number of ways */
 844         switch (cache_id & L2X0_CACHE_ID_PART_MASK) {
 845         case L2X0_CACHE_ID_PART_L310:
 846                 if ((aux_val | ~aux_mask) & (L2C_AUX_CTRL_WAY_SIZE_MASK | L310_AUX_CTRL_ASSOCIATIVITY_16))
 847                         pr_warn("L2C: DT/platform tries to modify or specify cache size\n");
 848                 if (aux & (1 << 16))
 849                         ways = 16;
 850                 else
 851                         ways = 8;
 852                 break;
 853
 854         case L2X0_CACHE_ID_PART_L210:
 855         case L2X0_CACHE_ID_PART_L220:
 856                 ways = (aux >> 13) & 0xf;
 857                 break;
 858
 859         case AURORA_CACHE_ID:
 860                 ways = (aux >> 13) & 0xf;
 861                 ways = 2 << ((ways + 1) >> 2);
 862                 break;
 863
 864         default:
 865                 /* Assume unknown chips have 8 ways */
 866                 ways = 8;
 867                 break;
 868         }
 869
 870         l2x0_way_mask = (1 << ways) - 1;
 871
 872         /*
 873          * way_size_0 is the size that a way_size value of zero would be
 874          * given the calculation: way_size = way_size_0 << way_size_bits.
 875          * So, if way_size_bits=0 is reserved, but way_size_bits=1 is 16k,
 876          * then way_size_0 would be 8k.
 877          *
 878          * L2 cache size = number of ways * way size.
 879          */
 880         way_size_bits = (aux & L2C_AUX_CTRL_WAY_SIZE_MASK) >>
 881                         L2C_AUX_CTRL_WAY_SIZE_SHIFT;
 882         l2x0_size = ways * (data->way_size_0 << way_size_bits);
 883
 884         fns = data->outer_cache;
 885         fns.write_sec = outer_cache.write_sec;
 886         if (data->fixup)
 887                 data->fixup(l2x0_base, cache_id, &fns);
 888
 889         /*
 890          * Check if l2x0 controller is already enabled.  If we are booting
 891          * in non-secure mode accessing the below registers will fault.
 892          */
 893         if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN))
 894                 data->enable(l2x0_base, aux, data->num_lock);
 895
 896         outer_cache = fns;
 897
 898         /*
 899          * It is strange to save the register state before initialisation,
 900          * but hey, this is what the DT implementations decided to do.
 901          */
 902         if (data->save)
 903                 data->save(l2x0_base);
 904
 905         /* Re-read it in case some bits are reserved. */
 906         aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL);
 907
 908         pr_info("%s cache controller enabled, %d ways, %d kB\n",
 909                 data->type, ways, l2x0_size >> 10);
 910         pr_info("%s: CACHE_ID 0x%08x, AUX_CTRL 0x%08x\n",
 911                 data->type, cache_id, aux);
 912 }
 913
 914 void __init l2x0_init(void __iomem *base, u32 aux_val, u32 aux_mask)
 915 {
 916         const struct l2c_init_data *data;
 917         u32 cache_id;
 918
 919         l2x0_base = base;
 920
 921         cache_id = readl_relaxed(base + L2X0_CACHE_ID);
 922
 923         switch (cache_id & L2X0_CACHE_ID_PART_MASK) {
 924         default:
 925         case L2X0_CACHE_ID_PART_L210:
 926                 data = &l2c210_data;
 927                 break;
 928
 929         case L2X0_CACHE_ID_PART_L220:
 930                 data = &l2c220_data;
 931                 break;
 932
 933         case L2X0_CACHE_ID_PART_L310:
 934                 data = &l2c310_init_fns;
 935                 break;
 936         }
 937
 938         __l2c_init(data, aux_val, aux_mask, cache_id);
 939 }
 940
 941 #ifdef CONFIG_OF
 942 static int l2_wt_override;
 943
 944 /* Aurora don't have the cache ID register available, so we have to
 945  * pass it though the device tree */
 946 static u32 cache_id_part_number_from_dt;
 947
 948 static void __init l2x0_of_parse(const struct device_node *np,
 949                                  u32 *aux_val, u32 *aux_mask)
 950 {
 951         u32 data[2] = { 0, 0 };
 952         u32 tag = 0;
 953         u32 dirty = 0;
 954         u32 val = 0, mask = 0;
 955
 956         of_property_read_u32(np, "arm,tag-latency", &tag);
 957         if (tag) {
 958                 mask |= L2X0_AUX_CTRL_TAG_LATENCY_MASK;
 959                 val |= (tag - 1) << L2X0_AUX_CTRL_TAG_LATENCY_SHIFT;
 960         }
 961
 962         of_property_read_u32_array(np, "arm,data-latency",
 963                                    data, ARRAY_SIZE(data));
 964         if (data[0] && data[1]) {
 965                 mask |= L2X0_AUX_CTRL_DATA_RD_LATENCY_MASK |
 966                         L2X0_AUX_CTRL_DATA_WR_LATENCY_MASK;
 967                 val |= ((data[0] - 1) << L2X0_AUX_CTRL_DATA_RD_LATENCY_SHIFT) |
 968                        ((data[1] - 1) << L2X0_AUX_CTRL_DATA_WR_LATENCY_SHIFT);
 969         }
 970
 971         of_property_read_u32(np, "arm,dirty-latency", &dirty);
 972         if (dirty) {
 973                 mask |= L2X0_AUX_CTRL_DIRTY_LATENCY_MASK;
 974                 val |= (dirty - 1) << L2X0_AUX_CTRL_DIRTY_LATENCY_SHIFT;
 975         }
 976
 977         *aux_val &= ~mask;
 978         *aux_val |= val;
 979         *aux_mask &= ~mask;
 980 }
 981
 982 static const struct l2c_init_data of_l2c210_data __initconst = {
 983         .type = "L2C-210",
 984         .way_size_0 = SZ_8K,
 985         .num_lock = 1,
 986         .of_parse = l2x0_of_parse,
 987         .enable = l2c_enable,
 988         .save = l2c_save,
 989         .outer_cache = {
 990                 .inv_range   = l2c210_inv_range,
 991                 .clean_range = l2c210_clean_range,
 992                 .flush_range = l2c210_flush_range,
 993                 .flush_all   = l2c210_flush_all,
 994                 .disable     = l2c_disable,
 995                 .sync        = l2c210_sync,
 996                 .resume      = l2c210_resume,
 997         },
 998 };
 999
1000 static const struct l2c_init_data of_l2c220_data __initconst = {
1001         .type = "L2C-220",
1002         .way_size_0 = SZ_8K,
1003         .num_lock = 1,
1004         .of_parse = l2x0_of_parse,
1005         .enable = l2c220_enable,
1006         .save = l2c_save,
1007         .outer_cache = {
1008                 .inv_range   = l2c220_inv_range,
1009                 .clean_range = l2c220_clean_range,
1010                 .flush_range = l2c220_flush_range,
1011                 .flush_all   = l2c220_flush_all,
1012                 .disable     = l2c_disable,
1013                 .sync        = l2c220_sync,
1014                 .resume      = l2c210_resume,
1015         },
1016 };
1017
1018 static void __init l2c310_of_parse(const struct device_node *np,
1019         u32 *aux_val, u32 *aux_mask)
1020 {
1021         u32 data[3] = { 0, 0, 0 };
1022         u32 tag[3] = { 0, 0, 0 };
1023         u32 filter[2] = { 0, 0 };
1024
1025         of_property_read_u32_array(np, "arm,tag-latency", tag, ARRAY_SIZE(tag));
1026         if (tag[0] && tag[1] && tag[2])
1027                 writel_relaxed(
1028                         L310_LATENCY_CTRL_RD(tag[0] - 1) |
1029                         L310_LATENCY_CTRL_WR(tag[1] - 1) |
1030                         L310_LATENCY_CTRL_SETUP(tag[2] - 1),
1031                         l2x0_base + L310_TAG_LATENCY_CTRL);
1032
1033         of_property_read_u32_array(np, "arm,data-latency",
1034                                    data, ARRAY_SIZE(data));
1035         if (data[0] && data[1] && data[2])
1036                 writel_relaxed(
1037                         L310_LATENCY_CTRL_RD(data[0] - 1) |
1038                         L310_LATENCY_CTRL_WR(data[1] - 1) |
1039                         L310_LATENCY_CTRL_SETUP(data[2] - 1),
1040                         l2x0_base + L310_DATA_LATENCY_CTRL);
1041
1042         of_property_read_u32_array(np, "arm,filter-ranges",
1043                                    filter, ARRAY_SIZE(filter));
1044         if (filter[1]) {
1045                 writel_relaxed(ALIGN(filter[0] + filter[1], SZ_1M),
1046                                l2x0_base + L310_ADDR_FILTER_END);
1047                 writel_relaxed((filter[0] & ~(SZ_1M - 1)) | L310_ADDR_FILTER_EN,
1048                                l2x0_base + L310_ADDR_FILTER_START);
1049         }
1050 }
1051
1052 static const struct l2c_init_data of_l2c310_data __initconst = {
1053         .type = "L2C-310",
1054         .way_size_0 = SZ_8K,
1055         .num_lock = 8,
1056         .of_parse = l2c310_of_parse,
1057         .enable = l2c310_enable,
1058         .fixup = l2c310_fixup,
1059         .save  = l2c310_save,
1060         .outer_cache = {
1061                 .inv_range   = l2c210_inv_range,
1062                 .clean_range = l2c210_clean_range,
1063                 .flush_range = l2c210_flush_range,
1064                 .flush_all   = l2c210_flush_all,
1065                 .disable     = l2c310_disable,
1066                 .sync        = l2c210_sync,
1067                 .resume      = l2c310_resume,
1068         },
1069 };
1070
1071 /*
1072  * This is a variant of the of_l2c310_data with .sync set to
1073  * NULL. Outer sync operations are not needed when the system is I/O
1074  * coherent, and potentially harmful in certain situations (PCIe/PL310
1075  * deadlock on Armada 375/38x due to hardware I/O coherency). The
1076  * other operations are kept because they are infrequent (therefore do
1077  * not cause the deadlock in practice) and needed for secondary CPU
1078  * boot and other power management activities.
1079  */
1080 static const struct l2c_init_data of_l2c310_coherent_data __initconst = {
1081         .type = "L2C-310 Coherent",
1082         .way_size_0 = SZ_8K,
1083         .num_lock = 8,
1084         .of_parse = l2c310_of_parse,
1085         .enable = l2c310_enable,
1086         .fixup = l2c310_fixup,
1087         .save  = l2c310_save,
1088         .outer_cache = {
1089                 .inv_range   = l2c210_inv_range,
1090                 .clean_range = l2c210_clean_range,
1091                 .flush_range = l2c210_flush_range,
1092                 .flush_all   = l2c210_flush_all,
1093                 .disable     = l2c310_disable,
1094                 .resume      = l2c310_resume,
1095         },
1096 };
1097
1098 /*
1099  * Note that the end addresses passed to Linux primitives are
1100  * noninclusive, while the hardware cache range operations use
1101  * inclusive start and end addresses.
1102  */
1103 static unsigned long calc_range_end(unsigned long start, unsigned long end)
1104 {
1105         /*
1106          * Limit the number of cache lines processed at once,
1107          * since cache range operations stall the CPU pipeline
1108          * until completion.
1109          */
1110         if (end > start + MAX_RANGE_SIZE)
1111                 end = start + MAX_RANGE_SIZE;
1112
1113         /*
1114          * Cache range operations can't straddle a page boundary.
1115          */
1116         if (end > PAGE_ALIGN(start+1))
1117                 end = PAGE_ALIGN(start+1);
1118
1119         return end;
1120 }
1121
1122 /*
1123  * Make sure 'start' and 'end' reference the same page, as L2 is PIPT
1124  * and range operations only do a TLB lookup on the start address.
1125  */
1126 static void aurora_pa_range(unsigned long start, unsigned long end,
1127                         unsigned long offset)
1128 {
1129         unsigned long flags;
1130
1131         raw_spin_lock_irqsave(&l2x0_lock, flags);
1132         writel_relaxed(start, l2x0_base + AURORA_RANGE_BASE_ADDR_REG);
1133         writel_relaxed(end, l2x0_base + offset);
1134         raw_spin_unlock_irqrestore(&l2x0_lock, flags);
1135
1136         cache_sync();
1137 }
1138
1139 static void aurora_inv_range(unsigned long start, unsigned long end)
1140 {
1141         /*
1142          * round start and end adresses up to cache line size
1143          */
1144         start &= ~(CACHE_LINE_SIZE - 1);
1145         end = ALIGN(end, CACHE_LINE_SIZE);
1146
1147         /*
1148          * Invalidate all full cache lines between 'start' and 'end'.
1149          */
1150         while (start < end) {
1151                 unsigned long range_end = calc_range_end(start, end);
1152                 aurora_pa_range(start, range_end - CACHE_LINE_SIZE,
1153                                 AURORA_INVAL_RANGE_REG);
1154                 start = range_end;
1155         }
1156 }
1157
1158 static void aurora_clean_range(unsigned long start, unsigned long end)
1159 {
1160         /*
1161          * If L2 is forced to WT, the L2 will always be clean and we
1162          * don't need to do anything here.
1163          */
1164         if (!l2_wt_override) {
1165                 start &= ~(CACHE_LINE_SIZE - 1);
1166                 end = ALIGN(end, CACHE_LINE_SIZE);
1167                 while (start != end) {
1168                         unsigned long range_end = calc_range_end(start, end);
1169                         aurora_pa_range(start, range_end - CACHE_LINE_SIZE,
1170                                         AURORA_CLEAN_RANGE_REG);
1171                         start = range_end;
1172                 }
1173         }
1174 }
1175
1176 static void aurora_flush_range(unsigned long start, unsigned long end)
1177 {
1178         start &= ~(CACHE_LINE_SIZE - 1);
1179         end = ALIGN(end, CACHE_LINE_SIZE);
1180         while (start != end) {
1181                 unsigned long range_end = calc_range_end(start, end);
1182                 /*
1183                  * If L2 is forced to WT, the L2 will always be clean and we
1184                  * just need to invalidate.
1185                  */
1186                 if (l2_wt_override)
1187                         aurora_pa_range(start, range_end - CACHE_LINE_SIZE,
1188                                                         AURORA_INVAL_RANGE_REG);
1189                 else
1190                         aurora_pa_range(start, range_end - CACHE_LINE_SIZE,
1191                                                         AURORA_FLUSH_RANGE_REG);
1192                 start = range_end;
1193         }
1194 }
1195
1196 static void aurora_save(void __iomem *base)
1197 {
1198         l2x0_saved_regs.ctrl = readl_relaxed(base + L2X0_CTRL);
1199         l2x0_saved_regs.aux_ctrl = readl_relaxed(base + L2X0_AUX_CTRL);
1200 }
1201
1202 static void aurora_resume(void)
1203 {
1204         void __iomem *base = l2x0_base;
1205
1206         if (!(readl(base + L2X0_CTRL) & L2X0_CTRL_EN)) {
1207                 writel_relaxed(l2x0_saved_regs.aux_ctrl, base + L2X0_AUX_CTRL);
1208                 writel_relaxed(l2x0_saved_regs.ctrl, base + L2X0_CTRL);
1209         }
1210 }
1211
1212 /*
1213  * For Aurora cache in no outer mode, enable via the CP15 coprocessor
1214  * broadcasting of cache commands to L2.
1215  */
1216 static void __init aurora_enable_no_outer(void __iomem *base, u32 aux,
1217         unsigned num_lock)
1218 {
1219         u32 u;
1220
1221         asm volatile("mrc p15, 1, %0, c15, c2, 0" : "=r" (u));
1222         u |= AURORA_CTRL_FW;            /* Set the FW bit */
1223         asm volatile("mcr p15, 1, %0, c15, c2, 0" : : "r" (u));
1224
1225         isb();
1226
1227         l2c_enable(base, aux, num_lock);
1228 }
1229
1230 static void __init aurora_fixup(void __iomem *base, u32 cache_id,
1231         struct outer_cache_fns *fns)
1232 {
1233         sync_reg_offset = AURORA_SYNC_REG;
1234 }
1235
1236 static void __init aurora_of_parse(const struct device_node *np,
1237                                 u32 *aux_val, u32 *aux_mask)
1238 {
1239         u32 val = AURORA_ACR_REPLACEMENT_TYPE_SEMIPLRU;
1240         u32 mask =  AURORA_ACR_REPLACEMENT_MASK;
1241
1242         of_property_read_u32(np, "cache-id-part",
1243                         &cache_id_part_number_from_dt);
1244
1245         /* Determine and save the write policy */
1246         l2_wt_override = of_property_read_bool(np, "wt-override");
1247
1248         if (l2_wt_override) {
1249                 val |= AURORA_ACR_FORCE_WRITE_THRO_POLICY;
1250                 mask |= AURORA_ACR_FORCE_WRITE_POLICY_MASK;
1251         }
1252
1253         *aux_val &= ~mask;
1254         *aux_val |= val;
1255         *aux_mask &= ~mask;
1256 }
1257
1258 static const struct l2c_init_data of_aurora_with_outer_data __initconst = {
1259         .type = "Aurora",
1260         .way_size_0 = SZ_4K,
1261         .num_lock = 4,
1262         .of_parse = aurora_of_parse,
1263         .enable = l2c_enable,
1264         .fixup = aurora_fixup,
1265         .save  = aurora_save,
1266         .outer_cache = {
1267                 .inv_range   = aurora_inv_range,
1268                 .clean_range = aurora_clean_range,
1269                 .flush_range = aurora_flush_range,
1270                 .flush_all   = l2x0_flush_all,
1271                 .disable     = l2x0_disable,
1272                 .sync        = l2x0_cache_sync,
1273                 .resume      = aurora_resume,
1274         },
1275 };
1276
1277 static const struct l2c_init_data of_aurora_no_outer_data __initconst = {
1278         .type = "Aurora",
1279         .way_size_0 = SZ_4K,
1280         .num_lock = 4,
1281         .of_parse = aurora_of_parse,
1282         .enable = aurora_enable_no_outer,
1283         .fixup = aurora_fixup,
1284         .save  = aurora_save,
1285         .outer_cache = {
1286                 .resume      = aurora_resume,
1287         },
1288 };
1289
1290 /*
1291  * For certain Broadcom SoCs, depending on the address range, different offsets
1292  * need to be added to the address before passing it to L2 for
1293  * invalidation/clean/flush
1294  *
1295  * Section Address Range              Offset        EMI
1296  *   1     0x00000000 - 0x3FFFFFFF    0x80000000    VC
1297  *   2     0x40000000 - 0xBFFFFFFF    0x40000000    SYS
1298  *   3     0xC0000000 - 0xFFFFFFFF    0x80000000    VC
1299  *
1300  * When the start and end addresses have crossed two different sections, we
1301  * need to break the L2 operation into two, each within its own section.
1302  * For example, if we need to invalidate addresses starts at 0xBFFF0000 and
1303  * ends at 0xC0001000, we need do invalidate 1) 0xBFFF0000 - 0xBFFFFFFF and 2)
1304  * 0xC0000000 - 0xC0001000
1305  *
1306  * Note 1:
1307  * By breaking a single L2 operation into two, we may potentially suffer some
1308  * performance hit, but keep in mind the cross section case is very rare
1309  *
1310  * Note 2:
1311  * We do not need to handle the case when the start address is in
1312  * Section 1 and the end address is in Section 3, since it is not a valid use
1313  * case
1314  *
1315  * Note 3:
1316  * Section 1 in practical terms can no longer be used on rev A2. Because of
1317  * that the code does not need to handle section 1 at all.
1318  *
1319  */
1320 #define BCM_SYS_EMI_START_ADDR        0x40000000UL
1321 #define BCM_VC_EMI_SEC3_START_ADDR    0xC0000000UL
1322
1323 #define BCM_SYS_EMI_OFFSET            0x40000000UL
1324 #define BCM_VC_EMI_OFFSET             0x80000000UL
1325
1326 static inline int bcm_addr_is_sys_emi(unsigned long addr)
1327 {
1328         return (addr >= BCM_SYS_EMI_START_ADDR) &&
1329                 (addr < BCM_VC_EMI_SEC3_START_ADDR);
1330 }
1331
1332 static inline unsigned long bcm_l2_phys_addr(unsigned long addr)
1333 {
1334         if (bcm_addr_is_sys_emi(addr))
1335                 return addr + BCM_SYS_EMI_OFFSET;
1336         else
1337                 return addr + BCM_VC_EMI_OFFSET;
1338 }
1339
1340 static void bcm_inv_range(unsigned long start, unsigned long end)
1341 {
1342         unsigned long new_start, new_end;
1343
1344         BUG_ON(start < BCM_SYS_EMI_START_ADDR);
1345
1346         if (unlikely(end <= start))
1347                 return;
1348
1349         new_start = bcm_l2_phys_addr(start);
1350         new_end = bcm_l2_phys_addr(end);
1351
1352         /* normal case, no cross section between start and end */
1353         if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) {
1354                 l2c210_inv_range(new_start, new_end);
1355                 return;
1356         }
1357
1358         /* They cross sections, so it can only be a cross from section
1359          * 2 to section 3
1360          */
1361         l2c210_inv_range(new_start,
1362                 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1));
1363         l2c210_inv_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR),
1364                 new_end);
1365 }
1366
1367 static void bcm_clean_range(unsigned long start, unsigned long end)
1368 {
1369         unsigned long new_start, new_end;
1370
1371         BUG_ON(start < BCM_SYS_EMI_START_ADDR);
1372
1373         if (unlikely(end <= start))
1374                 return;
1375
1376         new_start = bcm_l2_phys_addr(start);
1377         new_end = bcm_l2_phys_addr(end);
1378
1379         /* normal case, no cross section between start and end */
1380         if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) {
1381                 l2c210_clean_range(new_start, new_end);
1382                 return;
1383         }
1384
1385         /* They cross sections, so it can only be a cross from section
1386          * 2 to section 3
1387          */
1388         l2c210_clean_range(new_start,
1389                 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1));
1390         l2c210_clean_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR),
1391                 new_end);
1392 }
1393
1394 static void bcm_flush_range(unsigned long start, unsigned long end)
1395 {
1396         unsigned long new_start, new_end;
1397
1398         BUG_ON(start < BCM_SYS_EMI_START_ADDR);
1399
1400         if (unlikely(end <= start))
1401                 return;
1402
1403         if ((end - start) >= l2x0_size) {
1404                 outer_cache.flush_all();
1405                 return;
1406         }
1407
1408         new_start = bcm_l2_phys_addr(start);
1409         new_end = bcm_l2_phys_addr(end);
1410
1411         /* normal case, no cross section between start and end */
1412         if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) {
1413                 l2c210_flush_range(new_start, new_end);
1414                 return;
1415         }
1416
1417         /* They cross sections, so it can only be a cross from section
1418          * 2 to section 3
1419          */
1420         l2c210_flush_range(new_start,
1421                 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1));
1422         l2c210_flush_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR),
1423                 new_end);
1424 }
1425
1426 /* Broadcom L2C-310 start from ARMs R3P2 or later, and require no fixups */
1427 static const struct l2c_init_data of_bcm_l2x0_data __initconst = {
1428         .type = "BCM-L2C-310",
1429         .way_size_0 = SZ_8K,
1430         .num_lock = 8,
1431         .of_parse = l2c310_of_parse,
1432         .enable = l2c310_enable,
1433         .save  = l2c310_save,
1434         .outer_cache = {
1435                 .inv_range   = bcm_inv_range,
1436                 .clean_range = bcm_clean_range,
1437                 .flush_range = bcm_flush_range,
1438                 .flush_all   = l2c210_flush_all,
1439                 .disable     = l2c310_disable,
1440                 .sync        = l2c210_sync,
1441                 .resume      = l2c310_resume,
1442         },
1443 };
1444
1445 static void __init tauros3_save(void __iomem *base)
1446 {
1447         l2c_save(base);
1448
1449         l2x0_saved_regs.aux2_ctrl =
1450                 readl_relaxed(base + TAUROS3_AUX2_CTRL);
1451         l2x0_saved_regs.prefetch_ctrl =
1452                 readl_relaxed(base + L310_PREFETCH_CTRL);
1453 }
1454
1455 static void tauros3_resume(void)
1456 {
1457         void __iomem *base = l2x0_base;
1458
1459         if (!(readl_relaxed(base + L2X0_CTRL) & L2X0_CTRL_EN)) {
1460                 writel_relaxed(l2x0_saved_regs.aux2_ctrl,
1461                                base + TAUROS3_AUX2_CTRL);
1462                 writel_relaxed(l2x0_saved_regs.prefetch_ctrl,
1463                                base + L310_PREFETCH_CTRL);
1464
1465                 l2c_enable(base, l2x0_saved_regs.aux_ctrl, 8);
1466         }
1467 }
1468
1469 static const struct l2c_init_data of_tauros3_data __initconst = {
1470         .type = "Tauros3",
1471         .way_size_0 = SZ_8K,
1472         .num_lock = 8,
1473         .enable = l2c_enable,
1474         .save  = tauros3_save,
1475         /* Tauros3 broadcasts L1 cache operations to L2 */
1476         .outer_cache = {
1477                 .resume      = tauros3_resume,
1478         },
1479 };
1480
1481 #define L2C_ID(name, fns) { .compatible = name, .data = (void *)&fns }
1482 static const struct of_device_id l2x0_ids[] __initconst = {
1483         L2C_ID("arm,l210-cache", of_l2c210_data),
1484         L2C_ID("arm,l220-cache", of_l2c220_data),
1485         L2C_ID("arm,pl310-cache", of_l2c310_data),
1486         L2C_ID("brcm,bcm11351-a2-pl310-cache", of_bcm_l2x0_data),
1487         L2C_ID("marvell,aurora-outer-cache", of_aurora_with_outer_data),
1488         L2C_ID("marvell,aurora-system-cache", of_aurora_no_outer_data),
1489         L2C_ID("marvell,tauros3-cache", of_tauros3_data),
1490         /* Deprecated IDs */
1491         L2C_ID("bcm,bcm11351-a2-pl310-cache", of_bcm_l2x0_data),
1492         {}
1493 };
1494
1495 int __init l2x0_of_init(u32 aux_val, u32 aux_mask)
1496 {
1497         const struct l2c_init_data *data;
1498         struct device_node *np;
1499         struct resource res;
1500         u32 cache_id, old_aux;
1501
1502         np = of_find_matching_node(NULL, l2x0_ids);
1503         if (!np)
1504                 return -ENODEV;
1505
1506         if (of_address_to_resource(np, 0, &res))
1507                 return -ENODEV;
1508
1509         l2x0_base = ioremap(res.start, resource_size(&res));
1510         if (!l2x0_base)
1511                 return -ENOMEM;
1512
1513         l2x0_saved_regs.phy_base = res.start;
1514
1515         data = of_match_node(l2x0_ids, np)->data;
1516
1517         if (of_device_is_compatible(np, "arm,pl310-cache") &&
1518             of_property_read_bool(np, "arm,io-coherent"))
1519                 data = &of_l2c310_coherent_data;
1520
1521         old_aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL);
1522         if (old_aux != ((old_aux & aux_mask) | aux_val)) {
1523                 pr_warn("L2C: platform modifies aux control register: 0x%08x -> 0x%08x\n",
1524                         old_aux, (old_aux & aux_mask) | aux_val);
1525         } else if (aux_mask != ~0U && aux_val != 0) {
1526                 pr_alert("L2C: platform provided aux values match the hardware, so have no effect.  Please remove them.\n");
1527         }
1528
1529         /* All L2 caches are unified, so this property should be specified */
1530         if (!of_property_read_bool(np, "cache-unified"))
1531                 pr_err("L2C: device tree omits to specify unified cache\n");
1532
1533         /* L2 configuration can only be changed if the cache is disabled */
1534         if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN))
1535                 if (data->of_parse)
1536                         data->of_parse(np, &aux_val, &aux_mask);
1537
1538         if (cache_id_part_number_from_dt)
1539                 cache_id = cache_id_part_number_from_dt;
1540         else
1541                 cache_id = readl_relaxed(l2x0_base + L2X0_CACHE_ID);
1542
1543         __l2c_init(data, aux_val, aux_mask, cache_id);
1544
1545         return 0;
1546 }
1547 #endif