arch/mips/cavium-octeon/executive/cvmx-l2c.c

   1 /***********************license start***************
   2  * Author: Cavium Networks
   3  *
   4  * Contact: support@caviumnetworks.com
   5  * This file is part of the OCTEON SDK
   6  *
   7  * Copyright (c) 2003-2010 Cavium Networks
   8  *
   9  * This file is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License, Version 2, as
  11  * published by the Free Software Foundation.
  12  *
  13  * This file is distributed in the hope that it will be useful, but
  14  * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
  15  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
  16  * NONINFRINGEMENT.  See the GNU General Public License for more
  17  * details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this file; if not, write to the Free Software
  21  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  22  * or visit http://www.gnu.org/licenses/.
  23  *
  24  * This file may also be available under a different license from Cavium.
  25  * Contact Cavium Networks for more information
  26  ***********************license end**************************************/
  27
  28 /*
  29  * Implementation of the Level 2 Cache (L2C) control,
  30  * measurement, and debugging facilities.
  31  */
  32
  33 #include <asm/octeon/cvmx.h>
  34 #include <asm/octeon/cvmx-l2c.h>
  35 #include <asm/octeon/cvmx-spinlock.h>
  36
  37 /*
  38  * This spinlock is used internally to ensure that only one core is
  39  * performing certain L2 operations at a time.
  40  *
  41  * NOTE: This only protects calls from within a single application -
  42  * if multiple applications or operating systems are running, then it
  43  * is up to the user program to coordinate between them.
  44  */
  45 cvmx_spinlock_t cvmx_l2c_spinlock;
  46
  47 int cvmx_l2c_get_core_way_partition(uint32_t core)
  48 {
  49         uint32_t field;
  50
  51         /* Validate the core number */
  52         if (core >= cvmx_octeon_num_cores())
  53                 return -1;
  54
  55         if (OCTEON_IS_MODEL(OCTEON_CN63XX))
  56                 return cvmx_read_csr(CVMX_L2C_WPAR_PPX(core)) & 0xffff;
  57
  58         /*
  59          * Use the lower two bits of the coreNumber to determine the
  60          * bit offset of the UMSK[] field in the L2C_SPAR register.
  61          */
  62         field = (core & 0x3) * 8;
  63
  64         /*
  65          * Return the UMSK[] field from the appropriate L2C_SPAR
  66          * register based on the coreNumber.
  67          */
  68
  69         switch (core & 0xC) {
  70         case 0x0:
  71                 return (cvmx_read_csr(CVMX_L2C_SPAR0) & (0xFF << field)) >> field;
  72         case 0x4:
  73                 return (cvmx_read_csr(CVMX_L2C_SPAR1) & (0xFF << field)) >> field;
  74         case 0x8:
  75                 return (cvmx_read_csr(CVMX_L2C_SPAR2) & (0xFF << field)) >> field;
  76         case 0xC:
  77                 return (cvmx_read_csr(CVMX_L2C_SPAR3) & (0xFF << field)) >> field;
  78         }
  79         return 0;
  80 }
  81
  82 int cvmx_l2c_set_core_way_partition(uint32_t core, uint32_t mask)
  83 {
  84         uint32_t field;
  85         uint32_t valid_mask;
  86
  87         valid_mask = (0x1 << cvmx_l2c_get_num_assoc()) - 1;
  88
  89         mask &= valid_mask;
  90
  91         /* A UMSK setting which blocks all L2C Ways is an error on some chips */
  92         if (mask == valid_mask && !OCTEON_IS_MODEL(OCTEON_CN63XX))
  93                 return -1;
  94
  95         /* Validate the core number */
  96         if (core >= cvmx_octeon_num_cores())
  97                 return -1;
  98
  99         if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
 100                 cvmx_write_csr(CVMX_L2C_WPAR_PPX(core), mask);
 101                 return 0;
 102         }
 103
 104         /*
 105          * Use the lower two bits of core to determine the bit offset of the
 106          * UMSK[] field in the L2C_SPAR register.
 107          */
 108         field = (core & 0x3) * 8;
 109
 110         /*
 111          * Assign the new mask setting to the UMSK[] field in the appropriate
 112          * L2C_SPAR register based on the core_num.
 113          *
 114          */
 115         switch (core & 0xC) {
 116         case 0x0:
 117                 cvmx_write_csr(CVMX_L2C_SPAR0,
 118                                (cvmx_read_csr(CVMX_L2C_SPAR0) & ~(0xFF << field)) |
 119                                mask << field);
 120                 break;
 121         case 0x4:
 122                 cvmx_write_csr(CVMX_L2C_SPAR1,
 123                                (cvmx_read_csr(CVMX_L2C_SPAR1) & ~(0xFF << field)) |
 124                                mask << field);
 125                 break;
 126         case 0x8:
 127                 cvmx_write_csr(CVMX_L2C_SPAR2,
 128                                (cvmx_read_csr(CVMX_L2C_SPAR2) & ~(0xFF << field)) |
 129                                mask << field);
 130                 break;
 131         case 0xC:
 132                 cvmx_write_csr(CVMX_L2C_SPAR3,
 133                                (cvmx_read_csr(CVMX_L2C_SPAR3) & ~(0xFF << field)) |
 134                                mask << field);
 135                 break;
 136         }
 137         return 0;
 138 }
 139
 140 int cvmx_l2c_set_hw_way_partition(uint32_t mask)
 141 {
 142         uint32_t valid_mask;
 143
 144         valid_mask = (0x1 << cvmx_l2c_get_num_assoc()) - 1;
 145         mask &= valid_mask;
 146
 147         /* A UMSK setting which blocks all L2C Ways is an error on some chips */
 148         if (mask == valid_mask  && !OCTEON_IS_MODEL(OCTEON_CN63XX))
 149                 return -1;
 150
 151         if (OCTEON_IS_MODEL(OCTEON_CN63XX))
 152                 cvmx_write_csr(CVMX_L2C_WPAR_IOBX(0), mask);
 153         else
 154                 cvmx_write_csr(CVMX_L2C_SPAR4,
 155                                (cvmx_read_csr(CVMX_L2C_SPAR4) & ~0xFF) | mask);
 156         return 0;
 157 }
 158
 159 int cvmx_l2c_get_hw_way_partition(void)
 160 {
 161         if (OCTEON_IS_MODEL(OCTEON_CN63XX))
 162                 return cvmx_read_csr(CVMX_L2C_WPAR_IOBX(0)) & 0xffff;
 163         else
 164                 return cvmx_read_csr(CVMX_L2C_SPAR4) & (0xFF);
 165 }
 166
 167 void cvmx_l2c_config_perf(uint32_t counter, enum cvmx_l2c_event event,
 168                           uint32_t clear_on_read)
 169 {
 170         if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX)) {
 171                 union cvmx_l2c_pfctl pfctl;
 172
 173                 pfctl.u64 = cvmx_read_csr(CVMX_L2C_PFCTL);
 174
 175                 switch (counter) {
 176                 case 0:
 177                         pfctl.s.cnt0sel = event;
 178                         pfctl.s.cnt0ena = 1;
 179                         pfctl.s.cnt0rdclr = clear_on_read;
 180                         break;
 181                 case 1:
 182                         pfctl.s.cnt1sel = event;
 183                         pfctl.s.cnt1ena = 1;
 184                         pfctl.s.cnt1rdclr = clear_on_read;
 185                         break;
 186                 case 2:
 187                         pfctl.s.cnt2sel = event;
 188                         pfctl.s.cnt2ena = 1;
 189                         pfctl.s.cnt2rdclr = clear_on_read;
 190                         break;
 191                 case 3:
 192                 default:
 193                         pfctl.s.cnt3sel = event;
 194                         pfctl.s.cnt3ena = 1;
 195                         pfctl.s.cnt3rdclr = clear_on_read;
 196                         break;
 197                 }
 198
 199                 cvmx_write_csr(CVMX_L2C_PFCTL, pfctl.u64);
 200         } else {
 201                 union cvmx_l2c_tadx_prf l2c_tadx_prf;
 202                 int tad;
 203
 204                 cvmx_dprintf("L2C performance counter events are different for this chip, mapping 'event' to cvmx_l2c_tad_event_t\n");
 205                 if (clear_on_read)
 206                         cvmx_dprintf("L2C counters don't support clear on read for this chip\n");
 207
 208                 l2c_tadx_prf.u64 = cvmx_read_csr(CVMX_L2C_TADX_PRF(0));
 209
 210                 switch (counter) {
 211                 case 0:
 212                         l2c_tadx_prf.s.cnt0sel = event;
 213                         break;
 214                 case 1:
 215                         l2c_tadx_prf.s.cnt1sel = event;
 216                         break;
 217                 case 2:
 218                         l2c_tadx_prf.s.cnt2sel = event;
 219                         break;
 220                 default:
 221                 case 3:
 222                         l2c_tadx_prf.s.cnt3sel = event;
 223                         break;
 224                 }
 225                 for (tad = 0; tad < CVMX_L2C_TADS; tad++)
 226                         cvmx_write_csr(CVMX_L2C_TADX_PRF(tad),
 227                                        l2c_tadx_prf.u64);
 228         }
 229 }
 230
 231 uint64_t cvmx_l2c_read_perf(uint32_t counter)
 232 {
 233         switch (counter) {
 234         case 0:
 235                 if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX))
 236                         return cvmx_read_csr(CVMX_L2C_PFC0);
 237                 else {
 238                         uint64_t counter = 0;
 239                         int tad;
 240                         for (tad = 0; tad < CVMX_L2C_TADS; tad++)
 241                                 counter += cvmx_read_csr(CVMX_L2C_TADX_PFC0(tad));
 242                         return counter;
 243                 }
 244         case 1:
 245                 if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX))
 246                         return cvmx_read_csr(CVMX_L2C_PFC1);
 247                 else {
 248                         uint64_t counter = 0;
 249                         int tad;
 250                         for (tad = 0; tad < CVMX_L2C_TADS; tad++)
 251                                 counter += cvmx_read_csr(CVMX_L2C_TADX_PFC1(tad));
 252                         return counter;
 253                 }
 254         case 2:
 255                 if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX))
 256                         return cvmx_read_csr(CVMX_L2C_PFC2);
 257                 else {
 258                         uint64_t counter = 0;
 259                         int tad;
 260                         for (tad = 0; tad < CVMX_L2C_TADS; tad++)
 261                                 counter += cvmx_read_csr(CVMX_L2C_TADX_PFC2(tad));
 262                         return counter;
 263                 }
 264         case 3:
 265         default:
 266                 if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX))
 267                         return cvmx_read_csr(CVMX_L2C_PFC3);
 268                 else {
 269                         uint64_t counter = 0;
 270                         int tad;
 271                         for (tad = 0; tad < CVMX_L2C_TADS; tad++)
 272                                 counter += cvmx_read_csr(CVMX_L2C_TADX_PFC3(tad));
 273                         return counter;
 274                 }
 275         }
 276 }
 277
 278 /**
 279  * @INTERNAL
 280  * Helper function use to fault in cache lines for L2 cache locking
 281  *
 282  * @addr:   Address of base of memory region to read into L2 cache
 283  * @len:    Length (in bytes) of region to fault in
 284  */
 285 static void fault_in(uint64_t addr, int len)
 286 {
 287         volatile char *ptr;
 288         volatile char dummy;
 289         /*
 290          * Adjust addr and length so we get all cache lines even for
 291          * small ranges spanning two cache lines.
 292          */
 293         len += addr & CVMX_CACHE_LINE_MASK;
 294         addr &= ~CVMX_CACHE_LINE_MASK;
 295         ptr = (volatile char *)cvmx_phys_to_ptr(addr);
 296         /*
 297          * Invalidate L1 cache to make sure all loads result in data
 298          * being in L2.
 299          */
 300         CVMX_DCACHE_INVALIDATE;
 301         while (len > 0) {
 302                 dummy += *ptr;
 303                 len -= CVMX_CACHE_LINE_SIZE;
 304                 ptr += CVMX_CACHE_LINE_SIZE;
 305         }
 306 }
 307
 308 int cvmx_l2c_lock_line(uint64_t addr)
 309 {
 310         if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
 311                 int shift = CVMX_L2C_TAG_ADDR_ALIAS_SHIFT;
 312                 uint64_t assoc = cvmx_l2c_get_num_assoc();
 313                 uint64_t tag = addr >> shift;
 314                 uint64_t index = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, cvmx_l2c_address_to_index(addr) << CVMX_L2C_IDX_ADDR_SHIFT);
 315                 uint64_t way;
 316                 union cvmx_l2c_tadx_tag l2c_tadx_tag;
 317
 318                 CVMX_CACHE_LCKL2(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, addr), 0);
 319
 320                 /* Make sure we were able to lock the line */
 321                 for (way = 0; way < assoc; way++) {
 322                         CVMX_CACHE_LTGL2I(index | (way << shift), 0);
 323                         /* make sure CVMX_L2C_TADX_TAG is updated */
 324                         CVMX_SYNC;
 325                         l2c_tadx_tag.u64 = cvmx_read_csr(CVMX_L2C_TADX_TAG(0));
 326                         if (l2c_tadx_tag.s.valid && l2c_tadx_tag.s.tag == tag)
 327                                 break;
 328                 }
 329
 330                 /* Check if a valid line is found */
 331                 if (way >= assoc) {
 332                         /* cvmx_dprintf("ERROR: cvmx_l2c_lock_line: line not found for locking at 0x%llx address\n", (unsigned long long)addr); */
 333                         return -1;
 334                 }
 335
 336                 /* Check if lock bit is not set */
 337                 if (!l2c_tadx_tag.s.lock) {
 338                         /* cvmx_dprintf("ERROR: cvmx_l2c_lock_line: Not able to lock at 0x%llx address\n", (unsigned long long)addr); */
 339                         return -1;
 340                 }
 341                 return way;
 342         } else {
 343                 int retval = 0;
 344                 union cvmx_l2c_dbg l2cdbg;
 345                 union cvmx_l2c_lckbase lckbase;
 346                 union cvmx_l2c_lckoff lckoff;
 347                 union cvmx_l2t_err l2t_err;
 348
 349                 cvmx_spinlock_lock(&cvmx_l2c_spinlock);
 350
 351                 l2cdbg.u64 = 0;
 352                 lckbase.u64 = 0;
 353                 lckoff.u64 = 0;
 354
 355                 /* Clear l2t error bits if set */
 356                 l2t_err.u64 = cvmx_read_csr(CVMX_L2T_ERR);
 357                 l2t_err.s.lckerr = 1;
 358                 l2t_err.s.lckerr2 = 1;
 359                 cvmx_write_csr(CVMX_L2T_ERR, l2t_err.u64);
 360
 361                 addr &= ~CVMX_CACHE_LINE_MASK;
 362
 363                 /* Set this core as debug core */
 364                 l2cdbg.s.ppnum = cvmx_get_core_num();
 365                 CVMX_SYNC;
 366                 cvmx_write_csr(CVMX_L2C_DBG, l2cdbg.u64);
 367                 cvmx_read_csr(CVMX_L2C_DBG);
 368
 369                 lckoff.s.lck_offset = 0; /* Only lock 1 line at a time */
 370                 cvmx_write_csr(CVMX_L2C_LCKOFF, lckoff.u64);
 371                 cvmx_read_csr(CVMX_L2C_LCKOFF);
 372
 373                 if (((union cvmx_l2c_cfg)(cvmx_read_csr(CVMX_L2C_CFG))).s.idxalias) {
 374                         int alias_shift = CVMX_L2C_IDX_ADDR_SHIFT + 2 * CVMX_L2_SET_BITS - 1;
 375                         uint64_t addr_tmp = addr ^ (addr & ((1 << alias_shift) - 1)) >> CVMX_L2_SET_BITS;
 376                         lckbase.s.lck_base = addr_tmp >> 7;
 377                 } else {
 378                         lckbase.s.lck_base = addr >> 7;
 379                 }
 380
 381                 lckbase.s.lck_ena = 1;
 382                 cvmx_write_csr(CVMX_L2C_LCKBASE, lckbase.u64);
 383                 /* Make sure it gets there */
 384                 cvmx_read_csr(CVMX_L2C_LCKBASE);
 385
 386                 fault_in(addr, CVMX_CACHE_LINE_SIZE);
 387
 388                 lckbase.s.lck_ena = 0;
 389                 cvmx_write_csr(CVMX_L2C_LCKBASE, lckbase.u64);
 390                 /* Make sure it gets there */
 391                 cvmx_read_csr(CVMX_L2C_LCKBASE);
 392
 393                 /* Stop being debug core */
 394                 cvmx_write_csr(CVMX_L2C_DBG, 0);
 395                 cvmx_read_csr(CVMX_L2C_DBG);
 396
 397                 l2t_err.u64 = cvmx_read_csr(CVMX_L2T_ERR);
 398                 if (l2t_err.s.lckerr || l2t_err.s.lckerr2)
 399                         retval = 1;  /* We were unable to lock the line */
 400
 401                 cvmx_spinlock_unlock(&cvmx_l2c_spinlock);
 402                 return retval;
 403         }
 404 }
 405
 406 int cvmx_l2c_lock_mem_region(uint64_t start, uint64_t len)
 407 {
 408         int retval = 0;
 409
 410         /* Round start/end to cache line boundaries */
 411         len += start & CVMX_CACHE_LINE_MASK;
 412         start &= ~CVMX_CACHE_LINE_MASK;
 413         len = (len + CVMX_CACHE_LINE_MASK) & ~CVMX_CACHE_LINE_MASK;
 414
 415         while (len) {
 416                 retval += cvmx_l2c_lock_line(start);
 417                 start += CVMX_CACHE_LINE_SIZE;
 418                 len -= CVMX_CACHE_LINE_SIZE;
 419         }
 420         return retval;
 421 }
 422
 423 void cvmx_l2c_flush(void)
 424 {
 425         uint64_t assoc, set;
 426         uint64_t n_assoc, n_set;
 427
 428         n_set = cvmx_l2c_get_num_sets();
 429         n_assoc = cvmx_l2c_get_num_assoc();
 430
 431         if (OCTEON_IS_MODEL(OCTEON_CN6XXX)) {
 432                 uint64_t address;
 433                 /* These may look like constants, but they aren't... */
 434                 int assoc_shift = CVMX_L2C_TAG_ADDR_ALIAS_SHIFT;
 435                 int set_shift = CVMX_L2C_IDX_ADDR_SHIFT;
 436                 for (set = 0; set < n_set; set++) {
 437                         for (assoc = 0; assoc < n_assoc; assoc++) {
 438                                 address = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
 439                                                        (assoc << assoc_shift) | (set << set_shift));
 440                                 CVMX_CACHE_WBIL2I(address, 0);
 441                         }
 442                 }
 443         } else {
 444                 for (set = 0; set < n_set; set++)
 445                         for (assoc = 0; assoc < n_assoc; assoc++)
 446                                 cvmx_l2c_flush_line(assoc, set);
 447         }
 448 }
 449
 450
 451 int cvmx_l2c_unlock_line(uint64_t address)
 452 {
 453
 454         if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
 455                 int assoc;
 456                 union cvmx_l2c_tag tag;
 457                 uint32_t tag_addr;
 458                 uint32_t index = cvmx_l2c_address_to_index(address);
 459
 460                 tag_addr = ((address >> CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) & ((1 << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) - 1));
 461
 462                 /*
 463                  * For 63XX, we can flush a line by using the physical
 464                  * address directly, so finding the cache line used by
 465                  * the address is only required to provide the proper
 466                  * return value for the function.
 467                  */
 468                 for (assoc = 0; assoc < CVMX_L2_ASSOC; assoc++) {
 469                         tag = cvmx_l2c_get_tag(assoc, index);
 470
 471                         if (tag.s.V && (tag.s.addr == tag_addr)) {
 472                                 CVMX_CACHE_WBIL2(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, address), 0);
 473                                 return tag.s.L;
 474                         }
 475                 }
 476         } else {
 477                 int assoc;
 478                 union cvmx_l2c_tag tag;
 479                 uint32_t tag_addr;
 480
 481                 uint32_t index = cvmx_l2c_address_to_index(address);
 482
 483                 /* Compute portion of address that is stored in tag */
 484                 tag_addr = ((address >> CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) & ((1 << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) - 1));
 485                 for (assoc = 0; assoc < CVMX_L2_ASSOC; assoc++) {
 486                         tag = cvmx_l2c_get_tag(assoc, index);
 487
 488                         if (tag.s.V && (tag.s.addr == tag_addr)) {
 489                                 cvmx_l2c_flush_line(assoc, index);
 490                                 return tag.s.L;
 491                         }
 492                 }
 493         }
 494         return 0;
 495 }
 496
 497 int cvmx_l2c_unlock_mem_region(uint64_t start, uint64_t len)
 498 {
 499         int num_unlocked = 0;
 500         /* Round start/end to cache line boundaries */
 501         len += start & CVMX_CACHE_LINE_MASK;
 502         start &= ~CVMX_CACHE_LINE_MASK;
 503         len = (len + CVMX_CACHE_LINE_MASK) & ~CVMX_CACHE_LINE_MASK;
 504         while (len > 0) {
 505                 num_unlocked += cvmx_l2c_unlock_line(start);
 506                 start += CVMX_CACHE_LINE_SIZE;
 507                 len -= CVMX_CACHE_LINE_SIZE;
 508         }
 509
 510         return num_unlocked;
 511 }
 512
 513 /*
 514  * Internal l2c tag types.  These are converted to a generic structure
 515  * that can be used on all chips.
 516  */
 517 union __cvmx_l2c_tag {
 518         uint64_t u64;
 519         struct cvmx_l2c_tag_cn50xx {
 520                 uint64_t reserved:40;
 521                 uint64_t V:1;           /* Line valid */
 522                 uint64_t D:1;           /* Line dirty */
 523                 uint64_t L:1;           /* Line locked */
 524                 uint64_t U:1;           /* Use, LRU eviction */
 525                 uint64_t addr:20;       /* Phys mem addr (33..14) */
 526         } cn50xx;
 527         struct cvmx_l2c_tag_cn30xx {
 528                 uint64_t reserved:41;
 529                 uint64_t V:1;           /* Line valid */
 530                 uint64_t D:1;           /* Line dirty */
 531                 uint64_t L:1;           /* Line locked */
 532                 uint64_t U:1;           /* Use, LRU eviction */
 533                 uint64_t addr:19;       /* Phys mem addr (33..15) */
 534         } cn30xx;
 535         struct cvmx_l2c_tag_cn31xx {
 536                 uint64_t reserved:42;
 537                 uint64_t V:1;           /* Line valid */
 538                 uint64_t D:1;           /* Line dirty */
 539                 uint64_t L:1;           /* Line locked */
 540                 uint64_t U:1;           /* Use, LRU eviction */
 541                 uint64_t addr:18;       /* Phys mem addr (33..16) */
 542         } cn31xx;
 543         struct cvmx_l2c_tag_cn38xx {
 544                 uint64_t reserved:43;
 545                 uint64_t V:1;           /* Line valid */
 546                 uint64_t D:1;           /* Line dirty */
 547                 uint64_t L:1;           /* Line locked */
 548                 uint64_t U:1;           /* Use, LRU eviction */
 549                 uint64_t addr:17;       /* Phys mem addr (33..17) */
 550         } cn38xx;
 551         struct cvmx_l2c_tag_cn58xx {
 552                 uint64_t reserved:44;
 553                 uint64_t V:1;           /* Line valid */
 554                 uint64_t D:1;           /* Line dirty */
 555                 uint64_t L:1;           /* Line locked */
 556                 uint64_t U:1;           /* Use, LRU eviction */
 557                 uint64_t addr:16;       /* Phys mem addr (33..18) */
 558         } cn58xx;
 559         struct cvmx_l2c_tag_cn58xx cn56xx;      /* 2048 sets */
 560         struct cvmx_l2c_tag_cn31xx cn52xx;      /* 512 sets */
 561 };
 562
 563
 564 /**
 565  * @INTERNAL
 566  * Function to read a L2C tag.  This code make the current core
 567  * the 'debug core' for the L2.  This code must only be executed by
 568  * 1 core at a time.
 569  *
 570  * @assoc:  Association (way) of the tag to dump
 571  * @index:  Index of the cacheline
 572  *
 573  * Returns The Octeon model specific tag structure.  This is
 574  *         translated by a wrapper function to a generic form that is
 575  *         easier for applications to use.
 576  */
 577 static union __cvmx_l2c_tag __read_l2_tag(uint64_t assoc, uint64_t index)
 578 {
 579
 580         uint64_t debug_tag_addr = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, (index << 7) + 96);
 581         uint64_t core = cvmx_get_core_num();
 582         union __cvmx_l2c_tag tag_val;
 583         uint64_t dbg_addr = CVMX_L2C_DBG;
 584         unsigned long flags;
 585
 586         union cvmx_l2c_dbg debug_val;
 587         debug_val.u64 = 0;
 588         /*
 589          * For low core count parts, the core number is always small
 590          * enough to stay in the correct field and not set any
 591          * reserved bits.
 592          */
 593         debug_val.s.ppnum = core;
 594         debug_val.s.l2t = 1;
 595         debug_val.s.set = assoc;
 596
 597         local_irq_save(flags);
 598         /*
 599          * Make sure core is quiet (no prefetches, etc.) before
 600          * entering debug mode.
 601          */
 602         CVMX_SYNC;
 603         /* Flush L1 to make sure debug load misses L1 */
 604         CVMX_DCACHE_INVALIDATE;
 605
 606         /*
 607          * The following must be done in assembly as when in debug
 608          * mode all data loads from L2 return special debug data, not
 609          * normal memory contents.  Also, interrupts must be disabled,
 610          * since if an interrupt occurs while in debug mode the ISR
 611          * will get debug data from all its memory * reads instead of
 612          * the contents of memory.
 613          */
 614
 615         asm volatile (
 616                 ".set push\n\t"
 617                 ".set mips64\n\t"
 618                 ".set noreorder\n\t"
 619                 "sd    %[dbg_val], 0(%[dbg_addr])\n\t"   /* Enter debug mode, wait for store */
 620                 "ld    $0, 0(%[dbg_addr])\n\t"
 621                 "ld    %[tag_val], 0(%[tag_addr])\n\t"   /* Read L2C tag data */
 622                 "sd    $0, 0(%[dbg_addr])\n\t"          /* Exit debug mode, wait for store */
 623                 "ld    $0, 0(%[dbg_addr])\n\t"
 624                 "cache 9, 0($0)\n\t"             /* Invalidate dcache to discard debug data */
 625                 ".set pop"
 626                 : [tag_val] "=r" (tag_val)
 627                 : [dbg_addr] "r" (dbg_addr), [dbg_val] "r" (debug_val), [tag_addr] "r" (debug_tag_addr)
 628                 : "memory");
 629
 630         local_irq_restore(flags);
 631
 632         return tag_val;
 633 }
 634
 635
 636 union cvmx_l2c_tag cvmx_l2c_get_tag(uint32_t association, uint32_t index)
 637 {
 638         union cvmx_l2c_tag tag;
 639         tag.u64 = 0;
 640
 641         if ((int)association >= cvmx_l2c_get_num_assoc()) {
 642                 cvmx_dprintf("ERROR: cvmx_l2c_get_tag association out of range\n");
 643                 return tag;
 644         }
 645         if ((int)index >= cvmx_l2c_get_num_sets()) {
 646                 cvmx_dprintf("ERROR: cvmx_l2c_get_tag index out of range (arg: %d, max: %d)\n",
 647                              (int)index, cvmx_l2c_get_num_sets());
 648                 return tag;
 649         }
 650         if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
 651                 union cvmx_l2c_tadx_tag l2c_tadx_tag;
 652                 uint64_t address = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
 653                                                 (association << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) |
 654                                                 (index << CVMX_L2C_IDX_ADDR_SHIFT));
 655                 /*
 656                  * Use L2 cache Index load tag cache instruction, as
 657                  * hardware loads the virtual tag for the L2 cache
 658                  * block with the contents of L2C_TAD0_TAG
 659                  * register.
 660                  */
 661                 CVMX_CACHE_LTGL2I(address, 0);
 662                 CVMX_SYNC;   /* make sure CVMX_L2C_TADX_TAG is updated */
 663                 l2c_tadx_tag.u64 = cvmx_read_csr(CVMX_L2C_TADX_TAG(0));
 664
 665                 tag.s.V     = l2c_tadx_tag.s.valid;
 666                 tag.s.D     = l2c_tadx_tag.s.dirty;
 667                 tag.s.L     = l2c_tadx_tag.s.lock;
 668                 tag.s.U     = l2c_tadx_tag.s.use;
 669                 tag.s.addr  = l2c_tadx_tag.s.tag;
 670         } else {
 671                 union __cvmx_l2c_tag tmp_tag;
 672                 /* __read_l2_tag is intended for internal use only */
 673                 tmp_tag = __read_l2_tag(association, index);
 674
 675                 /*
 676                  * Convert all tag structure types to generic version,
 677                  * as it can represent all models.
 678                  */
 679                 if (OCTEON_IS_MODEL(OCTEON_CN58XX) || OCTEON_IS_MODEL(OCTEON_CN56XX)) {
 680                         tag.s.V    = tmp_tag.cn58xx.V;
 681                         tag.s.D    = tmp_tag.cn58xx.D;
 682                         tag.s.L    = tmp_tag.cn58xx.L;
 683                         tag.s.U    = tmp_tag.cn58xx.U;
 684                         tag.s.addr = tmp_tag.cn58xx.addr;
 685                 } else if (OCTEON_IS_MODEL(OCTEON_CN38XX)) {
 686                         tag.s.V    = tmp_tag.cn38xx.V;
 687                         tag.s.D    = tmp_tag.cn38xx.D;
 688                         tag.s.L    = tmp_tag.cn38xx.L;
 689                         tag.s.U    = tmp_tag.cn38xx.U;
 690                         tag.s.addr = tmp_tag.cn38xx.addr;
 691                 } else if (OCTEON_IS_MODEL(OCTEON_CN31XX) || OCTEON_IS_MODEL(OCTEON_CN52XX)) {
 692                         tag.s.V    = tmp_tag.cn31xx.V;
 693                         tag.s.D    = tmp_tag.cn31xx.D;
 694                         tag.s.L    = tmp_tag.cn31xx.L;
 695                         tag.s.U    = tmp_tag.cn31xx.U;
 696                         tag.s.addr = tmp_tag.cn31xx.addr;
 697                 } else if (OCTEON_IS_MODEL(OCTEON_CN30XX)) {
 698                         tag.s.V    = tmp_tag.cn30xx.V;
 699                         tag.s.D    = tmp_tag.cn30xx.D;
 700                         tag.s.L    = tmp_tag.cn30xx.L;
 701                         tag.s.U    = tmp_tag.cn30xx.U;
 702                         tag.s.addr = tmp_tag.cn30xx.addr;
 703                 } else if (OCTEON_IS_MODEL(OCTEON_CN50XX)) {
 704                         tag.s.V    = tmp_tag.cn50xx.V;
 705                         tag.s.D    = tmp_tag.cn50xx.D;
 706                         tag.s.L    = tmp_tag.cn50xx.L;
 707                         tag.s.U    = tmp_tag.cn50xx.U;
 708                         tag.s.addr = tmp_tag.cn50xx.addr;
 709                 } else {
 710                         cvmx_dprintf("Unsupported OCTEON Model in %s\n", __func__);
 711                 }
 712         }
 713         return tag;
 714 }
 715
 716 uint32_t cvmx_l2c_address_to_index(uint64_t addr)
 717 {
 718         uint64_t idx = addr >> CVMX_L2C_IDX_ADDR_SHIFT;
 719         int indxalias = 0;
 720
 721         if (OCTEON_IS_MODEL(OCTEON_CN6XXX)) {
 722                 union cvmx_l2c_ctl l2c_ctl;
 723                 l2c_ctl.u64 = cvmx_read_csr(CVMX_L2C_CTL);
 724                 indxalias = !l2c_ctl.s.disidxalias;
 725         } else {
 726                 union cvmx_l2c_cfg l2c_cfg;
 727                 l2c_cfg.u64 = cvmx_read_csr(CVMX_L2C_CFG);
 728                 indxalias = l2c_cfg.s.idxalias;
 729         }
 730
 731         if (indxalias) {
 732                 if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
 733                         uint32_t a_14_12 = (idx / (CVMX_L2C_MEMBANK_SELECT_SIZE/(1<<CVMX_L2C_IDX_ADDR_SHIFT))) & 0x7;
 734                         idx ^= idx / cvmx_l2c_get_num_sets();
 735                         idx ^= a_14_12;
 736                 } else {
 737                         idx ^= ((addr & CVMX_L2C_ALIAS_MASK) >> CVMX_L2C_TAG_ADDR_ALIAS_SHIFT);
 738                 }
 739         }
 740         idx &= CVMX_L2C_IDX_MASK;
 741         return idx;
 742 }
 743
 744 int cvmx_l2c_get_cache_size_bytes(void)
 745 {
 746         return cvmx_l2c_get_num_sets() * cvmx_l2c_get_num_assoc() *
 747                 CVMX_CACHE_LINE_SIZE;
 748 }
 749
 750 /**
 751  * Return log base 2 of the number of sets in the L2 cache
 752  * Returns
 753  */
 754 int cvmx_l2c_get_set_bits(void)
 755 {
 756         int l2_set_bits;
 757         if (OCTEON_IS_MODEL(OCTEON_CN56XX) || OCTEON_IS_MODEL(OCTEON_CN58XX))
 758                 l2_set_bits = 11;       /* 2048 sets */
 759         else if (OCTEON_IS_MODEL(OCTEON_CN38XX) || OCTEON_IS_MODEL(OCTEON_CN63XX))
 760                 l2_set_bits = 10;       /* 1024 sets */
 761         else if (OCTEON_IS_MODEL(OCTEON_CN31XX) || OCTEON_IS_MODEL(OCTEON_CN52XX))
 762                 l2_set_bits = 9;        /* 512 sets */
 763         else if (OCTEON_IS_MODEL(OCTEON_CN30XX))
 764                 l2_set_bits = 8;        /* 256 sets */
 765         else if (OCTEON_IS_MODEL(OCTEON_CN50XX))
 766                 l2_set_bits = 7;        /* 128 sets */
 767         else {
 768                 cvmx_dprintf("Unsupported OCTEON Model in %s\n", __func__);
 769                 l2_set_bits = 11;       /* 2048 sets */
 770         }
 771         return l2_set_bits;
 772 }
 773
 774 /* Return the number of sets in the L2 Cache */
 775 int cvmx_l2c_get_num_sets(void)
 776 {
 777         return 1 << cvmx_l2c_get_set_bits();
 778 }
 779
 780 /* Return the number of associations in the L2 Cache */
 781 int cvmx_l2c_get_num_assoc(void)
 782 {
 783         int l2_assoc;
 784         if (OCTEON_IS_MODEL(OCTEON_CN56XX) ||
 785             OCTEON_IS_MODEL(OCTEON_CN52XX) ||
 786             OCTEON_IS_MODEL(OCTEON_CN58XX) ||
 787             OCTEON_IS_MODEL(OCTEON_CN50XX) ||
 788             OCTEON_IS_MODEL(OCTEON_CN38XX))
 789                 l2_assoc = 8;
 790         else if (OCTEON_IS_MODEL(OCTEON_CN63XX))
 791                 l2_assoc = 16;
 792         else if (OCTEON_IS_MODEL(OCTEON_CN31XX) ||
 793                  OCTEON_IS_MODEL(OCTEON_CN30XX))
 794                 l2_assoc = 4;
 795         else {
 796                 cvmx_dprintf("Unsupported OCTEON Model in %s\n", __func__);
 797                 l2_assoc = 8;
 798         }
 799
 800         /* Check to see if part of the cache is disabled */
 801         if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
 802                 union cvmx_mio_fus_dat3 mio_fus_dat3;
 803
 804                 mio_fus_dat3.u64 = cvmx_read_csr(CVMX_MIO_FUS_DAT3);
 805                 /*
 806                  * cvmx_mio_fus_dat3.s.l2c_crip fuses map as follows
 807                  * <2> will be not used for 63xx
 808                  * <1> disables 1/2 ways
 809                  * <0> disables 1/4 ways
 810                  * They are cumulative, so for 63xx:
 811                  * <1> <0>
 812                  * 0 0 16-way 2MB cache
 813                  * 0 1 12-way 1.5MB cache
 814                  * 1 0 8-way 1MB cache
 815                  * 1 1 4-way 512KB cache
 816                  */
 817
 818                 if (mio_fus_dat3.s.l2c_crip == 3)
 819                         l2_assoc = 4;
 820                 else if (mio_fus_dat3.s.l2c_crip == 2)
 821                         l2_assoc = 8;
 822                 else if (mio_fus_dat3.s.l2c_crip == 1)
 823                         l2_assoc = 12;
 824         } else {
 825                 union cvmx_l2d_fus3 val;
 826                 val.u64 = cvmx_read_csr(CVMX_L2D_FUS3);
 827                 /*
 828                  * Using shifts here, as bit position names are
 829                  * different for each model but they all mean the
 830                  * same.
 831                  */
 832                 if ((val.u64 >> 35) & 0x1)
 833                         l2_assoc = l2_assoc >> 2;
 834                 else if ((val.u64 >> 34) & 0x1)
 835                         l2_assoc = l2_assoc >> 1;
 836         }
 837         return l2_assoc;
 838 }
 839
 840 /**
 841  * Flush a line from the L2 cache
 842  * This should only be called from one core at a time, as this routine
 843  * sets the core to the 'debug' core in order to flush the line.
 844  *
 845  * @assoc:  Association (or way) to flush
 846  * @index:  Index to flush
 847  */
 848 void cvmx_l2c_flush_line(uint32_t assoc, uint32_t index)
 849 {
 850         /* Check the range of the index. */
 851         if (index > (uint32_t)cvmx_l2c_get_num_sets()) {
 852                 cvmx_dprintf("ERROR: cvmx_l2c_flush_line index out of range.\n");
 853                 return;
 854         }
 855
 856         /* Check the range of association. */
 857         if (assoc > (uint32_t)cvmx_l2c_get_num_assoc()) {
 858                 cvmx_dprintf("ERROR: cvmx_l2c_flush_line association out of range.\n");
 859                 return;
 860         }
 861
 862         if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
 863                 uint64_t address;
 864                 /* Create the address based on index and association.
 865                  * Bits<20:17> select the way of the cache block involved in
 866                  *             the operation
 867                  * Bits<16:7> of the effect address select the index
 868                  */
 869                 address = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
 870                                 (assoc << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) |
 871                                 (index << CVMX_L2C_IDX_ADDR_SHIFT));
 872                 CVMX_CACHE_WBIL2I(address, 0);
 873         } else {
 874                 union cvmx_l2c_dbg l2cdbg;
 875
 876                 l2cdbg.u64 = 0;
 877                 if (!OCTEON_IS_MODEL(OCTEON_CN30XX))
 878                         l2cdbg.s.ppnum = cvmx_get_core_num();
 879                 l2cdbg.s.finv = 1;
 880
 881                 l2cdbg.s.set = assoc;
 882                 cvmx_spinlock_lock(&cvmx_l2c_spinlock);
 883                 /*
 884                  * Enter debug mode, and make sure all other writes
 885                  * complete before we enter debug mode
 886                  */
 887                 CVMX_SYNC;
 888                 cvmx_write_csr(CVMX_L2C_DBG, l2cdbg.u64);
 889                 cvmx_read_csr(CVMX_L2C_DBG);
 890
 891                 CVMX_PREPARE_FOR_STORE(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
 892                                                     index * CVMX_CACHE_LINE_SIZE),
 893                                        0);
 894                 /* Exit debug mode */
 895                 CVMX_SYNC;
 896                 cvmx_write_csr(CVMX_L2C_DBG, 0);
 897                 cvmx_read_csr(CVMX_L2C_DBG);
 898                 cvmx_spinlock_unlock(&cvmx_l2c_spinlock);
 899         }
 900 }