arch/mips/mm/cerr-sb1.c

   1 /*
   2  * Copyright (C) 2001,2002,2003 Broadcom Corporation
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public License
   6  * as published by the Free Software Foundation; either version 2
   7  * of the License, or (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  17  */
  18 #include <linux/config.h>
  19 #include <linux/sched.h>
  20 #include <asm/mipsregs.h>
  21 #include <asm/sibyte/sb1250.h>
  22
  23 #ifndef CONFIG_SIBYTE_BUS_WATCHER
  24 #include <asm/io.h>
  25 #include <asm/sibyte/sb1250_regs.h>
  26 #include <asm/sibyte/sb1250_scd.h>
  27 #endif
  28
  29 /* SB1 definitions */
  30
  31 /* XXX should come from config1 XXX */
  32 #define SB1_CACHE_INDEX_MASK   0x1fe0
  33
  34 #define CP0_ERRCTL_RECOVERABLE (1 << 31)
  35 #define CP0_ERRCTL_DCACHE      (1 << 30)
  36 #define CP0_ERRCTL_ICACHE      (1 << 29)
  37 #define CP0_ERRCTL_MULTIBUS    (1 << 23)
  38 #define CP0_ERRCTL_MC_TLB      (1 << 15)
  39 #define CP0_ERRCTL_MC_TIMEOUT  (1 << 14)
  40
  41 #define CP0_CERRI_TAG_PARITY   (1 << 29)
  42 #define CP0_CERRI_DATA_PARITY  (1 << 28)
  43 #define CP0_CERRI_EXTERNAL     (1 << 26)
  44
  45 #define CP0_CERRI_IDX_VALID(c) (!((c) & CP0_CERRI_EXTERNAL))
  46 #define CP0_CERRI_DATA         (CP0_CERRI_DATA_PARITY)
  47
  48 #define CP0_CERRD_MULTIPLE     (1 << 31)
  49 #define CP0_CERRD_TAG_STATE    (1 << 30)
  50 #define CP0_CERRD_TAG_ADDRESS  (1 << 29)
  51 #define CP0_CERRD_DATA_SBE     (1 << 28)
  52 #define CP0_CERRD_DATA_DBE     (1 << 27)
  53 #define CP0_CERRD_EXTERNAL     (1 << 26)
  54 #define CP0_CERRD_LOAD         (1 << 25)
  55 #define CP0_CERRD_STORE        (1 << 24)
  56 #define CP0_CERRD_FILLWB       (1 << 23)
  57 #define CP0_CERRD_COHERENCY    (1 << 22)
  58 #define CP0_CERRD_DUPTAG       (1 << 21)
  59
  60 #define CP0_CERRD_DPA_VALID(c) (!((c) & CP0_CERRD_EXTERNAL))
  61 #define CP0_CERRD_IDX_VALID(c) \
  62    (((c) & (CP0_CERRD_LOAD | CP0_CERRD_STORE)) ? (!((c) & CP0_CERRD_EXTERNAL)) : 0)
  63 #define CP0_CERRD_CAUSES \
  64    (CP0_CERRD_LOAD | CP0_CERRD_STORE | CP0_CERRD_FILLWB | CP0_CERRD_COHERENCY | CP0_CERRD_DUPTAG)
  65 #define CP0_CERRD_TYPES \
  66    (CP0_CERRD_TAG_STATE | CP0_CERRD_TAG_ADDRESS | CP0_CERRD_DATA_SBE | CP0_CERRD_DATA_DBE | CP0_CERRD_EXTERNAL)
  67 #define CP0_CERRD_DATA         (CP0_CERRD_DATA_SBE | CP0_CERRD_DATA_DBE)
  68
  69 static uint32_t extract_ic(unsigned short addr, int data);
  70 static uint32_t extract_dc(unsigned short addr, int data);
  71
  72 static inline void breakout_errctl(unsigned int val)
  73 {
  74         if (val & CP0_ERRCTL_RECOVERABLE)
  75                 prom_printf(" recoverable");
  76         if (val & CP0_ERRCTL_DCACHE)
  77                 prom_printf(" dcache");
  78         if (val & CP0_ERRCTL_ICACHE)
  79                 prom_printf(" icache");
  80         if (val & CP0_ERRCTL_MULTIBUS)
  81                 prom_printf(" multiple-buserr");
  82         prom_printf("\n");
  83 }
  84
  85 static inline void breakout_cerri(unsigned int val)
  86 {
  87         if (val & CP0_CERRI_TAG_PARITY)
  88                 prom_printf(" tag-parity");
  89         if (val & CP0_CERRI_DATA_PARITY)
  90                 prom_printf(" data-parity");
  91         if (val & CP0_CERRI_EXTERNAL)
  92                 prom_printf(" external");
  93         prom_printf("\n");
  94 }
  95
  96 static inline void breakout_cerrd(unsigned int val)
  97 {
  98         switch (val & CP0_CERRD_CAUSES) {
  99         case CP0_CERRD_LOAD:
 100                 prom_printf(" load,");
 101                 break;
 102         case CP0_CERRD_STORE:
 103                 prom_printf(" store,");
 104                 break;
 105         case CP0_CERRD_FILLWB:
 106                 prom_printf(" fill/wb,");
 107                 break;
 108         case CP0_CERRD_COHERENCY:
 109                 prom_printf(" coherency,");
 110                 break;
 111         case CP0_CERRD_DUPTAG:
 112                 prom_printf(" duptags,");
 113                 break;
 114         default:
 115                 prom_printf(" NO CAUSE,");
 116                 break;
 117         }
 118         if (!(val & CP0_CERRD_TYPES))
 119                 prom_printf(" NO TYPE");
 120         else {
 121                 if (val & CP0_CERRD_MULTIPLE)
 122                         prom_printf(" multi-err");
 123                 if (val & CP0_CERRD_TAG_STATE)
 124                         prom_printf(" tag-state");
 125                 if (val & CP0_CERRD_TAG_ADDRESS)
 126                         prom_printf(" tag-address");
 127                 if (val & CP0_CERRD_DATA_SBE)
 128                         prom_printf(" data-SBE");
 129                 if (val & CP0_CERRD_DATA_DBE)
 130                         prom_printf(" data-DBE");
 131                 if (val & CP0_CERRD_EXTERNAL)
 132                         prom_printf(" external");
 133         }
 134         prom_printf("\n");
 135 }
 136
 137 #ifndef CONFIG_SIBYTE_BUS_WATCHER
 138
 139 static void check_bus_watcher(void)
 140 {
 141         uint32_t status, l2_err, memio_err;
 142
 143         /* Destructive read, clears register and interrupt */
 144         status = csr_in32(IOADDR(A_SCD_BUS_ERR_STATUS));
 145         /* Bit 31 is always on, but there's no #define for that */
 146         if (status & ~(1UL << 31)) {
 147                 l2_err = csr_in32(IOADDR(A_BUS_L2_ERRORS));
 148                 memio_err = csr_in32(IOADDR(A_BUS_MEM_IO_ERRORS));
 149                 prom_printf("Bus watcher error counters: %08x %08x\n", l2_err, memio_err);
 150                 prom_printf("\nLast recorded signature:\n");
 151                 prom_printf("Request %02x from %d, answered by %d with Dcode %d\n",
 152                        (unsigned int)(G_SCD_BERR_TID(status) & 0x3f),
 153                        (int)(G_SCD_BERR_TID(status) >> 6),
 154                        (int)G_SCD_BERR_RID(status),
 155                        (int)G_SCD_BERR_DCODE(status));
 156         } else {
 157                 prom_printf("Bus watcher indicates no error\n");
 158         }
 159 }
 160 #else
 161 extern void check_bus_watcher(void);
 162 #endif
 163
 164 asmlinkage void sb1_cache_error(void)
 165 {
 166         uint64_t cerr_dpa;
 167         uint32_t errctl, cerr_i, cerr_d, dpalo, dpahi, eepc, res;
 168
 169         prom_printf("Cache error exception on CPU %x:\n",
 170                     (read_c0_prid() >> 25) & 0x7);
 171
 172         __asm__ __volatile__ (
 173         "       .set    push\n\t"
 174         "       .set    mips64\n\t"
 175         "       .set    noat\n\t"
 176         "       mfc0    %0, $26\n\t"
 177         "       mfc0    %1, $27\n\t"
 178         "       mfc0    %2, $27, 1\n\t"
 179         "       dmfc0   $1, $27, 3\n\t"
 180         "       dsrl32  %3, $1, 0 \n\t"
 181         "       sll     %4, $1, 0 \n\t"
 182         "       mfc0    %5, $30\n\t"
 183         "       .set    pop"
 184         : "=r" (errctl), "=r" (cerr_i), "=r" (cerr_d),
 185           "=r" (dpahi), "=r" (dpalo), "=r" (eepc));
 186
 187         cerr_dpa = (((uint64_t)dpahi) << 32) | dpalo;
 188         prom_printf(" c0_errorepc ==   %08x\n", eepc);
 189         prom_printf(" c0_errctl   ==   %08x", errctl);
 190         breakout_errctl(errctl);
 191         if (errctl & CP0_ERRCTL_ICACHE) {
 192                 prom_printf(" c0_cerr_i   ==   %08x", cerr_i);
 193                 breakout_cerri(cerr_i);
 194                 if (CP0_CERRI_IDX_VALID(cerr_i)) {
 195                         /* Check index of EPC, allowing for delay slot */
 196                         if (((eepc & SB1_CACHE_INDEX_MASK) != (cerr_i & SB1_CACHE_INDEX_MASK)) &&
 197                             ((eepc & SB1_CACHE_INDEX_MASK) != ((cerr_i & SB1_CACHE_INDEX_MASK) - 4)))
 198                                 prom_printf(" cerr_i idx doesn't match eepc\n");
 199                         else {
 200                                 res = extract_ic(cerr_i & SB1_CACHE_INDEX_MASK,
 201                                                  (cerr_i & CP0_CERRI_DATA) != 0);
 202                                 if (!(res & cerr_i))
 203                                         prom_printf("...didn't see indicated icache problem\n");
 204                         }
 205                 }
 206         }
 207         if (errctl & CP0_ERRCTL_DCACHE) {
 208                 prom_printf(" c0_cerr_d   ==   %08x", cerr_d);
 209                 breakout_cerrd(cerr_d);
 210                 if (CP0_CERRD_DPA_VALID(cerr_d)) {
 211                         prom_printf(" c0_cerr_dpa == %010llx\n", cerr_dpa);
 212                         if (!CP0_CERRD_IDX_VALID(cerr_d)) {
 213                                 res = extract_dc(cerr_dpa & SB1_CACHE_INDEX_MASK,
 214                                                  (cerr_d & CP0_CERRD_DATA) != 0);
 215                                 if (!(res & cerr_d))
 216                                         prom_printf("...didn't see indicated dcache problem\n");
 217                         } else {
 218                                 if ((cerr_dpa & SB1_CACHE_INDEX_MASK) != (cerr_d & SB1_CACHE_INDEX_MASK))
 219                                         prom_printf(" cerr_d idx doesn't match cerr_dpa\n");
 220                                 else {
 221                                         res = extract_dc(cerr_d & SB1_CACHE_INDEX_MASK,
 222                                                          (cerr_d & CP0_CERRD_DATA) != 0);
 223                                         if (!(res & cerr_d))
 224                                                 prom_printf("...didn't see indicated problem\n");
 225                                 }
 226                         }
 227                 }
 228         }
 229
 230         check_bus_watcher();
 231
 232         while (1);
 233         /*
 234          * This tends to make things get really ugly; let's just stall instead.
 235          *    panic("Can't handle the cache error!");
 236          */
 237 }
 238
 239
 240 /* Parity lookup table. */
 241 static const uint8_t parity[256] = {
 242         0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0,1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,
 243         1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0,
 244         1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0,
 245         0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0,1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,
 246         1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0,
 247         0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0,1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,
 248         0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0,1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,
 249         1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0
 250 };
 251
 252 /* Masks to select bits for Hamming parity, mask_72_64[i] for bit[i] */
 253 static const uint64_t mask_72_64[8] = {
 254         0x0738C808099264FFULL,
 255         0x38C808099264FF07ULL,
 256         0xC808099264FF0738ULL,
 257         0x08099264FF0738C8ULL,
 258         0x099264FF0738C808ULL,
 259         0x9264FF0738C80809ULL,
 260         0x64FF0738C8080992ULL,
 261         0xFF0738C808099264ULL
 262 };
 263
 264 /* Calculate the parity on a range of bits */
 265 static char range_parity(uint64_t dword, int max, int min)
 266 {
 267         char parity = 0;
 268         int i;
 269         dword >>= min;
 270         for (i=max-min; i>=0; i--) {
 271                 if (dword & 0x1)
 272                         parity = !parity;
 273                 dword >>= 1;
 274         }
 275         return parity;
 276 }
 277
 278 /* Calculate the 4-bit even byte-parity for an instruction */
 279 static unsigned char inst_parity(uint32_t word)
 280 {
 281         int i, j;
 282         char parity = 0;
 283         for (j=0; j<4; j++) {
 284                 char byte_parity = 0;
 285                 for (i=0; i<8; i++) {
 286                         if (word & 0x80000000)
 287                                 byte_parity = !byte_parity;
 288                         word <<= 1;
 289                 }
 290                 parity <<= 1;
 291                 parity |= byte_parity;
 292         }
 293         return parity;
 294 }
 295
 296 static uint32_t extract_ic(unsigned short addr, int data)
 297 {
 298         unsigned short way;
 299         int valid;
 300         uint64_t taglo, va, tlo_tmp;
 301         uint32_t taghi, taglolo, taglohi;
 302         uint8_t lru;
 303         int res = 0;
 304
 305         prom_printf("Icache index 0x%04x  ", addr);
 306         for (way = 0; way < 4; way++) {
 307                 /* Index-load-tag-I */
 308                 __asm__ __volatile__ (
 309                 "       .set    push            \n\t"
 310                 "       .set    noreorder       \n\t"
 311                 "       .set    mips64          \n\t"
 312                 "       .set    noat            \n\t"
 313                 "       cache   4, 0(%3)        \n\t"
 314                 "       mfc0    %0, $29         \n\t"
 315                 "       dmfc0   $1, $28         \n\t"
 316                 "       dsrl32  %1, $1, 0       \n\t"
 317                 "       sll     %2, $1, 0       \n\t"
 318                 "       .set    pop"
 319                 : "=r" (taghi), "=r" (taglohi), "=r" (taglolo)
 320                 : "r" ((way << 13) | addr));
 321
 322                 taglo = ((unsigned long long)taglohi << 32) | taglolo;
 323                 if (way == 0) {
 324                         lru = (taghi >> 14) & 0xff;
 325                         prom_printf("[Bank %d Set 0x%02x]  LRU > %d %d %d %d > MRU\n",
 326                                     ((addr >> 5) & 0x3), /* bank */
 327                                     ((addr >> 7) & 0x3f), /* index */
 328                                     (lru & 0x3),
 329                                     ((lru >> 2) & 0x3),
 330                                     ((lru >> 4) & 0x3),
 331                                     ((lru >> 6) & 0x3));
 332                 }
 333                 va = (taglo & 0xC0000FFFFFFFE000ULL) | addr;
 334                 if ((taglo & (1 << 31)) && (((taglo >> 62) & 0x3) == 3))
 335                         va |= 0x3FFFF00000000000ULL;
 336                 valid = ((taghi >> 29) & 1);
 337                 if (valid) {
 338                         tlo_tmp = taglo & 0xfff3ff;
 339                         if (((taglo >> 10) & 1) ^ range_parity(tlo_tmp, 23, 0)) {
 340                                 prom_printf("   ** bad parity in VTag0/G/ASID\n");
 341                                 res |= CP0_CERRI_TAG_PARITY;
 342                         }
 343                         if (((taglo >> 11) & 1) ^ range_parity(taglo, 63, 24)) {
 344                                 prom_printf("   ** bad parity in R/VTag1\n");
 345                                 res |= CP0_CERRI_TAG_PARITY;
 346                         }
 347                 }
 348                 if (valid ^ ((taghi >> 27) & 1)) {
 349                         prom_printf("   ** bad parity for valid bit\n");
 350                         res |= CP0_CERRI_TAG_PARITY;
 351                 }
 352                 prom_printf(" %d  [VA %016llx]  [Vld? %d]  raw tags: %08X-%016llX\n",
 353                             way, va, valid, taghi, taglo);
 354
 355                 if (data) {
 356                         uint32_t datahi, insta, instb;
 357                         uint8_t predecode;
 358                         int offset;
 359
 360                         /* (hit all banks and ways) */
 361                         for (offset = 0; offset < 4; offset++) {
 362                                 /* Index-load-data-I */
 363                                 __asm__ __volatile__ (
 364                                 "       .set    push\n\t"
 365                                 "       .set    noreorder\n\t"
 366                                 "       .set    mips64\n\t"
 367                                 "       .set    noat\n\t"
 368                                 "       cache   6, 0(%3)  \n\t"
 369                                 "       mfc0    %0, $29, 1\n\t"
 370                                 "       dmfc0  $1, $28, 1\n\t"
 371                                 "       dsrl32 %1, $1, 0 \n\t"
 372                                 "       sll    %2, $1, 0 \n\t"
 373                                 "       .set    pop         \n"
 374                                 : "=r" (datahi), "=r" (insta), "=r" (instb)
 375                                 : "r" ((way << 13) | addr | (offset << 3)));
 376                                 predecode = (datahi >> 8) & 0xff;
 377                                 if (((datahi >> 16) & 1) != (uint32_t)range_parity(predecode, 7, 0)) {
 378                                         prom_printf("   ** bad parity in predecode\n");
 379                                         res |= CP0_CERRI_DATA_PARITY;
 380                                 }
 381                                 /* XXXKW should/could check predecode bits themselves */
 382                                 if (((datahi >> 4) & 0xf) ^ inst_parity(insta)) {
 383                                         prom_printf("   ** bad parity in instruction a\n");
 384                                         res |= CP0_CERRI_DATA_PARITY;
 385                                 }
 386                                 if ((datahi & 0xf) ^ inst_parity(instb)) {
 387                                         prom_printf("   ** bad parity in instruction b\n");
 388                                         res |= CP0_CERRI_DATA_PARITY;
 389                                 }
 390                                 prom_printf("  %05X-%08X%08X", datahi, insta, instb);
 391                         }
 392                         prom_printf("\n");
 393                 }
 394         }
 395         return res;
 396 }
 397
 398 /* Compute the ECC for a data doubleword */
 399 static uint8_t dc_ecc(uint64_t dword)
 400 {
 401         uint64_t t;
 402         uint32_t w;
 403         uint8_t  p;
 404         int      i;
 405
 406         p = 0;
 407         for (i = 7; i >= 0; i--)
 408         {
 409                 p <<= 1;
 410                 t = dword & mask_72_64[i];
 411                 w = (uint32_t)(t >> 32);
 412                 p ^= (parity[w>>24] ^ parity[(w>>16) & 0xFF]
 413                       ^ parity[(w>>8) & 0xFF] ^ parity[w & 0xFF]);
 414                 w = (uint32_t)(t & 0xFFFFFFFF);
 415                 p ^= (parity[w>>24] ^ parity[(w>>16) & 0xFF]
 416                       ^ parity[(w>>8) & 0xFF] ^ parity[w & 0xFF]);
 417         }
 418         return p;
 419 }
 420
 421 struct dc_state {
 422         unsigned char val;
 423         char *name;
 424 };
 425
 426 static struct dc_state dc_states[] = {
 427         { 0x00, "INVALID" },
 428         { 0x0f, "COH-SHD" },
 429         { 0x13, "NCO-E-C" },
 430         { 0x19, "NCO-E-D" },
 431         { 0x16, "COH-E-C" },
 432         { 0x1c, "COH-E-D" },
 433         { 0xff, "*ERROR*" }
 434 };
 435
 436 #define DC_TAG_VALID(state) \
 437     (((state) == 0xf) || ((state) == 0x13) || ((state) == 0x19) || ((state == 0x16)) || ((state) == 0x1c))
 438
 439 static char *dc_state_str(unsigned char state)
 440 {
 441         struct dc_state *dsc = dc_states;
 442         while (dsc->val != 0xff) {
 443                 if (dsc->val == state)
 444                         break;
 445                 dsc++;
 446         }
 447         return dsc->name;
 448 }
 449
 450 static uint32_t extract_dc(unsigned short addr, int data)
 451 {
 452         int valid, way;
 453         unsigned char state;
 454         uint64_t taglo, pa;
 455         uint32_t taghi, taglolo, taglohi;
 456         uint8_t ecc, lru;
 457         int res = 0;
 458
 459         prom_printf("Dcache index 0x%04x  ", addr);
 460         for (way = 0; way < 4; way++) {
 461                 __asm__ __volatile__ (
 462                 "       .set    push\n\t"
 463                 "       .set    noreorder\n\t"
 464                 "       .set    mips64\n\t"
 465                 "       .set    noat\n\t"
 466                 "       cache   5, 0(%3)\n\t"   /* Index-load-tag-D */
 467                 "       mfc0    %0, $29, 2\n\t"
 468                 "       dmfc0   $1, $28, 2\n\t"
 469                 "       dsrl32  %1, $1, 0\n\t"
 470                 "       sll     %2, $1, 0\n\t"
 471                 "       .set    pop"
 472                 : "=r" (taghi), "=r" (taglohi), "=r" (taglolo)
 473                 : "r" ((way << 13) | addr));
 474
 475                 taglo = ((unsigned long long)taglohi << 32) | taglolo;
 476                 pa = (taglo & 0xFFFFFFE000ULL) | addr;
 477                 if (way == 0) {
 478                         lru = (taghi >> 14) & 0xff;
 479                         prom_printf("[Bank %d Set 0x%02x]  LRU > %d %d %d %d > MRU\n",
 480                                     ((addr >> 11) & 0x2) | ((addr >> 5) & 1), /* bank */
 481                                     ((addr >> 6) & 0x3f), /* index */
 482                                     (lru & 0x3),
 483                                     ((lru >> 2) & 0x3),
 484                                     ((lru >> 4) & 0x3),
 485                                     ((lru >> 6) & 0x3));
 486                 }
 487                 state = (taghi >> 25) & 0x1f;
 488                 valid = DC_TAG_VALID(state);
 489                 prom_printf(" %d  [PA %010llx]  [state %s (%02x)]  raw tags: %08X-%016llX\n",
 490                             way, pa, dc_state_str(state), state, taghi, taglo);
 491                 if (valid) {
 492                         if (((taglo >> 11) & 1) ^ range_parity(taglo, 39, 26)) {
 493                                 prom_printf("   ** bad parity in PTag1\n");
 494                                 res |= CP0_CERRD_TAG_ADDRESS;
 495                         }
 496                         if (((taglo >> 10) & 1) ^ range_parity(taglo, 25, 13)) {
 497                                 prom_printf("   ** bad parity in PTag0\n");
 498                                 res |= CP0_CERRD_TAG_ADDRESS;
 499                         }
 500                 } else {
 501                         res |= CP0_CERRD_TAG_STATE;
 502                 }
 503
 504                 if (data) {
 505                         uint64_t datalo;
 506                         uint32_t datalohi, datalolo, datahi;
 507                         int offset;
 508
 509                         for (offset = 0; offset < 4; offset++) {
 510                                 /* Index-load-data-D */
 511                                 __asm__ __volatile__ (
 512                                 "       .set    push\n\t"
 513                                 "       .set    noreorder\n\t"
 514                                 "       .set    mips64\n\t"
 515                                 "       .set    noat\n\t"
 516                                 "       cache   7, 0(%3)\n\t" /* Index-load-data-D */
 517                                 "       mfc0    %0, $29, 3\n\t"
 518                                 "       dmfc0   $1, $28, 3\n\t"
 519                                 "       dsrl32  %1, $1, 0 \n\t"
 520                                 "       sll     %2, $1, 0 \n\t"
 521                                 "       .set    pop"
 522                                 : "=r" (datahi), "=r" (datalohi), "=r" (datalolo)
 523                                 : "r" ((way << 13) | addr | (offset << 3)));
 524                                 datalo = ((unsigned long long)datalohi << 32) | datalolo;
 525                                 ecc = dc_ecc(datalo);
 526                                 if (ecc != datahi) {
 527                                         int bits = 0;
 528                                         prom_printf("  ** bad ECC (%02x %02x) ->",
 529                                                     datahi, ecc);
 530                                         ecc ^= datahi;
 531                                         while (ecc) {
 532                                                 if (ecc & 1) bits++;
 533                                                 ecc >>= 1;
 534                                         }
 535                                         res |= (bits == 1) ? CP0_CERRD_DATA_SBE : CP0_CERRD_DATA_DBE;
 536                                 }
 537                                 prom_printf("  %02X-%016llX", datahi, datalo);
 538                         }
 539                         prom_printf("\n");
 540                 }
 541         }
 542         return res;
 543 }