arch/x86/kernel/cpu/mcheck/mce_amd.c

   1 /*
   2  *  (c) 2005, 2006 Advanced Micro Devices, Inc.
   3  *  Your use of this code is subject to the terms and conditions of the
   4  *  GNU general public license version 2. See "COPYING" or
   5  *  http://www.gnu.org/licenses/gpl.html
   6  *
   7  *  Written by Jacob Shin - AMD, Inc.
   8  *
   9  *  Support : jacob.shin@amd.com
  10  *
  11  *  April 2006
  12  *     - added support for AMD Family 0x10 processors
  13  *
  14  *  All MC4_MISCi registers are shared between multi-cores
  15  */
  16 #include <linux/interrupt.h>
  17 #include <linux/notifier.h>
  18 #include <linux/kobject.h>
  19 #include <linux/percpu.h>
  20 #include <linux/sysdev.h>
  21 #include <linux/errno.h>
  22 #include <linux/sched.h>
  23 #include <linux/sysfs.h>
  24 #include <linux/slab.h>
  25 #include <linux/init.h>
  26 #include <linux/cpu.h>
  27 #include <linux/smp.h>
  28
  29 #include <asm/apic.h>
  30 #include <asm/idle.h>
  31 #include <asm/mce.h>
  32 #include <asm/msr.h>
  33
  34 #define PFX               "mce_threshold: "
  35 #define VERSION           "version 1.1.1"
  36 #define NR_BANKS          6
  37 #define NR_BLOCKS         9
  38 #define THRESHOLD_MAX     0xFFF
  39 #define INT_TYPE_APIC     0x00020000
  40 #define MASK_VALID_HI     0x80000000
  41 #define MASK_CNTP_HI      0x40000000
  42 #define MASK_LOCKED_HI    0x20000000
  43 #define MASK_LVTOFF_HI    0x00F00000
  44 #define MASK_COUNT_EN_HI  0x00080000
  45 #define MASK_INT_TYPE_HI  0x00060000
  46 #define MASK_OVERFLOW_HI  0x00010000
  47 #define MASK_ERR_COUNT_HI 0x00000FFF
  48 #define MASK_BLKPTR_LO    0xFF000000
  49 #define MCG_XBLK_ADDR     0xC0000400
  50
  51 struct threshold_block {
  52         unsigned int            block;
  53         unsigned int            bank;
  54         unsigned int            cpu;
  55         u32                     address;
  56         u16                     interrupt_enable;
  57         u16                     threshold_limit;
  58         struct kobject          kobj;
  59         struct list_head        miscj;
  60 };
  61
  62 /* defaults used early on boot */
  63 static struct threshold_block threshold_defaults = {
  64         .interrupt_enable       = 0,
  65         .threshold_limit        = THRESHOLD_MAX,
  66 };
  67
  68 struct threshold_bank {
  69         struct kobject          *kobj;
  70         struct threshold_block  *blocks;
  71         cpumask_var_t           cpus;
  72 };
  73 static DEFINE_PER_CPU(struct threshold_bank * [NR_BANKS], threshold_banks);
  74
  75 #ifdef CONFIG_SMP
  76 static unsigned char shared_bank[NR_BANKS] = {
  77         0, 0, 0, 0, 1
  78 };
  79 #endif
  80
  81 static DEFINE_PER_CPU(unsigned char, bank_map); /* see which banks are on */
  82
  83 static void amd_threshold_interrupt(void);
  84
  85 /*
  86  * CPU Initialization
  87  */
  88
  89 struct thresh_restart {
  90         struct threshold_block  *b;
  91         int                     reset;
  92         u16                     old_limit;
  93 };
  94
  95 /* must be called with correct cpu affinity */
  96 /* Called via smp_call_function_single() */
  97 static void threshold_restart_bank(void *_tr)
  98 {
  99         struct thresh_restart *tr = _tr;
 100         u32 mci_misc_hi, mci_misc_lo;
 101
 102         rdmsr(tr->b->address, mci_misc_lo, mci_misc_hi);
 103
 104         if (tr->b->threshold_limit < (mci_misc_hi & THRESHOLD_MAX))
 105                 tr->reset = 1;  /* limit cannot be lower than err count */
 106
 107         if (tr->reset) {                /* reset err count and overflow bit */
 108                 mci_misc_hi =
 109                     (mci_misc_hi & ~(MASK_ERR_COUNT_HI | MASK_OVERFLOW_HI)) |
 110                     (THRESHOLD_MAX - tr->b->threshold_limit);
 111         } else if (tr->old_limit) {     /* change limit w/o reset */
 112                 int new_count = (mci_misc_hi & THRESHOLD_MAX) +
 113                     (tr->old_limit - tr->b->threshold_limit);
 114
 115                 mci_misc_hi = (mci_misc_hi & ~MASK_ERR_COUNT_HI) |
 116                     (new_count & THRESHOLD_MAX);
 117         }
 118
 119         tr->b->interrupt_enable ?
 120             (mci_misc_hi = (mci_misc_hi & ~MASK_INT_TYPE_HI) | INT_TYPE_APIC) :
 121             (mci_misc_hi &= ~MASK_INT_TYPE_HI);
 122
 123         mci_misc_hi |= MASK_COUNT_EN_HI;
 124         wrmsr(tr->b->address, mci_misc_lo, mci_misc_hi);
 125 }
 126
 127 /* cpu init entry point, called from mce.c with preempt off */
 128 void mce_amd_feature_init(struct cpuinfo_x86 *c)
 129 {
 130         unsigned int cpu = smp_processor_id();
 131         u32 low = 0, high = 0, address = 0;
 132         unsigned int bank, block;
 133         struct thresh_restart tr;
 134         u8 lvt_off;
 135
 136         for (bank = 0; bank < NR_BANKS; ++bank) {
 137                 for (block = 0; block < NR_BLOCKS; ++block) {
 138                         if (block == 0)
 139                                 address = MSR_IA32_MC0_MISC + bank * 4;
 140                         else if (block == 1) {
 141                                 address = (low & MASK_BLKPTR_LO) >> 21;
 142                                 if (!address)
 143                                         break;
 144
 145                                 address += MCG_XBLK_ADDR;
 146                         } else
 147                                 ++address;
 148
 149                         if (rdmsr_safe(address, &low, &high))
 150                                 break;
 151
 152                         if (!(high & MASK_VALID_HI))
 153                                 continue;
 154
 155                         if (!(high & MASK_CNTP_HI)  ||
 156                              (high & MASK_LOCKED_HI))
 157                                 continue;
 158
 159                         if (!block)
 160                                 per_cpu(bank_map, cpu) |= (1 << bank);
 161 #ifdef CONFIG_SMP
 162                         if (shared_bank[bank] && c->cpu_core_id)
 163                                 break;
 164 #endif
 165                         lvt_off = setup_APIC_eilvt_mce(THRESHOLD_APIC_VECTOR,
 166                                                        APIC_EILVT_MSG_FIX, 0);
 167
 168                         high &= ~MASK_LVTOFF_HI;
 169                         high |= lvt_off << 20;
 170                         wrmsr(address, low, high);
 171
 172                         threshold_defaults.address = address;
 173                         tr.b = &threshold_defaults;
 174                         tr.reset = 0;
 175                         tr.old_limit = 0;
 176                         threshold_restart_bank(&tr);
 177
 178                         mce_threshold_vector = amd_threshold_interrupt;
 179                 }
 180         }
 181 }
 182
 183 /*
 184  * APIC Interrupt Handler
 185  */
 186
 187 /*
 188  * threshold interrupt handler will service THRESHOLD_APIC_VECTOR.
 189  * the interrupt goes off when error_count reaches threshold_limit.
 190  * the handler will simply log mcelog w/ software defined bank number.
 191  */
 192 static void amd_threshold_interrupt(void)
 193 {
 194         u32 low = 0, high = 0, address = 0;
 195         unsigned int bank, block;
 196         struct mce m;
 197
 198         mce_setup(&m);
 199
 200         /* assume first bank caused it */
 201         for (bank = 0; bank < NR_BANKS; ++bank) {
 202                 if (!(per_cpu(bank_map, m.cpu) & (1 << bank)))
 203                         continue;
 204                 for (block = 0; block < NR_BLOCKS; ++block) {
 205                         if (block == 0) {
 206                                 address = MSR_IA32_MC0_MISC + bank * 4;
 207                         } else if (block == 1) {
 208                                 address = (low & MASK_BLKPTR_LO) >> 21;
 209                                 if (!address)
 210                                         break;
 211                                 address += MCG_XBLK_ADDR;
 212                         } else {
 213                                 ++address;
 214                         }
 215
 216                         if (rdmsr_safe(address, &low, &high))
 217                                 break;
 218
 219                         if (!(high & MASK_VALID_HI)) {
 220                                 if (block)
 221                                         continue;
 222                                 else
 223                                         break;
 224                         }
 225
 226                         if (!(high & MASK_CNTP_HI)  ||
 227                              (high & MASK_LOCKED_HI))
 228                                 continue;
 229
 230                         /*
 231                          * Log the machine check that caused the threshold
 232                          * event.
 233                          */
 234                         machine_check_poll(MCP_TIMESTAMP,
 235                                         &__get_cpu_var(mce_poll_banks));
 236
 237                         if (high & MASK_OVERFLOW_HI) {
 238                                 rdmsrl(address, m.misc);
 239                                 rdmsrl(MSR_IA32_MC0_STATUS + bank * 4,
 240                                        m.status);
 241                                 m.bank = K8_MCE_THRESHOLD_BASE
 242                                        + bank * NR_BLOCKS
 243                                        + block;
 244                                 mce_log(&m);
 245                                 return;
 246                         }
 247                 }
 248         }
 249 }
 250
 251 /*
 252  * Sysfs Interface
 253  */
 254
 255 struct threshold_attr {
 256         struct attribute attr;
 257         ssize_t (*show) (struct threshold_block *, char *);
 258         ssize_t (*store) (struct threshold_block *, const char *, size_t count);
 259 };
 260
 261 #define SHOW_FIELDS(name)                                               \
 262 static ssize_t show_ ## name(struct threshold_block *b, char *buf)      \
 263 {                                                                       \
 264         return sprintf(buf, "%lx\n", (unsigned long) b->name);          \
 265 }
 266 SHOW_FIELDS(interrupt_enable)
 267 SHOW_FIELDS(threshold_limit)
 268
 269 static ssize_t
 270 store_interrupt_enable(struct threshold_block *b, const char *buf, size_t size)
 271 {
 272         struct thresh_restart tr;
 273         unsigned long new;
 274
 275         if (strict_strtoul(buf, 0, &new) < 0)
 276                 return -EINVAL;
 277
 278         b->interrupt_enable = !!new;
 279
 280         tr.b            = b;
 281         tr.reset        = 0;
 282         tr.old_limit    = 0;
 283
 284         smp_call_function_single(b->cpu, threshold_restart_bank, &tr, 1);
 285
 286         return size;
 287 }
 288
 289 static ssize_t
 290 store_threshold_limit(struct threshold_block *b, const char *buf, size_t size)
 291 {
 292         struct thresh_restart tr;
 293         unsigned long new;
 294
 295         if (strict_strtoul(buf, 0, &new) < 0)
 296                 return -EINVAL;
 297
 298         if (new > THRESHOLD_MAX)
 299                 new = THRESHOLD_MAX;
 300         if (new < 1)
 301                 new = 1;
 302
 303         tr.old_limit = b->threshold_limit;
 304         b->threshold_limit = new;
 305         tr.b = b;
 306         tr.reset = 0;
 307
 308         smp_call_function_single(b->cpu, threshold_restart_bank, &tr, 1);
 309
 310         return size;
 311 }
 312
 313 struct threshold_block_cross_cpu {
 314         struct threshold_block  *tb;
 315         long                    retval;
 316 };
 317
 318 static void local_error_count_handler(void *_tbcc)
 319 {
 320         struct threshold_block_cross_cpu *tbcc = _tbcc;
 321         struct threshold_block *b = tbcc->tb;
 322         u32 low, high;
 323
 324         rdmsr(b->address, low, high);
 325         tbcc->retval = (high & 0xFFF) - (THRESHOLD_MAX - b->threshold_limit);
 326 }
 327
 328 static ssize_t show_error_count(struct threshold_block *b, char *buf)
 329 {
 330         struct threshold_block_cross_cpu tbcc = { .tb = b, };
 331
 332         smp_call_function_single(b->cpu, local_error_count_handler, &tbcc, 1);
 333         return sprintf(buf, "%lx\n", tbcc.retval);
 334 }
 335
 336 static ssize_t store_error_count(struct threshold_block *b,
 337                                  const char *buf, size_t count)
 338 {
 339         struct thresh_restart tr = { .b = b, .reset = 1, .old_limit = 0 };
 340
 341         smp_call_function_single(b->cpu, threshold_restart_bank, &tr, 1);
 342         return 1;
 343 }
 344
 345 #define RW_ATTR(val)                                                    \
 346 static struct threshold_attr val = {                                    \
 347         .attr   = {.name = __stringify(val), .mode = 0644 },            \
 348         .show   = show_## val,                                          \
 349         .store  = store_## val,                                         \
 350 };
 351
 352 RW_ATTR(interrupt_enable);
 353 RW_ATTR(threshold_limit);
 354 RW_ATTR(error_count);
 355
 356 static struct attribute *default_attrs[] = {
 357         &interrupt_enable.attr,
 358         &threshold_limit.attr,
 359         &error_count.attr,
 360         NULL
 361 };
 362
 363 #define to_block(k)     container_of(k, struct threshold_block, kobj)
 364 #define to_attr(a)      container_of(a, struct threshold_attr, attr)
 365
 366 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 367 {
 368         struct threshold_block *b = to_block(kobj);
 369         struct threshold_attr *a = to_attr(attr);
 370         ssize_t ret;
 371
 372         ret = a->show ? a->show(b, buf) : -EIO;
 373
 374         return ret;
 375 }
 376
 377 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 378                      const char *buf, size_t count)
 379 {
 380         struct threshold_block *b = to_block(kobj);
 381         struct threshold_attr *a = to_attr(attr);
 382         ssize_t ret;
 383
 384         ret = a->store ? a->store(b, buf, count) : -EIO;
 385
 386         return ret;
 387 }
 388
 389 static const struct sysfs_ops threshold_ops = {
 390         .show                   = show,
 391         .store                  = store,
 392 };
 393
 394 static struct kobj_type threshold_ktype = {
 395         .sysfs_ops              = &threshold_ops,
 396         .default_attrs          = default_attrs,
 397 };
 398
 399 static __cpuinit int allocate_threshold_blocks(unsigned int cpu,
 400                                                unsigned int bank,
 401                                                unsigned int block,
 402                                                u32 address)
 403 {
 404         struct threshold_block *b = NULL;
 405         u32 low, high;
 406         int err;
 407
 408         if ((bank >= NR_BANKS) || (block >= NR_BLOCKS))
 409                 return 0;
 410
 411         if (rdmsr_safe_on_cpu(cpu, address, &low, &high))
 412                 return 0;
 413
 414         if (!(high & MASK_VALID_HI)) {
 415                 if (block)
 416                         goto recurse;
 417                 else
 418                         return 0;
 419         }
 420
 421         if (!(high & MASK_CNTP_HI)  ||
 422              (high & MASK_LOCKED_HI))
 423                 goto recurse;
 424
 425         b = kzalloc(sizeof(struct threshold_block), GFP_KERNEL);
 426         if (!b)
 427                 return -ENOMEM;
 428
 429         b->block                = block;
 430         b->bank                 = bank;
 431         b->cpu                  = cpu;
 432         b->address              = address;
 433         b->interrupt_enable     = 0;
 434         b->threshold_limit      = THRESHOLD_MAX;
 435
 436         INIT_LIST_HEAD(&b->miscj);
 437
 438         if (per_cpu(threshold_banks, cpu)[bank]->blocks) {
 439                 list_add(&b->miscj,
 440                          &per_cpu(threshold_banks, cpu)[bank]->blocks->miscj);
 441         } else {
 442                 per_cpu(threshold_banks, cpu)[bank]->blocks = b;
 443         }
 444
 445         err = kobject_init_and_add(&b->kobj, &threshold_ktype,
 446                                    per_cpu(threshold_banks, cpu)[bank]->kobj,
 447                                    "misc%i", block);
 448         if (err)
 449                 goto out_free;
 450 recurse:
 451         if (!block) {
 452                 address = (low & MASK_BLKPTR_LO) >> 21;
 453                 if (!address)
 454                         return 0;
 455                 address += MCG_XBLK_ADDR;
 456         } else {
 457                 ++address;
 458         }
 459
 460         err = allocate_threshold_blocks(cpu, bank, ++block, address);
 461         if (err)
 462                 goto out_free;
 463
 464         if (b)
 465                 kobject_uevent(&b->kobj, KOBJ_ADD);
 466
 467         return err;
 468
 469 out_free:
 470         if (b) {
 471                 kobject_put(&b->kobj);
 472                 kfree(b);
 473         }
 474         return err;
 475 }
 476
 477 static __cpuinit long
 478 local_allocate_threshold_blocks(int cpu, unsigned int bank)
 479 {
 480         return allocate_threshold_blocks(cpu, bank, 0,
 481                                          MSR_IA32_MC0_MISC + bank * 4);
 482 }
 483
 484 /* symlinks sibling shared banks to first core.  first core owns dir/files. */
 485 static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank)
 486 {
 487         int i, err = 0;
 488         struct threshold_bank *b = NULL;
 489         char name[32];
 490 #ifdef CONFIG_SMP
 491         struct cpuinfo_x86 *c = &cpu_data(cpu);
 492 #endif
 493
 494         sprintf(name, "threshold_bank%i", bank);
 495
 496 #ifdef CONFIG_SMP
 497         if (cpu_data(cpu).cpu_core_id && shared_bank[bank]) {   /* symlink */
 498                 i = cpumask_first(c->llc_shared_map);
 499
 500                 /* first core not up yet */
 501                 if (cpu_data(i).cpu_core_id)
 502                         goto out;
 503
 504                 /* already linked */
 505                 if (per_cpu(threshold_banks, cpu)[bank])
 506                         goto out;
 507
 508                 b = per_cpu(threshold_banks, i)[bank];
 509
 510                 if (!b)
 511                         goto out;
 512
 513                 err = sysfs_create_link(&per_cpu(mce_dev, cpu).kobj,
 514                                         b->kobj, name);
 515                 if (err)
 516                         goto out;
 517
 518                 cpumask_copy(b->cpus, c->llc_shared_map);
 519                 per_cpu(threshold_banks, cpu)[bank] = b;
 520
 521                 goto out;
 522         }
 523 #endif
 524
 525         b = kzalloc(sizeof(struct threshold_bank), GFP_KERNEL);
 526         if (!b) {
 527                 err = -ENOMEM;
 528                 goto out;
 529         }
 530         if (!zalloc_cpumask_var(&b->cpus, GFP_KERNEL)) {
 531                 kfree(b);
 532                 err = -ENOMEM;
 533                 goto out;
 534         }
 535
 536         b->kobj = kobject_create_and_add(name, &per_cpu(mce_dev, cpu).kobj);
 537         if (!b->kobj)
 538                 goto out_free;
 539
 540 #ifndef CONFIG_SMP
 541         cpumask_setall(b->cpus);
 542 #else
 543         cpumask_set_cpu(cpu, b->cpus);
 544 #endif
 545
 546         per_cpu(threshold_banks, cpu)[bank] = b;
 547
 548         err = local_allocate_threshold_blocks(cpu, bank);
 549         if (err)
 550                 goto out_free;
 551
 552         for_each_cpu(i, b->cpus) {
 553                 if (i == cpu)
 554                         continue;
 555
 556                 err = sysfs_create_link(&per_cpu(mce_dev, i).kobj,
 557                                         b->kobj, name);
 558                 if (err)
 559                         goto out;
 560
 561                 per_cpu(threshold_banks, i)[bank] = b;
 562         }
 563
 564         goto out;
 565
 566 out_free:
 567         per_cpu(threshold_banks, cpu)[bank] = NULL;
 568         free_cpumask_var(b->cpus);
 569         kfree(b);
 570 out:
 571         return err;
 572 }
 573
 574 /* create dir/files for all valid threshold banks */
 575 static __cpuinit int threshold_create_device(unsigned int cpu)
 576 {
 577         unsigned int bank;
 578         int err = 0;
 579
 580         for (bank = 0; bank < NR_BANKS; ++bank) {
 581                 if (!(per_cpu(bank_map, cpu) & (1 << bank)))
 582                         continue;
 583                 err = threshold_create_bank(cpu, bank);
 584                 if (err)
 585                         goto out;
 586         }
 587 out:
 588         return err;
 589 }
 590
 591 /*
 592  * let's be hotplug friendly.
 593  * in case of multiple core processors, the first core always takes ownership
 594  *   of shared sysfs dir/files, and rest of the cores will be symlinked to it.
 595  */
 596
 597 static void deallocate_threshold_block(unsigned int cpu,
 598                                                  unsigned int bank)
 599 {
 600         struct threshold_block *pos = NULL;
 601         struct threshold_block *tmp = NULL;
 602         struct threshold_bank *head = per_cpu(threshold_banks, cpu)[bank];
 603
 604         if (!head)
 605                 return;
 606
 607         list_for_each_entry_safe(pos, tmp, &head->blocks->miscj, miscj) {
 608                 kobject_put(&pos->kobj);
 609                 list_del(&pos->miscj);
 610                 kfree(pos);
 611         }
 612
 613         kfree(per_cpu(threshold_banks, cpu)[bank]->blocks);
 614         per_cpu(threshold_banks, cpu)[bank]->blocks = NULL;
 615 }
 616
 617 static void threshold_remove_bank(unsigned int cpu, int bank)
 618 {
 619         struct threshold_bank *b;
 620         char name[32];
 621         int i = 0;
 622
 623         b = per_cpu(threshold_banks, cpu)[bank];
 624         if (!b)
 625                 return;
 626         if (!b->blocks)
 627                 goto free_out;
 628
 629         sprintf(name, "threshold_bank%i", bank);
 630
 631 #ifdef CONFIG_SMP
 632         /* sibling symlink */
 633         if (shared_bank[bank] && b->blocks->cpu != cpu) {
 634                 sysfs_remove_link(&per_cpu(mce_dev, cpu).kobj, name);
 635                 per_cpu(threshold_banks, cpu)[bank] = NULL;
 636
 637                 return;
 638         }
 639 #endif
 640
 641         /* remove all sibling symlinks before unregistering */
 642         for_each_cpu(i, b->cpus) {
 643                 if (i == cpu)
 644                         continue;
 645
 646                 sysfs_remove_link(&per_cpu(mce_dev, i).kobj, name);
 647                 per_cpu(threshold_banks, i)[bank] = NULL;
 648         }
 649
 650         deallocate_threshold_block(cpu, bank);
 651
 652 free_out:
 653         kobject_del(b->kobj);
 654         kobject_put(b->kobj);
 655         free_cpumask_var(b->cpus);
 656         kfree(b);
 657         per_cpu(threshold_banks, cpu)[bank] = NULL;
 658 }
 659
 660 static void threshold_remove_device(unsigned int cpu)
 661 {
 662         unsigned int bank;
 663
 664         for (bank = 0; bank < NR_BANKS; ++bank) {
 665                 if (!(per_cpu(bank_map, cpu) & (1 << bank)))
 666                         continue;
 667                 threshold_remove_bank(cpu, bank);
 668         }
 669 }
 670
 671 /* get notified when a cpu comes on/off */
 672 static void __cpuinit
 673 amd_64_threshold_cpu_callback(unsigned long action, unsigned int cpu)
 674 {
 675         switch (action) {
 676         case CPU_ONLINE:
 677         case CPU_ONLINE_FROZEN:
 678                 threshold_create_device(cpu);
 679                 break;
 680         case CPU_DEAD:
 681         case CPU_DEAD_FROZEN:
 682                 threshold_remove_device(cpu);
 683                 break;
 684         default:
 685                 break;
 686         }
 687 }
 688
 689 static __init int threshold_init_device(void)
 690 {
 691         unsigned lcpu = 0;
 692
 693         /* to hit CPUs online before the notifier is up */
 694         for_each_online_cpu(lcpu) {
 695                 int err = threshold_create_device(lcpu);
 696
 697                 if (err)
 698                         return err;
 699         }
 700         threshold_cpu_callback = amd_64_threshold_cpu_callback;
 701
 702         return 0;
 703 }
 704 device_initcall(threshold_init_device);