arch/x86/kernel/cpu/mcheck/mce_amd.c

   1 /*
   2  *  (c) 2005, 2006 Advanced Micro Devices, Inc.
   3  *  Your use of this code is subject to the terms and conditions of the
   4  *  GNU general public license version 2. See "COPYING" or
   5  *  http://www.gnu.org/licenses/gpl.html
   6  *
   7  *  Written by Jacob Shin - AMD, Inc.
   8  *
   9  *  Support : jacob.shin@amd.com
  10  *
  11  *  April 2006
  12  *     - added support for AMD Family 0x10 processors
  13  *
  14  *  All MC4_MISCi registers are shared between multi-cores
  15  */
  16 #include <linux/interrupt.h>
  17 #include <linux/notifier.h>
  18 #include <linux/kobject.h>
  19 #include <linux/percpu.h>
  20 #include <linux/sysdev.h>
  21 #include <linux/errno.h>
  22 #include <linux/sched.h>
  23 #include <linux/sysfs.h>
  24 #include <linux/slab.h>
  25 #include <linux/init.h>
  26 #include <linux/cpu.h>
  27 #include <linux/smp.h>
  28
  29 #include <asm/apic.h>
  30 #include <asm/idle.h>
  31 #include <asm/mce.h>
  32 #include <asm/msr.h>
  33
  34 #define PFX               "mce_threshold: "
  35 #define VERSION           "version 1.1.1"
  36 #define NR_BANKS          6
  37 #define NR_BLOCKS         9
  38 #define THRESHOLD_MAX     0xFFF
  39 #define INT_TYPE_APIC     0x00020000
  40 #define MASK_VALID_HI     0x80000000
  41 #define MASK_CNTP_HI      0x40000000
  42 #define MASK_LOCKED_HI    0x20000000
  43 #define MASK_LVTOFF_HI    0x00F00000
  44 #define MASK_COUNT_EN_HI  0x00080000
  45 #define MASK_INT_TYPE_HI  0x00060000
  46 #define MASK_OVERFLOW_HI  0x00010000
  47 #define MASK_ERR_COUNT_HI 0x00000FFF
  48 #define MASK_BLKPTR_LO    0xFF000000
  49 #define MCG_XBLK_ADDR     0xC0000400
  50
  51 struct threshold_block {
  52         unsigned int            block;
  53         unsigned int            bank;
  54         unsigned int            cpu;
  55         u32                     address;
  56         u16                     interrupt_enable;
  57         u16                     threshold_limit;
  58         struct kobject          kobj;
  59         struct list_head        miscj;
  60 };
  61
  62 /* defaults used early on boot */
  63 static struct threshold_block threshold_defaults = {
  64         .interrupt_enable       = 0,
  65         .threshold_limit        = THRESHOLD_MAX,
  66 };
  67
  68 struct threshold_bank {
  69         struct kobject          *kobj;
  70         struct threshold_block  *blocks;
  71         cpumask_var_t           cpus;
  72 };
  73 static DEFINE_PER_CPU(struct threshold_bank * [NR_BANKS], threshold_banks);
  74
  75 #ifdef CONFIG_SMP
  76 static unsigned char shared_bank[NR_BANKS] = {
  77         0, 0, 0, 0, 1
  78 };
  79 #endif
  80
  81 static DEFINE_PER_CPU(unsigned char, bank_map); /* see which banks are on */
  82
  83 static void amd_threshold_interrupt(void);
  84
  85 /*
  86  * CPU Initialization
  87  */
  88
  89 struct thresh_restart {
  90         struct threshold_block  *b;
  91         int                     reset;
  92         u16                     old_limit;
  93 };
  94
  95 /* must be called with correct cpu affinity */
  96 /* Called via smp_call_function_single() */
  97 static void threshold_restart_bank(void *_tr)
  98 {
  99         struct thresh_restart *tr = _tr;
 100         u32 mci_misc_hi, mci_misc_lo;
 101
 102         rdmsr(tr->b->address, mci_misc_lo, mci_misc_hi);
 103
 104         if (tr->b->threshold_limit < (mci_misc_hi & THRESHOLD_MAX))
 105                 tr->reset = 1;  /* limit cannot be lower than err count */
 106
 107         if (tr->reset) {                /* reset err count and overflow bit */
 108                 mci_misc_hi =
 109                     (mci_misc_hi & ~(MASK_ERR_COUNT_HI | MASK_OVERFLOW_HI)) |
 110                     (THRESHOLD_MAX - tr->b->threshold_limit);
 111         } else if (tr->old_limit) {     /* change limit w/o reset */
 112                 int new_count = (mci_misc_hi & THRESHOLD_MAX) +
 113                     (tr->old_limit - tr->b->threshold_limit);
 114
 115                 mci_misc_hi = (mci_misc_hi & ~MASK_ERR_COUNT_HI) |
 116                     (new_count & THRESHOLD_MAX);
 117         }
 118
 119         tr->b->interrupt_enable ?
 120             (mci_misc_hi = (mci_misc_hi & ~MASK_INT_TYPE_HI) | INT_TYPE_APIC) :
 121             (mci_misc_hi &= ~MASK_INT_TYPE_HI);
 122
 123         mci_misc_hi |= MASK_COUNT_EN_HI;
 124         wrmsr(tr->b->address, mci_misc_lo, mci_misc_hi);
 125 }
 126
 127 /* cpu init entry point, called from mce.c with preempt off */
 128 void mce_amd_feature_init(struct cpuinfo_x86 *c)
 129 {
 130         unsigned int cpu = smp_processor_id();
 131         u32 low = 0, high = 0, address = 0;
 132         unsigned int bank, block;
 133         struct thresh_restart tr;
 134         u8 lvt_off;
 135
 136         for (bank = 0; bank < NR_BANKS; ++bank) {
 137                 for (block = 0; block < NR_BLOCKS; ++block) {
 138                         if (block == 0)
 139                                 address = MSR_IA32_MC0_MISC + bank * 4;
 140                         else if (block == 1) {
 141                                 address = (low & MASK_BLKPTR_LO) >> 21;
 142                                 if (!address)
 143                                         break;
 144                                 address += MCG_XBLK_ADDR;
 145                         } else
 146                                 ++address;
 147
 148                         if (rdmsr_safe(address, &low, &high))
 149                                 break;
 150
 151                         if (!(high & MASK_VALID_HI)) {
 152                                 if (block)
 153                                         continue;
 154                                 else
 155                                         break;
 156                         }
 157
 158                         if (!(high & MASK_CNTP_HI)  ||
 159                              (high & MASK_LOCKED_HI))
 160                                 continue;
 161
 162                         if (!block)
 163                                 per_cpu(bank_map, cpu) |= (1 << bank);
 164 #ifdef CONFIG_SMP
 165                         if (shared_bank[bank] && c->cpu_core_id)
 166                                 break;
 167 #endif
 168                         lvt_off = setup_APIC_eilvt_mce(THRESHOLD_APIC_VECTOR,
 169                                                        APIC_EILVT_MSG_FIX, 0);
 170
 171                         high &= ~MASK_LVTOFF_HI;
 172                         high |= lvt_off << 20;
 173                         wrmsr(address, low, high);
 174
 175                         threshold_defaults.address = address;
 176                         tr.b = &threshold_defaults;
 177                         tr.reset = 0;
 178                         tr.old_limit = 0;
 179                         threshold_restart_bank(&tr);
 180
 181                         mce_threshold_vector = amd_threshold_interrupt;
 182                 }
 183         }
 184 }
 185
 186 /*
 187  * APIC Interrupt Handler
 188  */
 189
 190 /*
 191  * threshold interrupt handler will service THRESHOLD_APIC_VECTOR.
 192  * the interrupt goes off when error_count reaches threshold_limit.
 193  * the handler will simply log mcelog w/ software defined bank number.
 194  */
 195 static void amd_threshold_interrupt(void)
 196 {
 197         u32 low = 0, high = 0, address = 0;
 198         unsigned int bank, block;
 199         struct mce m;
 200
 201         mce_setup(&m);
 202
 203         /* assume first bank caused it */
 204         for (bank = 0; bank < NR_BANKS; ++bank) {
 205                 if (!(per_cpu(bank_map, m.cpu) & (1 << bank)))
 206                         continue;
 207                 for (block = 0; block < NR_BLOCKS; ++block) {
 208                         if (block == 0) {
 209                                 address = MSR_IA32_MC0_MISC + bank * 4;
 210                         } else if (block == 1) {
 211                                 address = (low & MASK_BLKPTR_LO) >> 21;
 212                                 if (!address)
 213                                         break;
 214                                 address += MCG_XBLK_ADDR;
 215                         } else {
 216                                 ++address;
 217                         }
 218
 219                         if (rdmsr_safe(address, &low, &high))
 220                                 break;
 221
 222                         if (!(high & MASK_VALID_HI)) {
 223                                 if (block)
 224                                         continue;
 225                                 else
 226                                         break;
 227                         }
 228
 229                         if (!(high & MASK_CNTP_HI)  ||
 230                              (high & MASK_LOCKED_HI))
 231                                 continue;
 232
 233                         /*
 234                          * Log the machine check that caused the threshold
 235                          * event.
 236                          */
 237                         machine_check_poll(MCP_TIMESTAMP,
 238                                         &__get_cpu_var(mce_poll_banks));
 239
 240                         if (high & MASK_OVERFLOW_HI) {
 241                                 rdmsrl(address, m.misc);
 242                                 rdmsrl(MSR_IA32_MC0_STATUS + bank * 4,
 243                                        m.status);
 244                                 m.bank = K8_MCE_THRESHOLD_BASE
 245                                        + bank * NR_BLOCKS
 246                                        + block;
 247                                 mce_log(&m);
 248                                 return;
 249                         }
 250                 }
 251         }
 252 }
 253
 254 /*
 255  * Sysfs Interface
 256  */
 257
 258 struct threshold_attr {
 259         struct attribute attr;
 260         ssize_t (*show) (struct threshold_block *, char *);
 261         ssize_t (*store) (struct threshold_block *, const char *, size_t count);
 262 };
 263
 264 #define SHOW_FIELDS(name)                                               \
 265 static ssize_t show_ ## name(struct threshold_block *b, char *buf)      \
 266 {                                                                       \
 267         return sprintf(buf, "%lx\n", (unsigned long) b->name);          \
 268 }
 269 SHOW_FIELDS(interrupt_enable)
 270 SHOW_FIELDS(threshold_limit)
 271
 272 static ssize_t
 273 store_interrupt_enable(struct threshold_block *b, const char *buf, size_t size)
 274 {
 275         struct thresh_restart tr;
 276         unsigned long new;
 277
 278         if (strict_strtoul(buf, 0, &new) < 0)
 279                 return -EINVAL;
 280
 281         b->interrupt_enable = !!new;
 282
 283         tr.b            = b;
 284         tr.reset        = 0;
 285         tr.old_limit    = 0;
 286
 287         smp_call_function_single(b->cpu, threshold_restart_bank, &tr, 1);
 288
 289         return size;
 290 }
 291
 292 static ssize_t
 293 store_threshold_limit(struct threshold_block *b, const char *buf, size_t size)
 294 {
 295         struct thresh_restart tr;
 296         unsigned long new;
 297
 298         if (strict_strtoul(buf, 0, &new) < 0)
 299                 return -EINVAL;
 300
 301         if (new > THRESHOLD_MAX)
 302                 new = THRESHOLD_MAX;
 303         if (new < 1)
 304                 new = 1;
 305
 306         tr.old_limit = b->threshold_limit;
 307         b->threshold_limit = new;
 308         tr.b = b;
 309         tr.reset = 0;
 310
 311         smp_call_function_single(b->cpu, threshold_restart_bank, &tr, 1);
 312
 313         return size;
 314 }
 315
 316 struct threshold_block_cross_cpu {
 317         struct threshold_block  *tb;
 318         long                    retval;
 319 };
 320
 321 static void local_error_count_handler(void *_tbcc)
 322 {
 323         struct threshold_block_cross_cpu *tbcc = _tbcc;
 324         struct threshold_block *b = tbcc->tb;
 325         u32 low, high;
 326
 327         rdmsr(b->address, low, high);
 328         tbcc->retval = (high & 0xFFF) - (THRESHOLD_MAX - b->threshold_limit);
 329 }
 330
 331 static ssize_t show_error_count(struct threshold_block *b, char *buf)
 332 {
 333         struct threshold_block_cross_cpu tbcc = { .tb = b, };
 334
 335         smp_call_function_single(b->cpu, local_error_count_handler, &tbcc, 1);
 336         return sprintf(buf, "%lx\n", tbcc.retval);
 337 }
 338
 339 static ssize_t store_error_count(struct threshold_block *b,
 340                                  const char *buf, size_t count)
 341 {
 342         struct thresh_restart tr = { .b = b, .reset = 1, .old_limit = 0 };
 343
 344         smp_call_function_single(b->cpu, threshold_restart_bank, &tr, 1);
 345         return 1;
 346 }
 347
 348 #define RW_ATTR(val)                                                    \
 349 static struct threshold_attr val = {                                    \
 350         .attr   = {.name = __stringify(val), .mode = 0644 },            \
 351         .show   = show_## val,                                          \
 352         .store  = store_## val,                                         \
 353 };
 354
 355 RW_ATTR(interrupt_enable);
 356 RW_ATTR(threshold_limit);
 357 RW_ATTR(error_count);
 358
 359 static struct attribute *default_attrs[] = {
 360         &interrupt_enable.attr,
 361         &threshold_limit.attr,
 362         &error_count.attr,
 363         NULL
 364 };
 365
 366 #define to_block(k)     container_of(k, struct threshold_block, kobj)
 367 #define to_attr(a)      container_of(a, struct threshold_attr, attr)
 368
 369 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 370 {
 371         struct threshold_block *b = to_block(kobj);
 372         struct threshold_attr *a = to_attr(attr);
 373         ssize_t ret;
 374
 375         ret = a->show ? a->show(b, buf) : -EIO;
 376
 377         return ret;
 378 }
 379
 380 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 381                      const char *buf, size_t count)
 382 {
 383         struct threshold_block *b = to_block(kobj);
 384         struct threshold_attr *a = to_attr(attr);
 385         ssize_t ret;
 386
 387         ret = a->store ? a->store(b, buf, count) : -EIO;
 388
 389         return ret;
 390 }
 391
 392 static const struct sysfs_ops threshold_ops = {
 393         .show                   = show,
 394         .store                  = store,
 395 };
 396
 397 static struct kobj_type threshold_ktype = {
 398         .sysfs_ops              = &threshold_ops,
 399         .default_attrs          = default_attrs,
 400 };
 401
 402 static __cpuinit int allocate_threshold_blocks(unsigned int cpu,
 403                                                unsigned int bank,
 404                                                unsigned int block,
 405                                                u32 address)
 406 {
 407         struct threshold_block *b = NULL;
 408         u32 low, high;
 409         int err;
 410
 411         if ((bank >= NR_BANKS) || (block >= NR_BLOCKS))
 412                 return 0;
 413
 414         if (rdmsr_safe_on_cpu(cpu, address, &low, &high))
 415                 return 0;
 416
 417         if (!(high & MASK_VALID_HI)) {
 418                 if (block)
 419                         goto recurse;
 420                 else
 421                         return 0;
 422         }
 423
 424         if (!(high & MASK_CNTP_HI)  ||
 425              (high & MASK_LOCKED_HI))
 426                 goto recurse;
 427
 428         b = kzalloc(sizeof(struct threshold_block), GFP_KERNEL);
 429         if (!b)
 430                 return -ENOMEM;
 431
 432         b->block                = block;
 433         b->bank                 = bank;
 434         b->cpu                  = cpu;
 435         b->address              = address;
 436         b->interrupt_enable     = 0;
 437         b->threshold_limit      = THRESHOLD_MAX;
 438
 439         INIT_LIST_HEAD(&b->miscj);
 440
 441         if (per_cpu(threshold_banks, cpu)[bank]->blocks) {
 442                 list_add(&b->miscj,
 443                          &per_cpu(threshold_banks, cpu)[bank]->blocks->miscj);
 444         } else {
 445                 per_cpu(threshold_banks, cpu)[bank]->blocks = b;
 446         }
 447
 448         err = kobject_init_and_add(&b->kobj, &threshold_ktype,
 449                                    per_cpu(threshold_banks, cpu)[bank]->kobj,
 450                                    "misc%i", block);
 451         if (err)
 452                 goto out_free;
 453 recurse:
 454         if (!block) {
 455                 address = (low & MASK_BLKPTR_LO) >> 21;
 456                 if (!address)
 457                         return 0;
 458                 address += MCG_XBLK_ADDR;
 459         } else {
 460                 ++address;
 461         }
 462
 463         err = allocate_threshold_blocks(cpu, bank, ++block, address);
 464         if (err)
 465                 goto out_free;
 466
 467         if (b)
 468                 kobject_uevent(&b->kobj, KOBJ_ADD);
 469
 470         return err;
 471
 472 out_free:
 473         if (b) {
 474                 kobject_put(&b->kobj);
 475                 kfree(b);
 476         }
 477         return err;
 478 }
 479
 480 static __cpuinit long
 481 local_allocate_threshold_blocks(int cpu, unsigned int bank)
 482 {
 483         return allocate_threshold_blocks(cpu, bank, 0,
 484                                          MSR_IA32_MC0_MISC + bank * 4);
 485 }
 486
 487 /* symlinks sibling shared banks to first core.  first core owns dir/files. */
 488 static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank)
 489 {
 490         int i, err = 0;
 491         struct threshold_bank *b = NULL;
 492         char name[32];
 493 #ifdef CONFIG_SMP
 494         struct cpuinfo_x86 *c = &cpu_data(cpu);
 495 #endif
 496
 497         sprintf(name, "threshold_bank%i", bank);
 498
 499 #ifdef CONFIG_SMP
 500         if (cpu_data(cpu).cpu_core_id && shared_bank[bank]) {   /* symlink */
 501                 i = cpumask_first(c->llc_shared_map);
 502
 503                 /* first core not up yet */
 504                 if (cpu_data(i).cpu_core_id)
 505                         goto out;
 506
 507                 /* already linked */
 508                 if (per_cpu(threshold_banks, cpu)[bank])
 509                         goto out;
 510
 511                 b = per_cpu(threshold_banks, i)[bank];
 512
 513                 if (!b)
 514                         goto out;
 515
 516                 err = sysfs_create_link(&per_cpu(mce_dev, cpu).kobj,
 517                                         b->kobj, name);
 518                 if (err)
 519                         goto out;
 520
 521                 cpumask_copy(b->cpus, c->llc_shared_map);
 522                 per_cpu(threshold_banks, cpu)[bank] = b;
 523
 524                 goto out;
 525         }
 526 #endif
 527
 528         b = kzalloc(sizeof(struct threshold_bank), GFP_KERNEL);
 529         if (!b) {
 530                 err = -ENOMEM;
 531                 goto out;
 532         }
 533         if (!alloc_cpumask_var(&b->cpus, GFP_KERNEL)) {
 534                 kfree(b);
 535                 err = -ENOMEM;
 536                 goto out;
 537         }
 538
 539         b->kobj = kobject_create_and_add(name, &per_cpu(mce_dev, cpu).kobj);
 540         if (!b->kobj)
 541                 goto out_free;
 542
 543 #ifndef CONFIG_SMP
 544         cpumask_setall(b->cpus);
 545 #else
 546         cpumask_copy(b->cpus, c->llc_shared_map);
 547 #endif
 548
 549         per_cpu(threshold_banks, cpu)[bank] = b;
 550
 551         err = local_allocate_threshold_blocks(cpu, bank);
 552         if (err)
 553                 goto out_free;
 554
 555         for_each_cpu(i, b->cpus) {
 556                 if (i == cpu)
 557                         continue;
 558
 559                 err = sysfs_create_link(&per_cpu(mce_dev, i).kobj,
 560                                         b->kobj, name);
 561                 if (err)
 562                         goto out;
 563
 564                 per_cpu(threshold_banks, i)[bank] = b;
 565         }
 566
 567         goto out;
 568
 569 out_free:
 570         per_cpu(threshold_banks, cpu)[bank] = NULL;
 571         free_cpumask_var(b->cpus);
 572         kfree(b);
 573 out:
 574         return err;
 575 }
 576
 577 /* create dir/files for all valid threshold banks */
 578 static __cpuinit int threshold_create_device(unsigned int cpu)
 579 {
 580         unsigned int bank;
 581         int err = 0;
 582
 583         for (bank = 0; bank < NR_BANKS; ++bank) {
 584                 if (!(per_cpu(bank_map, cpu) & (1 << bank)))
 585                         continue;
 586                 err = threshold_create_bank(cpu, bank);
 587                 if (err)
 588                         goto out;
 589         }
 590 out:
 591         return err;
 592 }
 593
 594 /*
 595  * let's be hotplug friendly.
 596  * in case of multiple core processors, the first core always takes ownership
 597  *   of shared sysfs dir/files, and rest of the cores will be symlinked to it.
 598  */
 599
 600 static void deallocate_threshold_block(unsigned int cpu,
 601                                                  unsigned int bank)
 602 {
 603         struct threshold_block *pos = NULL;
 604         struct threshold_block *tmp = NULL;
 605         struct threshold_bank *head = per_cpu(threshold_banks, cpu)[bank];
 606
 607         if (!head)
 608                 return;
 609
 610         list_for_each_entry_safe(pos, tmp, &head->blocks->miscj, miscj) {
 611                 kobject_put(&pos->kobj);
 612                 list_del(&pos->miscj);
 613                 kfree(pos);
 614         }
 615
 616         kfree(per_cpu(threshold_banks, cpu)[bank]->blocks);
 617         per_cpu(threshold_banks, cpu)[bank]->blocks = NULL;
 618 }
 619
 620 static void threshold_remove_bank(unsigned int cpu, int bank)
 621 {
 622         struct threshold_bank *b;
 623         char name[32];
 624         int i = 0;
 625
 626         b = per_cpu(threshold_banks, cpu)[bank];
 627         if (!b)
 628                 return;
 629         if (!b->blocks)
 630                 goto free_out;
 631
 632         sprintf(name, "threshold_bank%i", bank);
 633
 634 #ifdef CONFIG_SMP
 635         /* sibling symlink */
 636         if (shared_bank[bank] && b->blocks->cpu != cpu) {
 637                 sysfs_remove_link(&per_cpu(mce_dev, cpu).kobj, name);
 638                 per_cpu(threshold_banks, cpu)[bank] = NULL;
 639
 640                 return;
 641         }
 642 #endif
 643
 644         /* remove all sibling symlinks before unregistering */
 645         for_each_cpu(i, b->cpus) {
 646                 if (i == cpu)
 647                         continue;
 648
 649                 sysfs_remove_link(&per_cpu(mce_dev, i).kobj, name);
 650                 per_cpu(threshold_banks, i)[bank] = NULL;
 651         }
 652
 653         deallocate_threshold_block(cpu, bank);
 654
 655 free_out:
 656         kobject_del(b->kobj);
 657         kobject_put(b->kobj);
 658         free_cpumask_var(b->cpus);
 659         kfree(b);
 660         per_cpu(threshold_banks, cpu)[bank] = NULL;
 661 }
 662
 663 static void threshold_remove_device(unsigned int cpu)
 664 {
 665         unsigned int bank;
 666
 667         for (bank = 0; bank < NR_BANKS; ++bank) {
 668                 if (!(per_cpu(bank_map, cpu) & (1 << bank)))
 669                         continue;
 670                 threshold_remove_bank(cpu, bank);
 671         }
 672 }
 673
 674 /* get notified when a cpu comes on/off */
 675 static void __cpuinit
 676 amd_64_threshold_cpu_callback(unsigned long action, unsigned int cpu)
 677 {
 678         switch (action) {
 679         case CPU_ONLINE:
 680         case CPU_ONLINE_FROZEN:
 681                 threshold_create_device(cpu);
 682                 break;
 683         case CPU_DEAD:
 684         case CPU_DEAD_FROZEN:
 685                 threshold_remove_device(cpu);
 686                 break;
 687         default:
 688                 break;
 689         }
 690 }
 691
 692 static __init int threshold_init_device(void)
 693 {
 694         unsigned lcpu = 0;
 695
 696         /* to hit CPUs online before the notifier is up */
 697         for_each_online_cpu(lcpu) {
 698                 int err = threshold_create_device(lcpu);
 699
 700                 if (err)
 701                         return err;
 702         }
 703         threshold_cpu_callback = amd_64_threshold_cpu_callback;
 704
 705         return 0;
 706 }
 707 device_initcall(threshold_init_device);