arch/i386/kernel/nmi.c

   1 /*
   2  *  linux/arch/i386/nmi.c
   3  *
   4  *  NMI watchdog support on APIC systems
   5  *
   6  *  Started by Ingo Molnar <mingo@redhat.com>
   7  *
   8  *  Fixes:
   9  *  Mikael Pettersson   : AMD K7 support for local APIC NMI watchdog.
  10  *  Mikael Pettersson   : Power Management for local APIC NMI watchdog.
  11  *  Mikael Pettersson   : Pentium 4 support for local APIC NMI watchdog.
  12  *  Pavel Machek and
  13  *  Mikael Pettersson   : PM converted to driver model. Disable/enable API.
  14  */
  15
  16 #include <linux/delay.h>
  17 #include <linux/interrupt.h>
  18 #include <linux/module.h>
  19 #include <linux/nmi.h>
  20 #include <linux/sysdev.h>
  21 #include <linux/sysctl.h>
  22 #include <linux/percpu.h>
  23 #include <linux/dmi.h>
  24 #include <linux/kprobes.h>
  25 #include <linux/cpumask.h>
  26 #include <linux/kernel_stat.h>
  27
  28 #include <asm/smp.h>
  29 #include <asm/nmi.h>
  30 #include <asm/kdebug.h>
  31 #include <asm/intel_arch_perfmon.h>
  32
  33 #include "mach_traps.h"
  34
  35 int unknown_nmi_panic;
  36 int nmi_watchdog_enabled;
  37
  38 /* perfctr_nmi_owner tracks the ownership of the perfctr registers:
  39  * evtsel_nmi_owner tracks the ownership of the event selection
  40  * - different performance counters/ event selection may be reserved for
  41  *   different subsystems this reservation system just tries to coordinate
  42  *   things a little
  43  */
  44 static DEFINE_PER_CPU(unsigned long, perfctr_nmi_owner);
  45 static DEFINE_PER_CPU(unsigned long, evntsel_nmi_owner[3]);
  46
  47 static cpumask_t backtrace_mask = CPU_MASK_NONE;
  48
  49 /* this number is calculated from Intel's MSR_P4_CRU_ESCR5 register and it's
  50  * offset from MSR_P4_BSU_ESCR0.  It will be the max for all platforms (for now)
  51  */
  52 #define NMI_MAX_COUNTER_BITS 66
  53
  54 /* nmi_active:
  55  * >0: the lapic NMI watchdog is active, but can be disabled
  56  * <0: the lapic NMI watchdog has not been set up, and cannot
  57  *     be enabled
  58  *  0: the lapic NMI watchdog is disabled, but can be enabled
  59  */
  60 atomic_t nmi_active = ATOMIC_INIT(0);           /* oprofile uses this */
  61
  62 unsigned int nmi_watchdog = NMI_DEFAULT;
  63 static unsigned int nmi_hz = HZ;
  64
  65 struct nmi_watchdog_ctlblk {
  66         int enabled;
  67         u64 check_bit;
  68         unsigned int cccr_msr;
  69         unsigned int perfctr_msr;  /* the MSR to reset in NMI handler */
  70         unsigned int evntsel_msr;  /* the MSR to select the events to handle */
  71 };
  72 static DEFINE_PER_CPU(struct nmi_watchdog_ctlblk, nmi_watchdog_ctlblk);
  73
  74 /* local prototypes */
  75 static int unknown_nmi_panic_callback(struct pt_regs *regs, int cpu);
  76
  77 extern void show_registers(struct pt_regs *regs);
  78 extern int unknown_nmi_panic;
  79
  80 /* converts an msr to an appropriate reservation bit */
  81 static inline unsigned int nmi_perfctr_msr_to_bit(unsigned int msr)
  82 {
  83         /* returns the bit offset of the performance counter register */
  84         switch (boot_cpu_data.x86_vendor) {
  85         case X86_VENDOR_AMD:
  86                 return (msr - MSR_K7_PERFCTR0);
  87         case X86_VENDOR_INTEL:
  88                 if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
  89                         return (msr - MSR_ARCH_PERFMON_PERFCTR0);
  90
  91                 switch (boot_cpu_data.x86) {
  92                 case 6:
  93                         return (msr - MSR_P6_PERFCTR0);
  94                 case 15:
  95                         return (msr - MSR_P4_BPU_PERFCTR0);
  96                 }
  97         }
  98         return 0;
  99 }
 100
 101 /* converts an msr to an appropriate reservation bit */
 102 static inline unsigned int nmi_evntsel_msr_to_bit(unsigned int msr)
 103 {
 104         /* returns the bit offset of the event selection register */
 105         switch (boot_cpu_data.x86_vendor) {
 106         case X86_VENDOR_AMD:
 107                 return (msr - MSR_K7_EVNTSEL0);
 108         case X86_VENDOR_INTEL:
 109                 if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
 110                         return (msr - MSR_ARCH_PERFMON_EVENTSEL0);
 111
 112                 switch (boot_cpu_data.x86) {
 113                 case 6:
 114                         return (msr - MSR_P6_EVNTSEL0);
 115                 case 15:
 116                         return (msr - MSR_P4_BSU_ESCR0);
 117                 }
 118         }
 119         return 0;
 120 }
 121
 122 /* checks for a bit availability (hack for oprofile) */
 123 int avail_to_resrv_perfctr_nmi_bit(unsigned int counter)
 124 {
 125         BUG_ON(counter > NMI_MAX_COUNTER_BITS);
 126
 127         return (!test_bit(counter, &__get_cpu_var(perfctr_nmi_owner)));
 128 }
 129
 130 /* checks the an msr for availability */
 131 int avail_to_resrv_perfctr_nmi(unsigned int msr)
 132 {
 133         unsigned int counter;
 134
 135         counter = nmi_perfctr_msr_to_bit(msr);
 136         BUG_ON(counter > NMI_MAX_COUNTER_BITS);
 137
 138         return (!test_bit(counter, &__get_cpu_var(perfctr_nmi_owner)));
 139 }
 140
 141 int reserve_perfctr_nmi(unsigned int msr)
 142 {
 143         unsigned int counter;
 144
 145         counter = nmi_perfctr_msr_to_bit(msr);
 146         BUG_ON(counter > NMI_MAX_COUNTER_BITS);
 147
 148         if (!test_and_set_bit(counter, &__get_cpu_var(perfctr_nmi_owner)))
 149                 return 1;
 150         return 0;
 151 }
 152
 153 void release_perfctr_nmi(unsigned int msr)
 154 {
 155         unsigned int counter;
 156
 157         counter = nmi_perfctr_msr_to_bit(msr);
 158         BUG_ON(counter > NMI_MAX_COUNTER_BITS);
 159
 160         clear_bit(counter, &__get_cpu_var(perfctr_nmi_owner));
 161 }
 162
 163 int reserve_evntsel_nmi(unsigned int msr)
 164 {
 165         unsigned int counter;
 166
 167         counter = nmi_evntsel_msr_to_bit(msr);
 168         BUG_ON(counter > NMI_MAX_COUNTER_BITS);
 169
 170         if (!test_and_set_bit(counter, &__get_cpu_var(evntsel_nmi_owner)[0]))
 171                 return 1;
 172         return 0;
 173 }
 174
 175 void release_evntsel_nmi(unsigned int msr)
 176 {
 177         unsigned int counter;
 178
 179         counter = nmi_evntsel_msr_to_bit(msr);
 180         BUG_ON(counter > NMI_MAX_COUNTER_BITS);
 181
 182         clear_bit(counter, &__get_cpu_var(evntsel_nmi_owner)[0]);
 183 }
 184
 185 static __cpuinit inline int nmi_known_cpu(void)
 186 {
 187         switch (boot_cpu_data.x86_vendor) {
 188         case X86_VENDOR_AMD:
 189                 return ((boot_cpu_data.x86 == 15) || (boot_cpu_data.x86 == 6)
 190                         || (boot_cpu_data.x86 == 16));
 191         case X86_VENDOR_INTEL:
 192                 if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
 193                         return 1;
 194                 else
 195                         return ((boot_cpu_data.x86 == 15) || (boot_cpu_data.x86 == 6));
 196         }
 197         return 0;
 198 }
 199
 200 static int endflag __initdata = 0;
 201
 202 #ifdef CONFIG_SMP
 203 /* The performance counters used by NMI_LOCAL_APIC don't trigger when
 204  * the CPU is idle. To make sure the NMI watchdog really ticks on all
 205  * CPUs during the test make them busy.
 206  */
 207 static __init void nmi_cpu_busy(void *data)
 208 {
 209         local_irq_enable_in_hardirq();
 210         /* Intentionally don't use cpu_relax here. This is
 211            to make sure that the performance counter really ticks,
 212            even if there is a simulator or similar that catches the
 213            pause instruction. On a real HT machine this is fine because
 214            all other CPUs are busy with "useless" delay loops and don't
 215            care if they get somewhat less cycles. */
 216         while (endflag == 0)
 217                 mb();
 218 }
 219 #endif
 220
 221 static unsigned int adjust_for_32bit_ctr(unsigned int hz)
 222 {
 223         u64 counter_val;
 224         unsigned int retval = hz;
 225
 226         /*
 227          * On Intel CPUs with P6/ARCH_PERFMON only 32 bits in the counter
 228          * are writable, with higher bits sign extending from bit 31.
 229          * So, we can only program the counter with 31 bit values and
 230          * 32nd bit should be 1, for 33.. to be 1.
 231          * Find the appropriate nmi_hz
 232          */
 233         counter_val = (u64)cpu_khz * 1000;
 234         do_div(counter_val, retval);
 235         if (counter_val > 0x7fffffffULL) {
 236                 u64 count = (u64)cpu_khz * 1000;
 237                 do_div(count, 0x7fffffffUL);
 238                 retval = count + 1;
 239         }
 240         return retval;
 241 }
 242
 243 static int __init check_nmi_watchdog(void)
 244 {
 245         unsigned int *prev_nmi_count;
 246         int cpu;
 247
 248         if ((nmi_watchdog == NMI_NONE) || (nmi_watchdog == NMI_DEFAULT))
 249                 return 0;
 250
 251         if (!atomic_read(&nmi_active))
 252                 return 0;
 253
 254         prev_nmi_count = kmalloc(NR_CPUS * sizeof(int), GFP_KERNEL);
 255         if (!prev_nmi_count)
 256                 return -1;
 257
 258         printk(KERN_INFO "Testing NMI watchdog ... ");
 259
 260         if (nmi_watchdog == NMI_LOCAL_APIC)
 261                 smp_call_function(nmi_cpu_busy, (void *)&endflag, 0, 0);
 262
 263         for_each_possible_cpu(cpu)
 264                 prev_nmi_count[cpu] = per_cpu(irq_stat, cpu).__nmi_count;
 265         local_irq_enable();
 266         mdelay((10*1000)/nmi_hz); // wait 10 ticks
 267
 268         for_each_possible_cpu(cpu) {
 269 #ifdef CONFIG_SMP
 270                 /* Check cpu_callin_map here because that is set
 271                    after the timer is started. */
 272                 if (!cpu_isset(cpu, cpu_callin_map))
 273                         continue;
 274 #endif
 275                 if (!per_cpu(nmi_watchdog_ctlblk, cpu).enabled)
 276                         continue;
 277                 if (nmi_count(cpu) - prev_nmi_count[cpu] <= 5) {
 278                         printk("CPU#%d: NMI appears to be stuck (%d->%d)!\n",
 279                                 cpu,
 280                                 prev_nmi_count[cpu],
 281                                 nmi_count(cpu));
 282                         per_cpu(nmi_watchdog_ctlblk, cpu).enabled = 0;
 283                         atomic_dec(&nmi_active);
 284                 }
 285         }
 286         if (!atomic_read(&nmi_active)) {
 287                 kfree(prev_nmi_count);
 288                 atomic_set(&nmi_active, -1);
 289                 return -1;
 290         }
 291         endflag = 1;
 292         printk("OK.\n");
 293
 294         /* now that we know it works we can reduce NMI frequency to
 295            something more reasonable; makes a difference in some configs */
 296         if (nmi_watchdog == NMI_LOCAL_APIC) {
 297                 struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
 298
 299                 nmi_hz = 1;
 300
 301                 if (wd->perfctr_msr == MSR_P6_PERFCTR0 ||
 302                     wd->perfctr_msr == MSR_ARCH_PERFMON_PERFCTR0) {
 303                         nmi_hz = adjust_for_32bit_ctr(nmi_hz);
 304                 }
 305         }
 306
 307         kfree(prev_nmi_count);
 308         return 0;
 309 }
 310 /* This needs to happen later in boot so counters are working */
 311 late_initcall(check_nmi_watchdog);
 312
 313 static int __init setup_nmi_watchdog(char *str)
 314 {
 315         int nmi;
 316
 317         get_option(&str, &nmi);
 318
 319         if ((nmi >= NMI_INVALID) || (nmi < NMI_NONE))
 320                 return 0;
 321
 322         nmi_watchdog = nmi;
 323         return 1;
 324 }
 325
 326 __setup("nmi_watchdog=", setup_nmi_watchdog);
 327
 328 static void disable_lapic_nmi_watchdog(void)
 329 {
 330         BUG_ON(nmi_watchdog != NMI_LOCAL_APIC);
 331
 332         if (atomic_read(&nmi_active) <= 0)
 333                 return;
 334
 335         on_each_cpu(stop_apic_nmi_watchdog, NULL, 0, 1);
 336
 337         BUG_ON(atomic_read(&nmi_active) != 0);
 338 }
 339
 340 static void enable_lapic_nmi_watchdog(void)
 341 {
 342         BUG_ON(nmi_watchdog != NMI_LOCAL_APIC);
 343
 344         /* are we already enabled */
 345         if (atomic_read(&nmi_active) != 0)
 346                 return;
 347
 348         /* are we lapic aware */
 349         if (nmi_known_cpu() <= 0)
 350                 return;
 351
 352         on_each_cpu(setup_apic_nmi_watchdog, NULL, 0, 1);
 353         touch_nmi_watchdog();
 354 }
 355
 356 void disable_timer_nmi_watchdog(void)
 357 {
 358         BUG_ON(nmi_watchdog != NMI_IO_APIC);
 359
 360         if (atomic_read(&nmi_active) <= 0)
 361                 return;
 362
 363         disable_irq(0);
 364         on_each_cpu(stop_apic_nmi_watchdog, NULL, 0, 1);
 365
 366         BUG_ON(atomic_read(&nmi_active) != 0);
 367 }
 368
 369 void enable_timer_nmi_watchdog(void)
 370 {
 371         BUG_ON(nmi_watchdog != NMI_IO_APIC);
 372
 373         if (atomic_read(&nmi_active) == 0) {
 374                 touch_nmi_watchdog();
 375                 on_each_cpu(setup_apic_nmi_watchdog, NULL, 0, 1);
 376                 enable_irq(0);
 377         }
 378 }
 379
 380 static void __acpi_nmi_disable(void *__unused)
 381 {
 382         apic_write_around(APIC_LVT0, APIC_DM_NMI | APIC_LVT_MASKED);
 383 }
 384
 385 /*
 386  * Disable timer based NMIs on all CPUs:
 387  */
 388 void acpi_nmi_disable(void)
 389 {
 390         if (atomic_read(&nmi_active) && nmi_watchdog == NMI_IO_APIC)
 391                 on_each_cpu(__acpi_nmi_disable, NULL, 0, 1);
 392 }
 393
 394 static void __acpi_nmi_enable(void *__unused)
 395 {
 396         apic_write_around(APIC_LVT0, APIC_DM_NMI);
 397 }
 398
 399 /*
 400  * Enable timer based NMIs on all CPUs:
 401  */
 402 void acpi_nmi_enable(void)
 403 {
 404         if (atomic_read(&nmi_active) && nmi_watchdog == NMI_IO_APIC)
 405                 on_each_cpu(__acpi_nmi_enable, NULL, 0, 1);
 406 }
 407
 408 #ifdef CONFIG_PM
 409
 410 static int nmi_pm_active; /* nmi_active before suspend */
 411
 412 static int lapic_nmi_suspend(struct sys_device *dev, pm_message_t state)
 413 {
 414         /* only CPU0 goes here, other CPUs should be offline */
 415         nmi_pm_active = atomic_read(&nmi_active);
 416         stop_apic_nmi_watchdog(NULL);
 417         BUG_ON(atomic_read(&nmi_active) != 0);
 418         return 0;
 419 }
 420
 421 static int lapic_nmi_resume(struct sys_device *dev)
 422 {
 423         /* only CPU0 goes here, other CPUs should be offline */
 424         if (nmi_pm_active > 0) {
 425                 setup_apic_nmi_watchdog(NULL);
 426                 touch_nmi_watchdog();
 427         }
 428         return 0;
 429 }
 430
 431
 432 static struct sysdev_class nmi_sysclass = {
 433         set_kset_name("lapic_nmi"),
 434         .resume         = lapic_nmi_resume,
 435         .suspend        = lapic_nmi_suspend,
 436 };
 437
 438 static struct sys_device device_lapic_nmi = {
 439         .id     = 0,
 440         .cls    = &nmi_sysclass,
 441 };
 442
 443 static int __init init_lapic_nmi_sysfs(void)
 444 {
 445         int error;
 446
 447         /* should really be a BUG_ON but b/c this is an
 448          * init call, it just doesn't work.  -dcz
 449          */
 450         if (nmi_watchdog != NMI_LOCAL_APIC)
 451                 return 0;
 452
 453         if ( atomic_read(&nmi_active) < 0 )
 454                 return 0;
 455
 456         error = sysdev_class_register(&nmi_sysclass);
 457         if (!error)
 458                 error = sysdev_register(&device_lapic_nmi);
 459         return error;
 460 }
 461 /* must come after the local APIC's device_initcall() */
 462 late_initcall(init_lapic_nmi_sysfs);
 463
 464 #endif  /* CONFIG_PM */
 465
 466 /*
 467  * Activate the NMI watchdog via the local APIC.
 468  * Original code written by Keith Owens.
 469  */
 470
 471 static void write_watchdog_counter(unsigned int perfctr_msr, const char *descr)
 472 {
 473         u64 count = (u64)cpu_khz * 1000;
 474
 475         do_div(count, nmi_hz);
 476         if(descr)
 477                 Dprintk("setting %s to -0x%08Lx\n", descr, count);
 478         wrmsrl(perfctr_msr, 0 - count);
 479 }
 480
 481 static void write_watchdog_counter32(unsigned int perfctr_msr,
 482                 const char *descr)
 483 {
 484         u64 count = (u64)cpu_khz * 1000;
 485
 486         do_div(count, nmi_hz);
 487         if(descr)
 488                 Dprintk("setting %s to -0x%08Lx\n", descr, count);
 489         wrmsr(perfctr_msr, (u32)(-count), 0);
 490 }
 491
 492 /* Note that these events don't tick when the CPU idles. This means
 493    the frequency varies with CPU load. */
 494
 495 #define K7_EVNTSEL_ENABLE       (1 << 22)
 496 #define K7_EVNTSEL_INT          (1 << 20)
 497 #define K7_EVNTSEL_OS           (1 << 17)
 498 #define K7_EVNTSEL_USR          (1 << 16)
 499 #define K7_EVENT_CYCLES_PROCESSOR_IS_RUNNING    0x76
 500 #define K7_NMI_EVENT            K7_EVENT_CYCLES_PROCESSOR_IS_RUNNING
 501
 502 static int setup_k7_watchdog(void)
 503 {
 504         unsigned int perfctr_msr, evntsel_msr;
 505         unsigned int evntsel;
 506         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
 507
 508         perfctr_msr = MSR_K7_PERFCTR0;
 509         evntsel_msr = MSR_K7_EVNTSEL0;
 510         if (!reserve_perfctr_nmi(perfctr_msr))
 511                 goto fail;
 512
 513         if (!reserve_evntsel_nmi(evntsel_msr))
 514                 goto fail1;
 515
 516         wrmsrl(perfctr_msr, 0UL);
 517
 518         evntsel = K7_EVNTSEL_INT
 519                 | K7_EVNTSEL_OS
 520                 | K7_EVNTSEL_USR
 521                 | K7_NMI_EVENT;
 522
 523         /* setup the timer */
 524         wrmsr(evntsel_msr, evntsel, 0);
 525         write_watchdog_counter(perfctr_msr, "K7_PERFCTR0");
 526         apic_write(APIC_LVTPC, APIC_DM_NMI);
 527         evntsel |= K7_EVNTSEL_ENABLE;
 528         wrmsr(evntsel_msr, evntsel, 0);
 529
 530         wd->perfctr_msr = perfctr_msr;
 531         wd->evntsel_msr = evntsel_msr;
 532         wd->cccr_msr = 0;  //unused
 533         wd->check_bit = 1ULL<<63;
 534         return 1;
 535 fail1:
 536         release_perfctr_nmi(perfctr_msr);
 537 fail:
 538         return 0;
 539 }
 540
 541 static void stop_k7_watchdog(void)
 542 {
 543         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
 544
 545         wrmsr(wd->evntsel_msr, 0, 0);
 546
 547         release_evntsel_nmi(wd->evntsel_msr);
 548         release_perfctr_nmi(wd->perfctr_msr);
 549 }
 550
 551 #define P6_EVNTSEL0_ENABLE      (1 << 22)
 552 #define P6_EVNTSEL_INT          (1 << 20)
 553 #define P6_EVNTSEL_OS           (1 << 17)
 554 #define P6_EVNTSEL_USR          (1 << 16)
 555 #define P6_EVENT_CPU_CLOCKS_NOT_HALTED  0x79
 556 #define P6_NMI_EVENT            P6_EVENT_CPU_CLOCKS_NOT_HALTED
 557
 558 static int setup_p6_watchdog(void)
 559 {
 560         unsigned int perfctr_msr, evntsel_msr;
 561         unsigned int evntsel;
 562         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
 563
 564         perfctr_msr = MSR_P6_PERFCTR0;
 565         evntsel_msr = MSR_P6_EVNTSEL0;
 566         if (!reserve_perfctr_nmi(perfctr_msr))
 567                 goto fail;
 568
 569         if (!reserve_evntsel_nmi(evntsel_msr))
 570                 goto fail1;
 571
 572         wrmsrl(perfctr_msr, 0UL);
 573
 574         evntsel = P6_EVNTSEL_INT
 575                 | P6_EVNTSEL_OS
 576                 | P6_EVNTSEL_USR
 577                 | P6_NMI_EVENT;
 578
 579         /* setup the timer */
 580         wrmsr(evntsel_msr, evntsel, 0);
 581         nmi_hz = adjust_for_32bit_ctr(nmi_hz);
 582         write_watchdog_counter32(perfctr_msr, "P6_PERFCTR0");
 583         apic_write(APIC_LVTPC, APIC_DM_NMI);
 584         evntsel |= P6_EVNTSEL0_ENABLE;
 585         wrmsr(evntsel_msr, evntsel, 0);
 586
 587         wd->perfctr_msr = perfctr_msr;
 588         wd->evntsel_msr = evntsel_msr;
 589         wd->cccr_msr = 0;  //unused
 590         wd->check_bit = 1ULL<<39;
 591         return 1;
 592 fail1:
 593         release_perfctr_nmi(perfctr_msr);
 594 fail:
 595         return 0;
 596 }
 597
 598 static void stop_p6_watchdog(void)
 599 {
 600         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
 601
 602         wrmsr(wd->evntsel_msr, 0, 0);
 603
 604         release_evntsel_nmi(wd->evntsel_msr);
 605         release_perfctr_nmi(wd->perfctr_msr);
 606 }
 607
 608 /* Note that these events don't tick when the CPU idles. This means
 609    the frequency varies with CPU load. */
 610
 611 #define MSR_P4_MISC_ENABLE_PERF_AVAIL   (1<<7)
 612 #define P4_ESCR_EVENT_SELECT(N) ((N)<<25)
 613 #define P4_ESCR_OS              (1<<3)
 614 #define P4_ESCR_USR             (1<<2)
 615 #define P4_CCCR_OVF_PMI0        (1<<26)
 616 #define P4_CCCR_OVF_PMI1        (1<<27)
 617 #define P4_CCCR_THRESHOLD(N)    ((N)<<20)
 618 #define P4_CCCR_COMPLEMENT      (1<<19)
 619 #define P4_CCCR_COMPARE         (1<<18)
 620 #define P4_CCCR_REQUIRED        (3<<16)
 621 #define P4_CCCR_ESCR_SELECT(N)  ((N)<<13)
 622 #define P4_CCCR_ENABLE          (1<<12)
 623 #define P4_CCCR_OVF             (1<<31)
 624 /* Set up IQ_COUNTER0 to behave like a clock, by having IQ_CCCR0 filter
 625    CRU_ESCR0 (with any non-null event selector) through a complemented
 626    max threshold. [IA32-Vol3, Section 14.9.9] */
 627
 628 static int setup_p4_watchdog(void)
 629 {
 630         unsigned int perfctr_msr, evntsel_msr, cccr_msr;
 631         unsigned int evntsel, cccr_val;
 632         unsigned int misc_enable, dummy;
 633         unsigned int ht_num;
 634         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
 635
 636         rdmsr(MSR_IA32_MISC_ENABLE, misc_enable, dummy);
 637         if (!(misc_enable & MSR_P4_MISC_ENABLE_PERF_AVAIL))
 638                 return 0;
 639
 640 #ifdef CONFIG_SMP
 641         /* detect which hyperthread we are on */
 642         if (smp_num_siblings == 2) {
 643                 unsigned int ebx, apicid;
 644
 645                 ebx = cpuid_ebx(1);
 646                 apicid = (ebx >> 24) & 0xff;
 647                 ht_num = apicid & 1;
 648         } else
 649 #endif
 650                 ht_num = 0;
 651
 652         /* performance counters are shared resources
 653          * assign each hyperthread its own set
 654          * (re-use the ESCR0 register, seems safe
 655          * and keeps the cccr_val the same)
 656          */
 657         if (!ht_num) {
 658                 /* logical cpu 0 */
 659                 perfctr_msr = MSR_P4_IQ_PERFCTR0;
 660                 evntsel_msr = MSR_P4_CRU_ESCR0;
 661                 cccr_msr = MSR_P4_IQ_CCCR0;
 662                 cccr_val = P4_CCCR_OVF_PMI0 | P4_CCCR_ESCR_SELECT(4);
 663         } else {
 664                 /* logical cpu 1 */
 665                 perfctr_msr = MSR_P4_IQ_PERFCTR1;
 666                 evntsel_msr = MSR_P4_CRU_ESCR0;
 667                 cccr_msr = MSR_P4_IQ_CCCR1;
 668                 cccr_val = P4_CCCR_OVF_PMI1 | P4_CCCR_ESCR_SELECT(4);
 669         }
 670
 671         if (!reserve_perfctr_nmi(perfctr_msr))
 672                 goto fail;
 673
 674         if (!reserve_evntsel_nmi(evntsel_msr))
 675                 goto fail1;
 676
 677         evntsel = P4_ESCR_EVENT_SELECT(0x3F)
 678                 | P4_ESCR_OS
 679                 | P4_ESCR_USR;
 680
 681         cccr_val |= P4_CCCR_THRESHOLD(15)
 682                  | P4_CCCR_COMPLEMENT
 683                  | P4_CCCR_COMPARE
 684                  | P4_CCCR_REQUIRED;
 685
 686         wrmsr(evntsel_msr, evntsel, 0);
 687         wrmsr(cccr_msr, cccr_val, 0);
 688         write_watchdog_counter(perfctr_msr, "P4_IQ_COUNTER0");
 689         apic_write(APIC_LVTPC, APIC_DM_NMI);
 690         cccr_val |= P4_CCCR_ENABLE;
 691         wrmsr(cccr_msr, cccr_val, 0);
 692         wd->perfctr_msr = perfctr_msr;
 693         wd->evntsel_msr = evntsel_msr;
 694         wd->cccr_msr = cccr_msr;
 695         wd->check_bit = 1ULL<<39;
 696         return 1;
 697 fail1:
 698         release_perfctr_nmi(perfctr_msr);
 699 fail:
 700         return 0;
 701 }
 702
 703 static void stop_p4_watchdog(void)
 704 {
 705         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
 706
 707         wrmsr(wd->cccr_msr, 0, 0);
 708         wrmsr(wd->evntsel_msr, 0, 0);
 709
 710         release_evntsel_nmi(wd->evntsel_msr);
 711         release_perfctr_nmi(wd->perfctr_msr);
 712 }
 713
 714 #define ARCH_PERFMON_NMI_EVENT_SEL      ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL
 715 #define ARCH_PERFMON_NMI_EVENT_UMASK    ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK
 716
 717 static int setup_intel_arch_watchdog(void)
 718 {
 719         unsigned int ebx;
 720         union cpuid10_eax eax;
 721         unsigned int unused;
 722         unsigned int perfctr_msr, evntsel_msr;
 723         unsigned int evntsel;
 724         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
 725
 726         /*
 727          * Check whether the Architectural PerfMon supports
 728          * Unhalted Core Cycles Event or not.
 729          * NOTE: Corresponding bit = 0 in ebx indicates event present.
 730          */
 731         cpuid(10, &(eax.full), &ebx, &unused, &unused);
 732         if ((eax.split.mask_length < (ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX+1)) ||
 733             (ebx & ARCH_PERFMON_UNHALTED_CORE_CYCLES_PRESENT))
 734                 goto fail;
 735
 736         perfctr_msr = MSR_ARCH_PERFMON_PERFCTR0;
 737         evntsel_msr = MSR_ARCH_PERFMON_EVENTSEL0;
 738
 739         if (!reserve_perfctr_nmi(perfctr_msr))
 740                 goto fail;
 741
 742         if (!reserve_evntsel_nmi(evntsel_msr))
 743                 goto fail1;
 744
 745         wrmsrl(perfctr_msr, 0UL);
 746
 747         evntsel = ARCH_PERFMON_EVENTSEL_INT
 748                 | ARCH_PERFMON_EVENTSEL_OS
 749                 | ARCH_PERFMON_EVENTSEL_USR
 750                 | ARCH_PERFMON_NMI_EVENT_SEL
 751                 | ARCH_PERFMON_NMI_EVENT_UMASK;
 752
 753         /* setup the timer */
 754         wrmsr(evntsel_msr, evntsel, 0);
 755         nmi_hz = adjust_for_32bit_ctr(nmi_hz);
 756         write_watchdog_counter32(perfctr_msr, "INTEL_ARCH_PERFCTR0");
 757         apic_write(APIC_LVTPC, APIC_DM_NMI);
 758         evntsel |= ARCH_PERFMON_EVENTSEL0_ENABLE;
 759         wrmsr(evntsel_msr, evntsel, 0);
 760
 761         wd->perfctr_msr = perfctr_msr;
 762         wd->evntsel_msr = evntsel_msr;
 763         wd->cccr_msr = 0;  //unused
 764         wd->check_bit = 1ULL << (eax.split.bit_width - 1);
 765         return 1;
 766 fail1:
 767         release_perfctr_nmi(perfctr_msr);
 768 fail:
 769         return 0;
 770 }
 771
 772 static void stop_intel_arch_watchdog(void)
 773 {
 774         unsigned int ebx;
 775         union cpuid10_eax eax;
 776         unsigned int unused;
 777         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
 778
 779         /*
 780          * Check whether the Architectural PerfMon supports
 781          * Unhalted Core Cycles Event or not.
 782          * NOTE: Corresponding bit = 0 in ebx indicates event present.
 783          */
 784         cpuid(10, &(eax.full), &ebx, &unused, &unused);
 785         if ((eax.split.mask_length < (ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX+1)) ||
 786             (ebx & ARCH_PERFMON_UNHALTED_CORE_CYCLES_PRESENT))
 787                 return;
 788
 789         wrmsr(wd->evntsel_msr, 0, 0);
 790         release_evntsel_nmi(wd->evntsel_msr);
 791         release_perfctr_nmi(wd->perfctr_msr);
 792 }
 793
 794 void setup_apic_nmi_watchdog (void *unused)
 795 {
 796         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
 797
 798         /* only support LOCAL and IO APICs for now */
 799         if ((nmi_watchdog != NMI_LOCAL_APIC) &&
 800             (nmi_watchdog != NMI_IO_APIC))
 801                 return;
 802
 803         if (wd->enabled == 1)
 804                 return;
 805
 806         /* cheap hack to support suspend/resume */
 807         /* if cpu0 is not active neither should the other cpus */
 808         if ((smp_processor_id() != 0) && (atomic_read(&nmi_active) <= 0))
 809                 return;
 810
 811         if (nmi_watchdog == NMI_LOCAL_APIC) {
 812                 switch (boot_cpu_data.x86_vendor) {
 813                 case X86_VENDOR_AMD:
 814                         if (boot_cpu_data.x86 != 6 && boot_cpu_data.x86 != 15 &&
 815                                 boot_cpu_data.x86 != 16)
 816                                 return;
 817                         if (!setup_k7_watchdog())
 818                                 return;
 819                         break;
 820                 case X86_VENDOR_INTEL:
 821                         if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
 822                                 if (!setup_intel_arch_watchdog())
 823                                         return;
 824                                 break;
 825                         }
 826                         switch (boot_cpu_data.x86) {
 827                         case 6:
 828                                 if (boot_cpu_data.x86_model > 0xd)
 829                                         return;
 830
 831                                 if (!setup_p6_watchdog())
 832                                         return;
 833                                 break;
 834                         case 15:
 835                                 if (boot_cpu_data.x86_model > 0x4)
 836                                         return;
 837
 838                                 if (!setup_p4_watchdog())
 839                                         return;
 840                                 break;
 841                         default:
 842                                 return;
 843                         }
 844                         break;
 845                 default:
 846                         return;
 847                 }
 848         }
 849         wd->enabled = 1;
 850         atomic_inc(&nmi_active);
 851 }
 852
 853 void stop_apic_nmi_watchdog(void *unused)
 854 {
 855         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
 856
 857         /* only support LOCAL and IO APICs for now */
 858         if ((nmi_watchdog != NMI_LOCAL_APIC) &&
 859             (nmi_watchdog != NMI_IO_APIC))
 860                 return;
 861
 862         if (wd->enabled == 0)
 863                 return;
 864
 865         if (nmi_watchdog == NMI_LOCAL_APIC) {
 866                 switch (boot_cpu_data.x86_vendor) {
 867                 case X86_VENDOR_AMD:
 868                         stop_k7_watchdog();
 869                         break;
 870                 case X86_VENDOR_INTEL:
 871                         if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
 872                                 stop_intel_arch_watchdog();
 873                                 break;
 874                         }
 875                         switch (boot_cpu_data.x86) {
 876                         case 6:
 877                                 if (boot_cpu_data.x86_model > 0xd)
 878                                         break;
 879                                 stop_p6_watchdog();
 880                                 break;
 881                         case 15:
 882                                 if (boot_cpu_data.x86_model > 0x4)
 883                                         break;
 884                                 stop_p4_watchdog();
 885                                 break;
 886                         }
 887                         break;
 888                 default:
 889                         return;
 890                 }
 891         }
 892         wd->enabled = 0;
 893         atomic_dec(&nmi_active);
 894 }
 895
 896 /*
 897  * the best way to detect whether a CPU has a 'hard lockup' problem
 898  * is to check it's local APIC timer IRQ counts. If they are not
 899  * changing then that CPU has some problem.
 900  *
 901  * as these watchdog NMI IRQs are generated on every CPU, we only
 902  * have to check the current processor.
 903  *
 904  * since NMIs don't listen to _any_ locks, we have to be extremely
 905  * careful not to rely on unsafe variables. The printk might lock
 906  * up though, so we have to break up any console locks first ...
 907  * [when there will be more tty-related locks, break them up
 908  *  here too!]
 909  */
 910
 911 static unsigned int
 912         last_irq_sums [NR_CPUS],
 913         alert_counter [NR_CPUS];
 914
 915 void touch_nmi_watchdog (void)
 916 {
 917         if (nmi_watchdog > 0) {
 918                 unsigned cpu;
 919
 920                 /*
 921                  * Just reset the alert counters, (other CPUs might be
 922                  * spinning on locks we hold):
 923                  */
 924                 for_each_present_cpu (cpu)
 925                         alert_counter[cpu] = 0;
 926         }
 927
 928         /*
 929          * Tickle the softlockup detector too:
 930          */
 931         touch_softlockup_watchdog();
 932 }
 933 EXPORT_SYMBOL(touch_nmi_watchdog);
 934
 935 extern void die_nmi(struct pt_regs *, const char *msg);
 936
 937 __kprobes int nmi_watchdog_tick(struct pt_regs * regs, unsigned reason)
 938 {
 939
 940         /*
 941          * Since current_thread_info()-> is always on the stack, and we
 942          * always switch the stack NMI-atomically, it's safe to use
 943          * smp_processor_id().
 944          */
 945         unsigned int sum;
 946         int touched = 0;
 947         int cpu = smp_processor_id();
 948         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
 949         u64 dummy;
 950         int rc=0;
 951
 952         /* check for other users first */
 953         if (notify_die(DIE_NMI, "nmi", regs, reason, 2, SIGINT)
 954                         == NOTIFY_STOP) {
 955                 rc = 1;
 956                 touched = 1;
 957         }
 958
 959         if (cpu_isset(cpu, backtrace_mask)) {
 960                 static DEFINE_SPINLOCK(lock);   /* Serialise the printks */
 961
 962                 spin_lock(&lock);
 963                 printk("NMI backtrace for cpu %d\n", cpu);
 964                 dump_stack();
 965                 spin_unlock(&lock);
 966                 cpu_clear(cpu, backtrace_mask);
 967         }
 968
 969         /*
 970          * Take the local apic timer and PIT/HPET into account. We don't
 971          * know which one is active, when we have highres/dyntick on
 972          */
 973         sum = per_cpu(irq_stat, cpu).apic_timer_irqs + kstat_irqs(0);
 974
 975         /* if the none of the timers isn't firing, this cpu isn't doing much */
 976         if (!touched && last_irq_sums[cpu] == sum) {
 977                 /*
 978                  * Ayiee, looks like this CPU is stuck ...
 979                  * wait a few IRQs (5 seconds) before doing the oops ...
 980                  */
 981                 alert_counter[cpu]++;
 982                 if (alert_counter[cpu] == 5*nmi_hz)
 983                         /*
 984                          * die_nmi will return ONLY if NOTIFY_STOP happens..
 985                          */
 986                         die_nmi(regs, "BUG: NMI Watchdog detected LOCKUP");
 987         } else {
 988                 last_irq_sums[cpu] = sum;
 989                 alert_counter[cpu] = 0;
 990         }
 991         /* see if the nmi watchdog went off */
 992         if (wd->enabled) {
 993                 if (nmi_watchdog == NMI_LOCAL_APIC) {
 994                         rdmsrl(wd->perfctr_msr, dummy);
 995                         if (dummy & wd->check_bit){
 996                                 /* this wasn't a watchdog timer interrupt */
 997                                 goto done;
 998                         }
 999
1000                         /* only Intel P4 uses the cccr msr */
1001                         if (wd->cccr_msr != 0) {
1002                                 /*
1003                                  * P4 quirks:
1004                                  * - An overflown perfctr will assert its interrupt
1005                                  *   until the OVF flag in its CCCR is cleared.
1006                                  * - LVTPC is masked on interrupt and must be
1007                                  *   unmasked by the LVTPC handler.
1008                                  */
1009                                 rdmsrl(wd->cccr_msr, dummy);
1010                                 dummy &= ~P4_CCCR_OVF;
1011                                 wrmsrl(wd->cccr_msr, dummy);
1012                                 apic_write(APIC_LVTPC, APIC_DM_NMI);
1013                                 /* start the cycle over again */
1014                                 write_watchdog_counter(wd->perfctr_msr, NULL);
1015                         }
1016                         else if (wd->perfctr_msr == MSR_P6_PERFCTR0 ||
1017                                  wd->perfctr_msr == MSR_ARCH_PERFMON_PERFCTR0) {
1018                                 /* P6 based Pentium M need to re-unmask
1019                                  * the apic vector but it doesn't hurt
1020                                  * other P6 variant.
1021                                  * ArchPerfom/Core Duo also needs this */
1022                                 apic_write(APIC_LVTPC, APIC_DM_NMI);
1023                                 /* P6/ARCH_PERFMON has 32 bit counter write */
1024                                 write_watchdog_counter32(wd->perfctr_msr, NULL);
1025                         } else {
1026                                 /* start the cycle over again */
1027                                 write_watchdog_counter(wd->perfctr_msr, NULL);
1028                         }
1029                         rc = 1;
1030                 } else if (nmi_watchdog == NMI_IO_APIC) {
1031                         /* don't know how to accurately check for this.
1032                          * just assume it was a watchdog timer interrupt
1033                          * This matches the old behaviour.
1034                          */
1035                         rc = 1;
1036                 }
1037         }
1038 done:
1039         return rc;
1040 }
1041
1042 int do_nmi_callback(struct pt_regs * regs, int cpu)
1043 {
1044 #ifdef CONFIG_SYSCTL
1045         if (unknown_nmi_panic)
1046                 return unknown_nmi_panic_callback(regs, cpu);
1047 #endif
1048         return 0;
1049 }
1050
1051 #ifdef CONFIG_SYSCTL
1052
1053 static int unknown_nmi_panic_callback(struct pt_regs *regs, int cpu)
1054 {
1055         unsigned char reason = get_nmi_reason();
1056         char buf[64];
1057
1058         sprintf(buf, "NMI received for unknown reason %02x\n", reason);
1059         die_nmi(regs, buf);
1060         return 0;
1061 }
1062
1063 /*
1064  * proc handler for /proc/sys/kernel/nmi
1065  */
1066 int proc_nmi_enabled(struct ctl_table *table, int write, struct file *file,
1067                         void __user *buffer, size_t *length, loff_t *ppos)
1068 {
1069         int old_state;
1070
1071         nmi_watchdog_enabled = (atomic_read(&nmi_active) > 0) ? 1 : 0;
1072         old_state = nmi_watchdog_enabled;
1073         proc_dointvec(table, write, file, buffer, length, ppos);
1074         if (!!old_state == !!nmi_watchdog_enabled)
1075                 return 0;
1076
1077         if (atomic_read(&nmi_active) < 0) {
1078                 printk( KERN_WARNING "NMI watchdog is permanently disabled\n");
1079                 return -EIO;
1080         }
1081
1082         if (nmi_watchdog == NMI_DEFAULT) {
1083                 if (nmi_known_cpu() > 0)
1084                         nmi_watchdog = NMI_LOCAL_APIC;
1085                 else
1086                         nmi_watchdog = NMI_IO_APIC;
1087         }
1088
1089         if (nmi_watchdog == NMI_LOCAL_APIC) {
1090                 if (nmi_watchdog_enabled)
1091                         enable_lapic_nmi_watchdog();
1092                 else
1093                         disable_lapic_nmi_watchdog();
1094         } else {
1095                 printk( KERN_WARNING
1096                         "NMI watchdog doesn't know what hardware to touch\n");
1097                 return -EIO;
1098         }
1099         return 0;
1100 }
1101
1102 #endif
1103
1104 void __trigger_all_cpu_backtrace(void)
1105 {
1106         int i;
1107
1108         backtrace_mask = cpu_online_map;
1109         /* Wait for up to 10 seconds for all CPUs to do the backtrace */
1110         for (i = 0; i < 10 * 1000; i++) {
1111                 if (cpus_empty(backtrace_mask))
1112                         break;
1113                 mdelay(1);
1114         }
1115 }
1116
1117 EXPORT_SYMBOL(nmi_active);
1118 EXPORT_SYMBOL(nmi_watchdog);
1119 EXPORT_SYMBOL(avail_to_resrv_perfctr_nmi);
1120 EXPORT_SYMBOL(avail_to_resrv_perfctr_nmi_bit);
1121 EXPORT_SYMBOL(reserve_perfctr_nmi);
1122 EXPORT_SYMBOL(release_perfctr_nmi);
1123 EXPORT_SYMBOL(reserve_evntsel_nmi);
1124 EXPORT_SYMBOL(release_evntsel_nmi);
1125 EXPORT_SYMBOL(disable_timer_nmi_watchdog);
1126 EXPORT_SYMBOL(enable_timer_nmi_watchdog);