arch/x86/oprofile/op_model_amd.c

   1 /*
   2  * @file op_model_amd.c
   3  * athlon / K7 / K8 / Family 10h model-specific MSR operations
   4  *
   5  * @remark Copyright 2002-2008 OProfile authors
   6  * @remark Read the file COPYING
   7  *
   8  * @author John Levon
   9  * @author Philippe Elie
  10  * @author Graydon Hoare
  11  * @author Robert Richter <robert.richter@amd.com>
  12  * @author Barry Kasindorf
  13 */
  14
  15 #include <linux/oprofile.h>
  16 #include <linux/device.h>
  17 #include <linux/pci.h>
  18
  19 #include <asm/ptrace.h>
  20 #include <asm/msr.h>
  21 #include <asm/nmi.h>
  22
  23 #include "op_x86_model.h"
  24 #include "op_counter.h"
  25
  26 #define NUM_COUNTERS 4
  27 #define NUM_CONTROLS 4
  28
  29 #define CTR_IS_RESERVED(msrs, c) (msrs->counters[(c)].addr ? 1 : 0)
  30 #define CTR_READ(l, h, msrs, c) do {rdmsr(msrs->counters[(c)].addr, (l), (h)); } while (0)
  31 #define CTR_WRITE(l, msrs, c) do {wrmsr(msrs->counters[(c)].addr, -(unsigned int)(l), -1); } while (0)
  32 #define CTR_OVERFLOWED(n) (!((n) & (1U<<31)))
  33
  34 #define CTRL_IS_RESERVED(msrs, c) (msrs->controls[(c)].addr ? 1 : 0)
  35 #define CTRL_READ(l, h, msrs, c) do {rdmsr(msrs->controls[(c)].addr, (l), (h)); } while (0)
  36 #define CTRL_WRITE(l, h, msrs, c) do {wrmsr(msrs->controls[(c)].addr, (l), (h)); } while (0)
  37 #define CTRL_SET_ACTIVE(n) (n |= (1<<22))
  38 #define CTRL_SET_INACTIVE(n) (n &= ~(1<<22))
  39 #define CTRL_CLEAR_LO(x) (x &= (1<<21))
  40 #define CTRL_CLEAR_HI(x) (x &= 0xfffffcf0)
  41 #define CTRL_SET_ENABLE(val) (val |= 1<<20)
  42 #define CTRL_SET_USR(val, u) (val |= ((u & 1) << 16))
  43 #define CTRL_SET_KERN(val, k) (val |= ((k & 1) << 17))
  44 #define CTRL_SET_UM(val, m) (val |= (m << 8))
  45 #define CTRL_SET_EVENT_LOW(val, e) (val |= (e & 0xff))
  46 #define CTRL_SET_EVENT_HIGH(val, e) (val |= ((e >> 8) & 0xf))
  47 #define CTRL_SET_HOST_ONLY(val, h) (val |= ((h & 1) << 9))
  48 #define CTRL_SET_GUEST_ONLY(val, h) (val |= ((h & 1) << 8))
  49
  50 static unsigned long reset_value[NUM_COUNTERS];
  51
  52 #ifdef CONFIG_OPROFILE_IBS
  53
  54 /* IbsFetchCtl bits/masks */
  55 #define IBS_FETCH_HIGH_VALID_BIT        (1UL << 17)     /* bit 49 */
  56 #define IBS_FETCH_HIGH_ENABLE           (1UL << 16)     /* bit 48 */
  57 #define IBS_FETCH_LOW_MAX_CNT_MASK      0x0000FFFFUL    /* MaxCnt mask */
  58
  59 /*IbsOpCtl bits */
  60 #define IBS_OP_LOW_VALID_BIT            (1ULL<<18)      /* bit 18 */
  61 #define IBS_OP_LOW_ENABLE               (1ULL<<17)      /* bit 17 */
  62
  63 /* Codes used in cpu_buffer.c */
  64 /* This produces duplicate code, need to be fixed */
  65 #define IBS_FETCH_BEGIN 3
  66 #define IBS_OP_BEGIN    4
  67
  68 /*
  69  * The function interface needs to be fixed, something like add
  70  * data. Should then be added to linux/oprofile.h.
  71  */
  72 extern void
  73 oprofile_add_ibs_sample(struct pt_regs * const regs,
  74                         unsigned int * const ibs_sample, int ibs_code);
  75
  76 struct ibs_fetch_sample {
  77         /* MSRC001_1031 IBS Fetch Linear Address Register */
  78         unsigned int ibs_fetch_lin_addr_low;
  79         unsigned int ibs_fetch_lin_addr_high;
  80         /* MSRC001_1030 IBS Fetch Control Register */
  81         unsigned int ibs_fetch_ctl_low;
  82         unsigned int ibs_fetch_ctl_high;
  83         /* MSRC001_1032 IBS Fetch Physical Address Register */
  84         unsigned int ibs_fetch_phys_addr_low;
  85         unsigned int ibs_fetch_phys_addr_high;
  86 };
  87
  88 struct ibs_op_sample {
  89         /* MSRC001_1034 IBS Op Logical Address Register (IbsRIP) */
  90         unsigned int ibs_op_rip_low;
  91         unsigned int ibs_op_rip_high;
  92         /* MSRC001_1035 IBS Op Data Register */
  93         unsigned int ibs_op_data1_low;
  94         unsigned int ibs_op_data1_high;
  95         /* MSRC001_1036 IBS Op Data 2 Register */
  96         unsigned int ibs_op_data2_low;
  97         unsigned int ibs_op_data2_high;
  98         /* MSRC001_1037 IBS Op Data 3 Register */
  99         unsigned int ibs_op_data3_low;
 100         unsigned int ibs_op_data3_high;
 101         /* MSRC001_1038 IBS DC Linear Address Register (IbsDcLinAd) */
 102         unsigned int ibs_dc_linear_low;
 103         unsigned int ibs_dc_linear_high;
 104         /* MSRC001_1039 IBS DC Physical Address Register (IbsDcPhysAd) */
 105         unsigned int ibs_dc_phys_low;
 106         unsigned int ibs_dc_phys_high;
 107 };
 108
 109 static int ibs_allowed; /* AMD Family10h and later */
 110
 111 struct op_ibs_config {
 112         unsigned long op_enabled;
 113         unsigned long fetch_enabled;
 114         unsigned long max_cnt_fetch;
 115         unsigned long max_cnt_op;
 116         unsigned long rand_en;
 117         unsigned long dispatched_ops;
 118 };
 119
 120 static struct op_ibs_config ibs_config;
 121
 122 #endif
 123
 124 /* functions for op_amd_spec */
 125
 126 static void op_amd_fill_in_addresses(struct op_msrs * const msrs)
 127 {
 128         int i;
 129
 130         for (i = 0; i < NUM_COUNTERS; i++) {
 131                 if (reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))
 132                         msrs->counters[i].addr = MSR_K7_PERFCTR0 + i;
 133                 else
 134                         msrs->counters[i].addr = 0;
 135         }
 136
 137         for (i = 0; i < NUM_CONTROLS; i++) {
 138                 if (reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i))
 139                         msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i;
 140                 else
 141                         msrs->controls[i].addr = 0;
 142         }
 143 }
 144
 145
 146 static void op_amd_setup_ctrs(struct op_msrs const * const msrs)
 147 {
 148         unsigned int low, high;
 149         int i;
 150
 151         /* clear all counters */
 152         for (i = 0 ; i < NUM_CONTROLS; ++i) {
 153                 if (unlikely(!CTRL_IS_RESERVED(msrs, i)))
 154                         continue;
 155                 CTRL_READ(low, high, msrs, i);
 156                 CTRL_CLEAR_LO(low);
 157                 CTRL_CLEAR_HI(high);
 158                 CTRL_WRITE(low, high, msrs, i);
 159         }
 160
 161         /* avoid a false detection of ctr overflows in NMI handler */
 162         for (i = 0; i < NUM_COUNTERS; ++i) {
 163                 if (unlikely(!CTR_IS_RESERVED(msrs, i)))
 164                         continue;
 165                 CTR_WRITE(1, msrs, i);
 166         }
 167
 168         /* enable active counters */
 169         for (i = 0; i < NUM_COUNTERS; ++i) {
 170                 if ((counter_config[i].enabled) && (CTR_IS_RESERVED(msrs, i))) {
 171                         reset_value[i] = counter_config[i].count;
 172
 173                         CTR_WRITE(counter_config[i].count, msrs, i);
 174
 175                         CTRL_READ(low, high, msrs, i);
 176                         CTRL_CLEAR_LO(low);
 177                         CTRL_CLEAR_HI(high);
 178                         CTRL_SET_ENABLE(low);
 179                         CTRL_SET_USR(low, counter_config[i].user);
 180                         CTRL_SET_KERN(low, counter_config[i].kernel);
 181                         CTRL_SET_UM(low, counter_config[i].unit_mask);
 182                         CTRL_SET_EVENT_LOW(low, counter_config[i].event);
 183                         CTRL_SET_EVENT_HIGH(high, counter_config[i].event);
 184                         CTRL_SET_HOST_ONLY(high, 0);
 185                         CTRL_SET_GUEST_ONLY(high, 0);
 186
 187                         CTRL_WRITE(low, high, msrs, i);
 188                 } else {
 189                         reset_value[i] = 0;
 190                 }
 191         }
 192 }
 193
 194 #ifdef CONFIG_OPROFILE_IBS
 195
 196 static inline int
 197 op_amd_handle_ibs(struct pt_regs * const regs,
 198                   struct op_msrs const * const msrs)
 199 {
 200         unsigned int low, high;
 201         struct ibs_fetch_sample ibs_fetch;
 202         struct ibs_op_sample ibs_op;
 203
 204         if (!ibs_allowed)
 205                 return 1;
 206
 207         if (ibs_config.fetch_enabled) {
 208                 rdmsr(MSR_AMD64_IBSFETCHCTL, low, high);
 209                 if (high & IBS_FETCH_HIGH_VALID_BIT) {
 210                         ibs_fetch.ibs_fetch_ctl_high = high;
 211                         ibs_fetch.ibs_fetch_ctl_low = low;
 212                         rdmsr(MSR_AMD64_IBSFETCHLINAD, low, high);
 213                         ibs_fetch.ibs_fetch_lin_addr_high = high;
 214                         ibs_fetch.ibs_fetch_lin_addr_low = low;
 215                         rdmsr(MSR_AMD64_IBSFETCHPHYSAD, low, high);
 216                         ibs_fetch.ibs_fetch_phys_addr_high = high;
 217                         ibs_fetch.ibs_fetch_phys_addr_low = low;
 218
 219                         oprofile_add_ibs_sample(regs,
 220                                                 (unsigned int *)&ibs_fetch,
 221                                                 IBS_FETCH_BEGIN);
 222
 223                         /* reenable the IRQ */
 224                         rdmsr(MSR_AMD64_IBSFETCHCTL, low, high);
 225                         high &= ~IBS_FETCH_HIGH_VALID_BIT;
 226                         high |= IBS_FETCH_HIGH_ENABLE;
 227                         low &= IBS_FETCH_LOW_MAX_CNT_MASK;
 228                         wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
 229                 }
 230         }
 231
 232         if (ibs_config.op_enabled) {
 233                 rdmsr(MSR_AMD64_IBSOPCTL, low, high);
 234                 if (low & IBS_OP_LOW_VALID_BIT) {
 235                         rdmsr(MSR_AMD64_IBSOPRIP, low, high);
 236                         ibs_op.ibs_op_rip_low = low;
 237                         ibs_op.ibs_op_rip_high = high;
 238                         rdmsr(MSR_AMD64_IBSOPDATA, low, high);
 239                         ibs_op.ibs_op_data1_low = low;
 240                         ibs_op.ibs_op_data1_high = high;
 241                         rdmsr(MSR_AMD64_IBSOPDATA2, low, high);
 242                         ibs_op.ibs_op_data2_low = low;
 243                         ibs_op.ibs_op_data2_high = high;
 244                         rdmsr(MSR_AMD64_IBSOPDATA3, low, high);
 245                         ibs_op.ibs_op_data3_low = low;
 246                         ibs_op.ibs_op_data3_high = high;
 247                         rdmsr(MSR_AMD64_IBSDCLINAD, low, high);
 248                         ibs_op.ibs_dc_linear_low = low;
 249                         ibs_op.ibs_dc_linear_high = high;
 250                         rdmsr(MSR_AMD64_IBSDCPHYSAD, low, high);
 251                         ibs_op.ibs_dc_phys_low = low;
 252                         ibs_op.ibs_dc_phys_high = high;
 253
 254                         /* reenable the IRQ */
 255                         oprofile_add_ibs_sample(regs,
 256                                                 (unsigned int *)&ibs_op,
 257                                                 IBS_OP_BEGIN);
 258                         rdmsr(MSR_AMD64_IBSOPCTL, low, high);
 259                         high = 0;
 260                         low &= ~IBS_OP_LOW_VALID_BIT;
 261                         low |= IBS_OP_LOW_ENABLE;
 262                         wrmsr(MSR_AMD64_IBSOPCTL, low, high);
 263                 }
 264         }
 265
 266         return 1;
 267 }
 268
 269 #endif
 270
 271 static int op_amd_check_ctrs(struct pt_regs * const regs,
 272                              struct op_msrs const * const msrs)
 273 {
 274         unsigned int low, high;
 275         int i;
 276
 277         for (i = 0 ; i < NUM_COUNTERS; ++i) {
 278                 if (!reset_value[i])
 279                         continue;
 280                 CTR_READ(low, high, msrs, i);
 281                 if (CTR_OVERFLOWED(low)) {
 282                         oprofile_add_sample(regs, i);
 283                         CTR_WRITE(reset_value[i], msrs, i);
 284                 }
 285         }
 286
 287 #ifdef CONFIG_OPROFILE_IBS
 288         op_amd_handle_ibs(regs, msrs);
 289 #endif
 290
 291         /* See op_model_ppro.c */
 292         return 1;
 293 }
 294
 295 static void op_amd_start(struct op_msrs const * const msrs)
 296 {
 297         unsigned int low, high;
 298         int i;
 299         for (i = 0 ; i < NUM_COUNTERS ; ++i) {
 300                 if (reset_value[i]) {
 301                         CTRL_READ(low, high, msrs, i);
 302                         CTRL_SET_ACTIVE(low);
 303                         CTRL_WRITE(low, high, msrs, i);
 304                 }
 305         }
 306
 307 #ifdef CONFIG_OPROFILE_IBS
 308         if (ibs_allowed && ibs_config.fetch_enabled) {
 309                 low = (ibs_config.max_cnt_fetch >> 4) & 0xFFFF;
 310                 high = ((ibs_config.rand_en & 0x1) << 25) /* bit 57 */
 311                         + IBS_FETCH_HIGH_ENABLE;
 312                 wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
 313         }
 314
 315         if (ibs_allowed && ibs_config.op_enabled) {
 316                 low = ((ibs_config.max_cnt_op >> 4) & 0xFFFF)
 317                         + ((ibs_config.dispatched_ops & 0x1) << 19) /* bit 19 */
 318                         + IBS_OP_LOW_ENABLE;
 319                 high = 0;
 320                 wrmsr(MSR_AMD64_IBSOPCTL, low, high);
 321         }
 322 #endif
 323 }
 324
 325
 326 static void op_amd_stop(struct op_msrs const * const msrs)
 327 {
 328         unsigned int low, high;
 329         int i;
 330
 331         /*
 332          * Subtle: stop on all counters to avoid race with setting our
 333          * pm callback
 334          */
 335         for (i = 0 ; i < NUM_COUNTERS ; ++i) {
 336                 if (!reset_value[i])
 337                         continue;
 338                 CTRL_READ(low, high, msrs, i);
 339                 CTRL_SET_INACTIVE(low);
 340                 CTRL_WRITE(low, high, msrs, i);
 341         }
 342
 343 #ifdef CONFIG_OPROFILE_IBS
 344         if (ibs_allowed && ibs_config.fetch_enabled) {
 345                 /* clear max count and enable */
 346                 low = 0;
 347                 high = 0;
 348                 wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
 349         }
 350
 351         if (ibs_allowed && ibs_config.op_enabled) {
 352                 /* clear max count and enable */
 353                 low = 0;
 354                 high = 0;
 355                 wrmsr(MSR_AMD64_IBSOPCTL, low, high);
 356         }
 357 #endif
 358 }
 359
 360 static void op_amd_shutdown(struct op_msrs const * const msrs)
 361 {
 362         int i;
 363
 364         for (i = 0 ; i < NUM_COUNTERS ; ++i) {
 365                 if (CTR_IS_RESERVED(msrs, i))
 366                         release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
 367         }
 368         for (i = 0 ; i < NUM_CONTROLS ; ++i) {
 369                 if (CTRL_IS_RESERVED(msrs, i))
 370                         release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
 371         }
 372 }
 373
 374 #ifdef CONFIG_OPROFILE_IBS
 375
 376 static u8 ibs_eilvt_off;
 377
 378 static inline void apic_init_ibs_nmi_per_cpu(void *arg)
 379 {
 380         ibs_eilvt_off = setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_NMI, 0);
 381 }
 382
 383 static inline void apic_clear_ibs_nmi_per_cpu(void *arg)
 384 {
 385         setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_FIX, 1);
 386 }
 387
 388 static int init_ibs_nmi(void)
 389 {
 390 #define IBSCTL_LVTOFFSETVAL             (1 << 8)
 391 #define IBSCTL                          0x1cc
 392         struct pci_dev *cpu_cfg;
 393         int nodes;
 394         u32 value = 0;
 395
 396         /* per CPU setup */
 397         on_each_cpu(apic_init_ibs_nmi_per_cpu, NULL, 1);
 398
 399         nodes = 0;
 400         cpu_cfg = NULL;
 401         do {
 402                 cpu_cfg = pci_get_device(PCI_VENDOR_ID_AMD,
 403                                          PCI_DEVICE_ID_AMD_10H_NB_MISC,
 404                                          cpu_cfg);
 405                 if (!cpu_cfg)
 406                         break;
 407                 ++nodes;
 408                 pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off
 409                                        | IBSCTL_LVTOFFSETVAL);
 410                 pci_read_config_dword(cpu_cfg, IBSCTL, &value);
 411                 if (value != (ibs_eilvt_off | IBSCTL_LVTOFFSETVAL)) {
 412                         printk(KERN_DEBUG "Failed to setup IBS LVT offset, "
 413                                 "IBSCTL = 0x%08x", value);
 414                         return 1;
 415                 }
 416         } while (1);
 417
 418         if (!nodes) {
 419                 printk(KERN_DEBUG "No CPU node configured for IBS");
 420                 return 1;
 421         }
 422
 423 #ifdef CONFIG_NUMA
 424         /* Sanity check */
 425         /* Works only for 64bit with proper numa implementation. */
 426         if (nodes != num_possible_nodes()) {
 427                 printk(KERN_DEBUG "Failed to setup CPU node(s) for IBS, "
 428                         "found: %d, expected %d",
 429                         nodes, num_possible_nodes());
 430                 return 1;
 431         }
 432 #endif
 433         return 0;
 434 }
 435
 436 /* uninitialize the APIC for the IBS interrupts if needed */
 437 static void clear_ibs_nmi(void)
 438 {
 439         if (ibs_allowed)
 440                 on_each_cpu(apic_clear_ibs_nmi_per_cpu, NULL, 1);
 441 }
 442
 443 /* initialize the APIC for the IBS interrupts if available */
 444 static void ibs_init(void)
 445 {
 446         ibs_allowed = boot_cpu_has(X86_FEATURE_IBS);
 447
 448         if (!ibs_allowed)
 449                 return;
 450
 451         if (init_ibs_nmi()) {
 452                 ibs_allowed = 0;
 453                 return;
 454         }
 455
 456         printk(KERN_INFO "oprofile: AMD IBS detected\n");
 457 }
 458
 459 static void ibs_exit(void)
 460 {
 461         if (!ibs_allowed)
 462                 return;
 463
 464         clear_ibs_nmi();
 465 }
 466
 467 static int (*create_arch_files)(struct super_block *sb, struct dentry *root);
 468
 469 static int setup_ibs_files(struct super_block *sb, struct dentry *root)
 470 {
 471         struct dentry *dir;
 472         int ret = 0;
 473
 474         /* architecture specific files */
 475         if (create_arch_files)
 476                 ret = create_arch_files(sb, root);
 477
 478         if (ret)
 479                 return ret;
 480
 481         if (!ibs_allowed)
 482                 return ret;
 483
 484         /* model specific files */
 485
 486         /* setup some reasonable defaults */
 487         ibs_config.max_cnt_fetch = 250000;
 488         ibs_config.fetch_enabled = 0;
 489         ibs_config.max_cnt_op = 250000;
 490         ibs_config.op_enabled = 0;
 491         ibs_config.dispatched_ops = 1;
 492
 493         dir = oprofilefs_mkdir(sb, root, "ibs_fetch");
 494         oprofilefs_create_ulong(sb, dir, "enable",
 495                                 &ibs_config.fetch_enabled);
 496         oprofilefs_create_ulong(sb, dir, "max_count",
 497                                 &ibs_config.max_cnt_fetch);
 498         oprofilefs_create_ulong(sb, dir, "rand_enable",
 499                                 &ibs_config.rand_en);
 500
 501         dir = oprofilefs_mkdir(sb, root, "ibs_op");
 502         oprofilefs_create_ulong(sb, dir, "enable",
 503                                 &ibs_config.op_enabled);
 504         oprofilefs_create_ulong(sb, dir, "max_count",
 505                                 &ibs_config.max_cnt_op);
 506         oprofilefs_create_ulong(sb, dir, "dispatched_ops",
 507                                 &ibs_config.dispatched_ops);
 508
 509         return 0;
 510 }
 511
 512 static int op_amd_init(struct oprofile_operations *ops)
 513 {
 514         ibs_init();
 515         create_arch_files = ops->create_files;
 516         ops->create_files = setup_ibs_files;
 517         return 0;
 518 }
 519
 520 static void op_amd_exit(void)
 521 {
 522         ibs_exit();
 523 }
 524
 525 #else
 526
 527 /* no IBS support */
 528
 529 static int op_amd_init(struct oprofile_operations *ops)
 530 {
 531         return 0;
 532 }
 533
 534 static void op_amd_exit(void) {}
 535
 536 #endif /* CONFIG_OPROFILE_IBS */
 537
 538 struct op_x86_model_spec const op_amd_spec = {
 539         .init                   = op_amd_init,
 540         .exit                   = op_amd_exit,
 541         .num_counters           = NUM_COUNTERS,
 542         .num_controls           = NUM_CONTROLS,
 543         .fill_in_addresses      = &op_amd_fill_in_addresses,
 544         .setup_ctrs             = &op_amd_setup_ctrs,
 545         .check_ctrs             = &op_amd_check_ctrs,
 546         .start                  = &op_amd_start,
 547         .stop                   = &op_amd_stop,
 548         .shutdown               = &op_amd_shutdown
 549 };