arch/x86/oprofile/op_model_amd.c

   1 /*
   2  * @file op_model_amd.c
   3  * athlon / K7 / K8 / Family 10h model-specific MSR operations
   4  *
   5  * @remark Copyright 2002-2009 OProfile authors
   6  * @remark Read the file COPYING
   7  *
   8  * @author John Levon
   9  * @author Philippe Elie
  10  * @author Graydon Hoare
  11  * @author Robert Richter <robert.richter@amd.com>
  12  * @author Barry Kasindorf <barry.kasindorf@amd.com>
  13  * @author Jason Yeh <jason.yeh@amd.com>
  14  * @author Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
  15  */
  16
  17 #include <linux/oprofile.h>
  18 #include <linux/device.h>
  19 #include <linux/pci.h>
  20 #include <linux/percpu.h>
  21
  22 #include <asm/ptrace.h>
  23 #include <asm/msr.h>
  24 #include <asm/nmi.h>
  25
  26 #include "op_x86_model.h"
  27 #include "op_counter.h"
  28
  29 #define NUM_COUNTERS 4
  30 #define NUM_CONTROLS 4
  31 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
  32 #define NUM_VIRT_COUNTERS 32
  33 #define NUM_VIRT_CONTROLS 32
  34 #else
  35 #define NUM_VIRT_COUNTERS NUM_COUNTERS
  36 #define NUM_VIRT_CONTROLS NUM_CONTROLS
  37 #endif
  38
  39 #define OP_EVENT_MASK                   0x0FFF
  40 #define OP_CTR_OVERFLOW                 (1ULL<<31)
  41
  42 #define MSR_AMD_EVENTSEL_RESERVED       ((0xFFFFFCF0ULL<<32)|(1ULL<<21))
  43
  44 static unsigned long reset_value[NUM_VIRT_COUNTERS];
  45 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
  46 DECLARE_PER_CPU(int, switch_index);
  47 #endif
  48
  49 #ifdef CONFIG_OPROFILE_IBS
  50
  51 /* IbsFetchCtl bits/masks */
  52 #define IBS_FETCH_RAND_EN               (1ULL<<57)
  53 #define IBS_FETCH_VAL                   (1ULL<<49)
  54 #define IBS_FETCH_ENABLE                (1ULL<<48)
  55 #define IBS_FETCH_CNT_MASK              0xFFFF0000ULL
  56
  57 /*IbsOpCtl bits */
  58 #define IBS_OP_CNT_CTL                  (1ULL<<19)
  59 #define IBS_OP_VAL                      (1ULL<<18)
  60 #define IBS_OP_ENABLE                   (1ULL<<17)
  61
  62 #define IBS_FETCH_SIZE                  6
  63 #define IBS_OP_SIZE                     12
  64
  65 static int has_ibs;     /* AMD Family10h and later */
  66
  67 struct op_ibs_config {
  68         unsigned long op_enabled;
  69         unsigned long fetch_enabled;
  70         unsigned long max_cnt_fetch;
  71         unsigned long max_cnt_op;
  72         unsigned long rand_en;
  73         unsigned long dispatched_ops;
  74 };
  75
  76 static struct op_ibs_config ibs_config;
  77
  78 #endif
  79
  80 /* functions for op_amd_spec */
  81
  82 static void op_amd_fill_in_addresses(struct op_msrs * const msrs)
  83 {
  84         int i;
  85
  86         for (i = 0; i < NUM_COUNTERS; i++) {
  87                 if (reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))
  88                         msrs->counters[i].addr = MSR_K7_PERFCTR0 + i;
  89                 else
  90                         msrs->counters[i].addr = 0;
  91         }
  92
  93         for (i = 0; i < NUM_CONTROLS; i++) {
  94                 if (reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i))
  95                         msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i;
  96                 else
  97                         msrs->controls[i].addr = 0;
  98         }
  99
 100 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
 101         for (i = 0; i < NUM_VIRT_COUNTERS; i++) {
 102                 int hw_counter = i % NUM_CONTROLS;
 103                 if (reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))
 104                         msrs->multiplex[i].addr = MSR_K7_PERFCTR0 + hw_counter;
 105                 else
 106                         msrs->multiplex[i].addr = 0;
 107         }
 108 #endif
 109 }
 110
 111 static void op_amd_setup_ctrs(struct op_x86_model_spec const *model,
 112                               struct op_msrs const * const msrs)
 113 {
 114         u64 val;
 115         int i;
 116
 117         /* setup reset_value */
 118         for (i = 0; i < NUM_VIRT_COUNTERS; ++i) {
 119                 if (counter_config[i].enabled) {
 120                         reset_value[i] = counter_config[i].count;
 121                 } else {
 122                         reset_value[i] = 0;
 123                 }
 124         }
 125
 126         /* clear all counters */
 127         for (i = 0; i < NUM_CONTROLS; ++i) {
 128                 if (unlikely(!msrs->controls[i].addr))
 129                         continue;
 130                 rdmsrl(msrs->controls[i].addr, val);
 131                 val &= model->reserved;
 132                 wrmsrl(msrs->controls[i].addr, val);
 133         }
 134
 135         /* avoid a false detection of ctr overflows in NMI handler */
 136         for (i = 0; i < NUM_COUNTERS; ++i) {
 137                 if (unlikely(!msrs->counters[i].addr))
 138                         continue;
 139                 wrmsrl(msrs->counters[i].addr, -1LL);
 140         }
 141
 142         /* enable active counters */
 143         for (i = 0; i < NUM_COUNTERS; ++i) {
 144 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
 145                 int offset = i + __get_cpu_var(switch_index);
 146 #else
 147                 int offset = i;
 148 #endif
 149                 if (counter_config[offset].enabled && msrs->counters[i].addr) {
 150                         /* setup counter registers */
 151                         wrmsrl(msrs->counters[i].addr, -(u64)reset_value[offset]);
 152
 153                         /* setup control registers */
 154                         rdmsrl(msrs->controls[i].addr, val);
 155                         val &= model->reserved;
 156                         val |= op_x86_get_ctrl(model, &counter_config[offset]);
 157                         wrmsrl(msrs->controls[i].addr, val);
 158                 }
 159         }
 160 }
 161
 162
 163 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
 164
 165 static void op_amd_switch_ctrl(struct op_x86_model_spec const *model,
 166                                struct op_msrs const * const msrs)
 167 {
 168         u64 val;
 169         int i;
 170
 171         /* enable active counters */
 172         for (i = 0; i < NUM_COUNTERS; ++i) {
 173                 int offset = i + __get_cpu_var(switch_index);
 174                 if (counter_config[offset].enabled) {
 175                         /* setup control registers */
 176                         rdmsrl(msrs->controls[i].addr, val);
 177                         val &= model->reserved;
 178                         val |= op_x86_get_ctrl(model, &counter_config[offset]);
 179                         wrmsrl(msrs->controls[i].addr, val);
 180                 }
 181         }
 182 }
 183
 184 #endif
 185
 186
 187 #ifdef CONFIG_OPROFILE_IBS
 188
 189 static inline int
 190 op_amd_handle_ibs(struct pt_regs * const regs,
 191                   struct op_msrs const * const msrs)
 192 {
 193         u64 val, ctl;
 194         struct op_entry entry;
 195
 196         if (!has_ibs)
 197                 return 0;
 198
 199         if (ibs_config.fetch_enabled) {
 200                 rdmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
 201                 if (ctl & IBS_FETCH_VAL) {
 202                         rdmsrl(MSR_AMD64_IBSFETCHLINAD, val);
 203                         oprofile_write_reserve(&entry, regs, val,
 204                                                IBS_FETCH_CODE, IBS_FETCH_SIZE);
 205                         oprofile_add_data64(&entry, val);
 206                         oprofile_add_data64(&entry, ctl);
 207                         rdmsrl(MSR_AMD64_IBSFETCHPHYSAD, val);
 208                         oprofile_add_data64(&entry, val);
 209                         oprofile_write_commit(&entry);
 210
 211                         /* reenable the IRQ */
 212                         ctl &= ~(IBS_FETCH_VAL | IBS_FETCH_CNT_MASK);
 213                         ctl |= IBS_FETCH_ENABLE;
 214                         wrmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
 215                 }
 216         }
 217
 218         if (ibs_config.op_enabled) {
 219                 rdmsrl(MSR_AMD64_IBSOPCTL, ctl);
 220                 if (ctl & IBS_OP_VAL) {
 221                         rdmsrl(MSR_AMD64_IBSOPRIP, val);
 222                         oprofile_write_reserve(&entry, regs, val,
 223                                                IBS_OP_CODE, IBS_OP_SIZE);
 224                         oprofile_add_data64(&entry, val);
 225                         rdmsrl(MSR_AMD64_IBSOPDATA, val);
 226                         oprofile_add_data64(&entry, val);
 227                         rdmsrl(MSR_AMD64_IBSOPDATA2, val);
 228                         oprofile_add_data64(&entry, val);
 229                         rdmsrl(MSR_AMD64_IBSOPDATA3, val);
 230                         oprofile_add_data64(&entry, val);
 231                         rdmsrl(MSR_AMD64_IBSDCLINAD, val);
 232                         oprofile_add_data64(&entry, val);
 233                         rdmsrl(MSR_AMD64_IBSDCPHYSAD, val);
 234                         oprofile_add_data64(&entry, val);
 235                         oprofile_write_commit(&entry);
 236
 237                         /* reenable the IRQ */
 238                         ctl &= ~IBS_OP_VAL & 0xFFFFFFFF;
 239                         ctl |= IBS_OP_ENABLE;
 240                         wrmsrl(MSR_AMD64_IBSOPCTL, ctl);
 241                 }
 242         }
 243
 244         return 1;
 245 }
 246
 247 static inline void op_amd_start_ibs(void)
 248 {
 249         u64 val;
 250         if (has_ibs && ibs_config.fetch_enabled) {
 251                 val = (ibs_config.max_cnt_fetch >> 4) & 0xFFFF;
 252                 val |= ibs_config.rand_en ? IBS_FETCH_RAND_EN : 0;
 253                 val |= IBS_FETCH_ENABLE;
 254                 wrmsrl(MSR_AMD64_IBSFETCHCTL, val);
 255         }
 256
 257         if (has_ibs && ibs_config.op_enabled) {
 258                 val = (ibs_config.max_cnt_op >> 4) & 0xFFFF;
 259                 val |= ibs_config.dispatched_ops ? IBS_OP_CNT_CTL : 0;
 260                 val |= IBS_OP_ENABLE;
 261                 wrmsrl(MSR_AMD64_IBSOPCTL, val);
 262         }
 263 }
 264
 265 static void op_amd_stop_ibs(void)
 266 {
 267         if (has_ibs && ibs_config.fetch_enabled)
 268                 /* clear max count and enable */
 269                 wrmsrl(MSR_AMD64_IBSFETCHCTL, 0);
 270
 271         if (has_ibs && ibs_config.op_enabled)
 272                 /* clear max count and enable */
 273                 wrmsrl(MSR_AMD64_IBSOPCTL, 0);
 274 }
 275
 276 #else
 277
 278 static inline int op_amd_handle_ibs(struct pt_regs * const regs,
 279                                     struct op_msrs const * const msrs)
 280 {
 281         return 0;
 282 }
 283 static inline void op_amd_start_ibs(void) { }
 284 static inline void op_amd_stop_ibs(void) { }
 285
 286 #endif
 287
 288 static int op_amd_check_ctrs(struct pt_regs * const regs,
 289                              struct op_msrs const * const msrs)
 290 {
 291         u64 val;
 292         int i;
 293
 294         for (i = 0; i < NUM_COUNTERS; ++i) {
 295 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
 296                 int offset = i + __get_cpu_var(switch_index);
 297 #else
 298                 int offset = i;
 299 #endif
 300                 if (!reset_value[offset])
 301                         continue;
 302                 rdmsrl(msrs->counters[i].addr, val);
 303                 /* bit is clear if overflowed: */
 304                 if (val & OP_CTR_OVERFLOW)
 305                         continue;
 306                 oprofile_add_sample(regs, offset);
 307                 wrmsrl(msrs->counters[i].addr, -(u64)reset_value[offset]);
 308         }
 309
 310         op_amd_handle_ibs(regs, msrs);
 311
 312         /* See op_model_ppro.c */
 313         return 1;
 314 }
 315
 316 static void op_amd_start(struct op_msrs const * const msrs)
 317 {
 318         u64 val;
 319         int i;
 320
 321         for (i = 0; i < NUM_COUNTERS; ++i) {
 322 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
 323                 int offset = i + __get_cpu_var(switch_index);
 324 #else
 325                 int offset = i;
 326 #endif
 327                 if (reset_value[offset]) {
 328                         rdmsrl(msrs->controls[i].addr, val);
 329                         val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
 330                         wrmsrl(msrs->controls[i].addr, val);
 331                 }
 332         }
 333
 334         op_amd_start_ibs();
 335 }
 336
 337 static void op_amd_stop(struct op_msrs const * const msrs)
 338 {
 339         u64 val;
 340         int i;
 341
 342         /*
 343          * Subtle: stop on all counters to avoid race with setting our
 344          * pm callback
 345          */
 346         for (i = 0; i < NUM_COUNTERS; ++i) {
 347 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
 348                 if (!reset_value[i + per_cpu(switch_index, smp_processor_id())])
 349 #else
 350                 if (!reset_value[i])
 351 #endif
 352                         continue;
 353                 rdmsrl(msrs->controls[i].addr, val);
 354                 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
 355                 wrmsrl(msrs->controls[i].addr, val);
 356         }
 357
 358         op_amd_stop_ibs();
 359 }
 360
 361 static void op_amd_shutdown(struct op_msrs const * const msrs)
 362 {
 363         int i;
 364
 365         for (i = 0; i < NUM_COUNTERS; ++i) {
 366                 if (msrs->counters[i].addr)
 367                         release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
 368         }
 369         for (i = 0; i < NUM_COUNTERS; ++i) {
 370                 if (msrs->controls[i].addr)
 371                         release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
 372         }
 373 }
 374
 375 #ifdef CONFIG_OPROFILE_IBS
 376
 377 static u8 ibs_eilvt_off;
 378
 379 static inline void apic_init_ibs_nmi_per_cpu(void *arg)
 380 {
 381         ibs_eilvt_off = setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_NMI, 0);
 382 }
 383
 384 static inline void apic_clear_ibs_nmi_per_cpu(void *arg)
 385 {
 386         setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_FIX, 1);
 387 }
 388
 389 static int init_ibs_nmi(void)
 390 {
 391 #define IBSCTL_LVTOFFSETVAL             (1 << 8)
 392 #define IBSCTL                          0x1cc
 393         struct pci_dev *cpu_cfg;
 394         int nodes;
 395         u32 value = 0;
 396
 397         /* per CPU setup */
 398         on_each_cpu(apic_init_ibs_nmi_per_cpu, NULL, 1);
 399
 400         nodes = 0;
 401         cpu_cfg = NULL;
 402         do {
 403                 cpu_cfg = pci_get_device(PCI_VENDOR_ID_AMD,
 404                                          PCI_DEVICE_ID_AMD_10H_NB_MISC,
 405                                          cpu_cfg);
 406                 if (!cpu_cfg)
 407                         break;
 408                 ++nodes;
 409                 pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off
 410                                        | IBSCTL_LVTOFFSETVAL);
 411                 pci_read_config_dword(cpu_cfg, IBSCTL, &value);
 412                 if (value != (ibs_eilvt_off | IBSCTL_LVTOFFSETVAL)) {
 413                         pci_dev_put(cpu_cfg);
 414                         printk(KERN_DEBUG "Failed to setup IBS LVT offset, "
 415                                 "IBSCTL = 0x%08x", value);
 416                         return 1;
 417                 }
 418         } while (1);
 419
 420         if (!nodes) {
 421                 printk(KERN_DEBUG "No CPU node configured for IBS");
 422                 return 1;
 423         }
 424
 425 #ifdef CONFIG_NUMA
 426         /* Sanity check */
 427         /* Works only for 64bit with proper numa implementation. */
 428         if (nodes != num_possible_nodes()) {
 429                 printk(KERN_DEBUG "Failed to setup CPU node(s) for IBS, "
 430                         "found: %d, expected %d",
 431                         nodes, num_possible_nodes());
 432                 return 1;
 433         }
 434 #endif
 435         return 0;
 436 }
 437
 438 /* uninitialize the APIC for the IBS interrupts if needed */
 439 static void clear_ibs_nmi(void)
 440 {
 441         if (has_ibs)
 442                 on_each_cpu(apic_clear_ibs_nmi_per_cpu, NULL, 1);
 443 }
 444
 445 /* initialize the APIC for the IBS interrupts if available */
 446 static void ibs_init(void)
 447 {
 448         has_ibs = boot_cpu_has(X86_FEATURE_IBS);
 449
 450         if (!has_ibs)
 451                 return;
 452
 453         if (init_ibs_nmi()) {
 454                 has_ibs = 0;
 455                 return;
 456         }
 457
 458         printk(KERN_INFO "oprofile: AMD IBS detected\n");
 459 }
 460
 461 static void ibs_exit(void)
 462 {
 463         if (!has_ibs)
 464                 return;
 465
 466         clear_ibs_nmi();
 467 }
 468
 469 static int (*create_arch_files)(struct super_block *sb, struct dentry *root);
 470
 471 static int setup_ibs_files(struct super_block *sb, struct dentry *root)
 472 {
 473         struct dentry *dir;
 474         int ret = 0;
 475
 476         /* architecture specific files */
 477         if (create_arch_files)
 478                 ret = create_arch_files(sb, root);
 479
 480         if (ret)
 481                 return ret;
 482
 483         if (!has_ibs)
 484                 return ret;
 485
 486         /* model specific files */
 487
 488         /* setup some reasonable defaults */
 489         ibs_config.max_cnt_fetch = 250000;
 490         ibs_config.fetch_enabled = 0;
 491         ibs_config.max_cnt_op = 250000;
 492         ibs_config.op_enabled = 0;
 493         ibs_config.dispatched_ops = 1;
 494
 495         dir = oprofilefs_mkdir(sb, root, "ibs_fetch");
 496         oprofilefs_create_ulong(sb, dir, "enable",
 497                                 &ibs_config.fetch_enabled);
 498         oprofilefs_create_ulong(sb, dir, "max_count",
 499                                 &ibs_config.max_cnt_fetch);
 500         oprofilefs_create_ulong(sb, dir, "rand_enable",
 501                                 &ibs_config.rand_en);
 502
 503         dir = oprofilefs_mkdir(sb, root, "ibs_op");
 504         oprofilefs_create_ulong(sb, dir, "enable",
 505                                 &ibs_config.op_enabled);
 506         oprofilefs_create_ulong(sb, dir, "max_count",
 507                                 &ibs_config.max_cnt_op);
 508         oprofilefs_create_ulong(sb, dir, "dispatched_ops",
 509                                 &ibs_config.dispatched_ops);
 510
 511         return 0;
 512 }
 513
 514 static int op_amd_init(struct oprofile_operations *ops)
 515 {
 516         ibs_init();
 517         create_arch_files = ops->create_files;
 518         ops->create_files = setup_ibs_files;
 519         return 0;
 520 }
 521
 522 static void op_amd_exit(void)
 523 {
 524         ibs_exit();
 525 }
 526
 527 #else
 528
 529 /* no IBS support */
 530
 531 static int op_amd_init(struct oprofile_operations *ops)
 532 {
 533         return 0;
 534 }
 535
 536 static void op_amd_exit(void) {}
 537
 538 #endif /* CONFIG_OPROFILE_IBS */
 539
 540 struct op_x86_model_spec const op_amd_spec = {
 541         .num_counters           = NUM_COUNTERS,
 542         .num_controls           = NUM_CONTROLS,
 543         .num_virt_counters      = NUM_VIRT_COUNTERS,
 544         .num_virt_controls      = NUM_VIRT_CONTROLS,
 545         .reserved               = MSR_AMD_EVENTSEL_RESERVED,
 546         .event_mask             = OP_EVENT_MASK,
 547         .init                   = op_amd_init,
 548         .exit                   = op_amd_exit,
 549         .fill_in_addresses      = &op_amd_fill_in_addresses,
 550         .setup_ctrs             = &op_amd_setup_ctrs,
 551         .check_ctrs             = &op_amd_check_ctrs,
 552         .start                  = &op_amd_start,
 553         .stop                   = &op_amd_stop,
 554         .shutdown               = &op_amd_shutdown,
 555 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
 556         .switch_ctrl            = &op_amd_switch_ctrl,
 557 #endif
 558 };