arch/x86/oprofile/op_model_amd.c

   1 /*
   2  * @file op_model_amd.c
   3  * athlon / K7 / K8 / Family 10h model-specific MSR operations
   4  *
   5  * @remark Copyright 2002-2009 OProfile authors
   6  * @remark Read the file COPYING
   7  *
   8  * @author John Levon
   9  * @author Philippe Elie
  10  * @author Graydon Hoare
  11  * @author Robert Richter <robert.richter@amd.com>
  12  * @author Barry Kasindorf
  13  */
  14
  15 #include <linux/oprofile.h>
  16 #include <linux/device.h>
  17 #include <linux/pci.h>
  18
  19 #include <asm/ptrace.h>
  20 #include <asm/msr.h>
  21 #include <asm/nmi.h>
  22
  23 #include "op_x86_model.h"
  24 #include "op_counter.h"
  25
  26 #define NUM_COUNTERS 4
  27 #define NUM_CONTROLS 4
  28 #define OP_EVENT_MASK                   0x0FFF
  29 #define OP_CTR_OVERFLOW                 (1ULL<<31)
  30
  31 #define MSR_AMD_EVENTSEL_RESERVED       ((0xFFFFFCF0ULL<<32)|(1ULL<<21))
  32
  33 static unsigned long reset_value[NUM_COUNTERS];
  34
  35 #ifdef CONFIG_OPROFILE_IBS
  36
  37 /* IbsFetchCtl bits/masks */
  38 #define IBS_FETCH_HIGH_VALID_BIT        (1UL << 17)     /* bit 49 */
  39 #define IBS_FETCH_HIGH_ENABLE           (1UL << 16)     /* bit 48 */
  40 #define IBS_FETCH_LOW_MAX_CNT_MASK      0x0000FFFFUL    /* MaxCnt mask */
  41
  42 /*IbsOpCtl bits */
  43 #define IBS_OP_LOW_VALID_BIT            (1ULL<<18)      /* bit 18 */
  44 #define IBS_OP_LOW_ENABLE               (1ULL<<17)      /* bit 17 */
  45
  46 #define IBS_FETCH_SIZE  6
  47 #define IBS_OP_SIZE     12
  48
  49 static int has_ibs;     /* AMD Family10h and later */
  50
  51 struct op_ibs_config {
  52         unsigned long op_enabled;
  53         unsigned long fetch_enabled;
  54         unsigned long max_cnt_fetch;
  55         unsigned long max_cnt_op;
  56         unsigned long rand_en;
  57         unsigned long dispatched_ops;
  58 };
  59
  60 static struct op_ibs_config ibs_config;
  61
  62 #endif
  63
  64 /* functions for op_amd_spec */
  65
  66 static void op_amd_fill_in_addresses(struct op_msrs * const msrs)
  67 {
  68         int i;
  69
  70         for (i = 0; i < NUM_COUNTERS; i++) {
  71                 if (reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))
  72                         msrs->counters[i].addr = MSR_K7_PERFCTR0 + i;
  73                 else
  74                         msrs->counters[i].addr = 0;
  75         }
  76
  77         for (i = 0; i < NUM_CONTROLS; i++) {
  78                 if (reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i))
  79                         msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i;
  80                 else
  81                         msrs->controls[i].addr = 0;
  82         }
  83 }
  84
  85 static void op_amd_setup_ctrs(struct op_x86_model_spec const *model,
  86                               struct op_msrs const * const msrs)
  87 {
  88         u64 val;
  89         int i;
  90
  91         /* clear all counters */
  92         for (i = 0 ; i < NUM_CONTROLS; ++i) {
  93                 if (unlikely(!CTRL_IS_RESERVED(msrs, i)))
  94                         continue;
  95                 rdmsrl(msrs->controls[i].addr, val);
  96                 val &= model->reserved;
  97                 wrmsrl(msrs->controls[i].addr, val);
  98         }
  99
 100         /* avoid a false detection of ctr overflows in NMI handler */
 101         for (i = 0; i < NUM_COUNTERS; ++i) {
 102                 if (unlikely(!CTR_IS_RESERVED(msrs, i)))
 103                         continue;
 104                 wrmsr(msrs->counters[i].addr, -1, -1);
 105         }
 106
 107         /* enable active counters */
 108         for (i = 0; i < NUM_COUNTERS; ++i) {
 109                 if ((counter_config[i].enabled) && (CTR_IS_RESERVED(msrs, i))) {
 110                         reset_value[i] = counter_config[i].count;
 111                         wrmsr(msrs->counters[i].addr, -(unsigned int)counter_config[i].count, -1);
 112                         rdmsrl(msrs->controls[i].addr, val);
 113                         val &= model->reserved;
 114                         val |= op_x86_get_ctrl(model, &counter_config[i]);
 115                         wrmsrl(msrs->controls[i].addr, val);
 116                 } else {
 117                         reset_value[i] = 0;
 118                 }
 119         }
 120 }
 121
 122 #ifdef CONFIG_OPROFILE_IBS
 123
 124 static inline int
 125 op_amd_handle_ibs(struct pt_regs * const regs,
 126                   struct op_msrs const * const msrs)
 127 {
 128         u32 low, high;
 129         u64 msr;
 130         struct op_entry entry;
 131
 132         if (!has_ibs)
 133                 return 1;
 134
 135         if (ibs_config.fetch_enabled) {
 136                 rdmsr(MSR_AMD64_IBSFETCHCTL, low, high);
 137                 if (high & IBS_FETCH_HIGH_VALID_BIT) {
 138                         rdmsrl(MSR_AMD64_IBSFETCHLINAD, msr);
 139                         oprofile_write_reserve(&entry, regs, msr,
 140                                                IBS_FETCH_CODE, IBS_FETCH_SIZE);
 141                         oprofile_add_data(&entry, (u32)msr);
 142                         oprofile_add_data(&entry, (u32)(msr >> 32));
 143                         oprofile_add_data(&entry, low);
 144                         oprofile_add_data(&entry, high);
 145                         rdmsrl(MSR_AMD64_IBSFETCHPHYSAD, msr);
 146                         oprofile_add_data(&entry, (u32)msr);
 147                         oprofile_add_data(&entry, (u32)(msr >> 32));
 148                         oprofile_write_commit(&entry);
 149
 150                         /* reenable the IRQ */
 151                         high &= ~IBS_FETCH_HIGH_VALID_BIT;
 152                         high |= IBS_FETCH_HIGH_ENABLE;
 153                         low &= IBS_FETCH_LOW_MAX_CNT_MASK;
 154                         wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
 155                 }
 156         }
 157
 158         if (ibs_config.op_enabled) {
 159                 rdmsr(MSR_AMD64_IBSOPCTL, low, high);
 160                 if (low & IBS_OP_LOW_VALID_BIT) {
 161                         rdmsrl(MSR_AMD64_IBSOPRIP, msr);
 162                         oprofile_write_reserve(&entry, regs, msr,
 163                                                IBS_OP_CODE, IBS_OP_SIZE);
 164                         oprofile_add_data(&entry, (u32)msr);
 165                         oprofile_add_data(&entry, (u32)(msr >> 32));
 166                         rdmsrl(MSR_AMD64_IBSOPDATA, msr);
 167                         oprofile_add_data(&entry, (u32)msr);
 168                         oprofile_add_data(&entry, (u32)(msr >> 32));
 169                         rdmsrl(MSR_AMD64_IBSOPDATA2, msr);
 170                         oprofile_add_data(&entry, (u32)msr);
 171                         oprofile_add_data(&entry, (u32)(msr >> 32));
 172                         rdmsrl(MSR_AMD64_IBSOPDATA3, msr);
 173                         oprofile_add_data(&entry, (u32)msr);
 174                         oprofile_add_data(&entry, (u32)(msr >> 32));
 175                         rdmsrl(MSR_AMD64_IBSDCLINAD, msr);
 176                         oprofile_add_data(&entry, (u32)msr);
 177                         oprofile_add_data(&entry, (u32)(msr >> 32));
 178                         rdmsrl(MSR_AMD64_IBSDCPHYSAD, msr);
 179                         oprofile_add_data(&entry, (u32)msr);
 180                         oprofile_add_data(&entry, (u32)(msr >> 32));
 181                         oprofile_write_commit(&entry);
 182
 183                         /* reenable the IRQ */
 184                         high = 0;
 185                         low &= ~IBS_OP_LOW_VALID_BIT;
 186                         low |= IBS_OP_LOW_ENABLE;
 187                         wrmsr(MSR_AMD64_IBSOPCTL, low, high);
 188                 }
 189         }
 190
 191         return 1;
 192 }
 193
 194 static inline void op_amd_start_ibs(void)
 195 {
 196         unsigned int low, high;
 197         if (has_ibs && ibs_config.fetch_enabled) {
 198                 low = (ibs_config.max_cnt_fetch >> 4) & 0xFFFF;
 199                 high = ((ibs_config.rand_en & 0x1) << 25) /* bit 57 */
 200                         + IBS_FETCH_HIGH_ENABLE;
 201                 wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
 202         }
 203
 204         if (has_ibs && ibs_config.op_enabled) {
 205                 low = ((ibs_config.max_cnt_op >> 4) & 0xFFFF)
 206                         + ((ibs_config.dispatched_ops & 0x1) << 19) /* bit 19 */
 207                         + IBS_OP_LOW_ENABLE;
 208                 high = 0;
 209                 wrmsr(MSR_AMD64_IBSOPCTL, low, high);
 210         }
 211 }
 212
 213 static void op_amd_stop_ibs(void)
 214 {
 215         unsigned int low, high;
 216         if (has_ibs && ibs_config.fetch_enabled) {
 217                 /* clear max count and enable */
 218                 low = 0;
 219                 high = 0;
 220                 wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
 221         }
 222
 223         if (has_ibs && ibs_config.op_enabled) {
 224                 /* clear max count and enable */
 225                 low = 0;
 226                 high = 0;
 227                 wrmsr(MSR_AMD64_IBSOPCTL, low, high);
 228         }
 229 }
 230
 231 #else
 232
 233 static inline int op_amd_handle_ibs(struct pt_regs * const regs,
 234                                     struct op_msrs const * const msrs) { }
 235 static inline void op_amd_start_ibs(void) { }
 236 static inline void op_amd_stop_ibs(void) { }
 237
 238 #endif
 239
 240 static int op_amd_check_ctrs(struct pt_regs * const regs,
 241                              struct op_msrs const * const msrs)
 242 {
 243         u64 val;
 244         int i;
 245
 246         for (i = 0 ; i < NUM_COUNTERS; ++i) {
 247                 if (!reset_value[i])
 248                         continue;
 249                 rdmsrl(msrs->counters[i].addr, val);
 250                 /* bit is clear if overflowed: */
 251                 if (val & OP_CTR_OVERFLOW)
 252                         continue;
 253                 oprofile_add_sample(regs, i);
 254                 wrmsr(msrs->counters[i].addr, -(unsigned int)reset_value[i], -1);
 255         }
 256
 257         op_amd_handle_ibs(regs, msrs);
 258
 259         /* See op_model_ppro.c */
 260         return 1;
 261 }
 262
 263 static void op_amd_start(struct op_msrs const * const msrs)
 264 {
 265         u64 val;
 266         int i;
 267         for (i = 0 ; i < NUM_COUNTERS ; ++i) {
 268                 if (reset_value[i]) {
 269                         rdmsrl(msrs->controls[i].addr, val);
 270                         val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
 271                         wrmsrl(msrs->controls[i].addr, val);
 272                 }
 273         }
 274
 275         op_amd_start_ibs();
 276 }
 277
 278 static void op_amd_stop(struct op_msrs const * const msrs)
 279 {
 280         u64 val;
 281         int i;
 282
 283         /*
 284          * Subtle: stop on all counters to avoid race with setting our
 285          * pm callback
 286          */
 287         for (i = 0 ; i < NUM_COUNTERS ; ++i) {
 288                 if (!reset_value[i])
 289                         continue;
 290                 rdmsrl(msrs->controls[i].addr, val);
 291                 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
 292                 wrmsrl(msrs->controls[i].addr, val);
 293         }
 294
 295         op_amd_stop_ibs();
 296 }
 297
 298 static void op_amd_shutdown(struct op_msrs const * const msrs)
 299 {
 300         int i;
 301
 302         for (i = 0 ; i < NUM_COUNTERS ; ++i) {
 303                 if (CTR_IS_RESERVED(msrs, i))
 304                         release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
 305         }
 306         for (i = 0 ; i < NUM_CONTROLS ; ++i) {
 307                 if (CTRL_IS_RESERVED(msrs, i))
 308                         release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
 309         }
 310 }
 311
 312 #ifdef CONFIG_OPROFILE_IBS
 313
 314 static u8 ibs_eilvt_off;
 315
 316 static inline void apic_init_ibs_nmi_per_cpu(void *arg)
 317 {
 318         ibs_eilvt_off = setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_NMI, 0);
 319 }
 320
 321 static inline void apic_clear_ibs_nmi_per_cpu(void *arg)
 322 {
 323         setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_FIX, 1);
 324 }
 325
 326 static int init_ibs_nmi(void)
 327 {
 328 #define IBSCTL_LVTOFFSETVAL             (1 << 8)
 329 #define IBSCTL                          0x1cc
 330         struct pci_dev *cpu_cfg;
 331         int nodes;
 332         u32 value = 0;
 333
 334         /* per CPU setup */
 335         on_each_cpu(apic_init_ibs_nmi_per_cpu, NULL, 1);
 336
 337         nodes = 0;
 338         cpu_cfg = NULL;
 339         do {
 340                 cpu_cfg = pci_get_device(PCI_VENDOR_ID_AMD,
 341                                          PCI_DEVICE_ID_AMD_10H_NB_MISC,
 342                                          cpu_cfg);
 343                 if (!cpu_cfg)
 344                         break;
 345                 ++nodes;
 346                 pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off
 347                                        | IBSCTL_LVTOFFSETVAL);
 348                 pci_read_config_dword(cpu_cfg, IBSCTL, &value);
 349                 if (value != (ibs_eilvt_off | IBSCTL_LVTOFFSETVAL)) {
 350                         pci_dev_put(cpu_cfg);
 351                         printk(KERN_DEBUG "Failed to setup IBS LVT offset, "
 352                                 "IBSCTL = 0x%08x", value);
 353                         return 1;
 354                 }
 355         } while (1);
 356
 357         if (!nodes) {
 358                 printk(KERN_DEBUG "No CPU node configured for IBS");
 359                 return 1;
 360         }
 361
 362 #ifdef CONFIG_NUMA
 363         /* Sanity check */
 364         /* Works only for 64bit with proper numa implementation. */
 365         if (nodes != num_possible_nodes()) {
 366                 printk(KERN_DEBUG "Failed to setup CPU node(s) for IBS, "
 367                         "found: %d, expected %d",
 368                         nodes, num_possible_nodes());
 369                 return 1;
 370         }
 371 #endif
 372         return 0;
 373 }
 374
 375 /* uninitialize the APIC for the IBS interrupts if needed */
 376 static void clear_ibs_nmi(void)
 377 {
 378         if (has_ibs)
 379                 on_each_cpu(apic_clear_ibs_nmi_per_cpu, NULL, 1);
 380 }
 381
 382 /* initialize the APIC for the IBS interrupts if available */
 383 static void ibs_init(void)
 384 {
 385         has_ibs = boot_cpu_has(X86_FEATURE_IBS);
 386
 387         if (!has_ibs)
 388                 return;
 389
 390         if (init_ibs_nmi()) {
 391                 has_ibs = 0;
 392                 return;
 393         }
 394
 395         printk(KERN_INFO "oprofile: AMD IBS detected\n");
 396 }
 397
 398 static void ibs_exit(void)
 399 {
 400         if (!has_ibs)
 401                 return;
 402
 403         clear_ibs_nmi();
 404 }
 405
 406 static int (*create_arch_files)(struct super_block *sb, struct dentry *root);
 407
 408 static int setup_ibs_files(struct super_block *sb, struct dentry *root)
 409 {
 410         struct dentry *dir;
 411         int ret = 0;
 412
 413         /* architecture specific files */
 414         if (create_arch_files)
 415                 ret = create_arch_files(sb, root);
 416
 417         if (ret)
 418                 return ret;
 419
 420         if (!has_ibs)
 421                 return ret;
 422
 423         /* model specific files */
 424
 425         /* setup some reasonable defaults */
 426         ibs_config.max_cnt_fetch = 250000;
 427         ibs_config.fetch_enabled = 0;
 428         ibs_config.max_cnt_op = 250000;
 429         ibs_config.op_enabled = 0;
 430         ibs_config.dispatched_ops = 1;
 431
 432         dir = oprofilefs_mkdir(sb, root, "ibs_fetch");
 433         oprofilefs_create_ulong(sb, dir, "enable",
 434                                 &ibs_config.fetch_enabled);
 435         oprofilefs_create_ulong(sb, dir, "max_count",
 436                                 &ibs_config.max_cnt_fetch);
 437         oprofilefs_create_ulong(sb, dir, "rand_enable",
 438                                 &ibs_config.rand_en);
 439
 440         dir = oprofilefs_mkdir(sb, root, "ibs_op");
 441         oprofilefs_create_ulong(sb, dir, "enable",
 442                                 &ibs_config.op_enabled);
 443         oprofilefs_create_ulong(sb, dir, "max_count",
 444                                 &ibs_config.max_cnt_op);
 445         oprofilefs_create_ulong(sb, dir, "dispatched_ops",
 446                                 &ibs_config.dispatched_ops);
 447
 448         return 0;
 449 }
 450
 451 static int op_amd_init(struct oprofile_operations *ops)
 452 {
 453         ibs_init();
 454         create_arch_files = ops->create_files;
 455         ops->create_files = setup_ibs_files;
 456         return 0;
 457 }
 458
 459 static void op_amd_exit(void)
 460 {
 461         ibs_exit();
 462 }
 463
 464 #else
 465
 466 /* no IBS support */
 467
 468 static int op_amd_init(struct oprofile_operations *ops)
 469 {
 470         return 0;
 471 }
 472
 473 static void op_amd_exit(void) {}
 474
 475 #endif /* CONFIG_OPROFILE_IBS */
 476
 477 struct op_x86_model_spec const op_amd_spec = {
 478         .num_counters           = NUM_COUNTERS,
 479         .num_controls           = NUM_CONTROLS,
 480         .reserved               = MSR_AMD_EVENTSEL_RESERVED,
 481         .event_mask             = OP_EVENT_MASK,
 482         .init                   = op_amd_init,
 483         .exit                   = op_amd_exit,
 484         .fill_in_addresses      = &op_amd_fill_in_addresses,
 485         .setup_ctrs             = &op_amd_setup_ctrs,
 486         .check_ctrs             = &op_amd_check_ctrs,
 487         .start                  = &op_amd_start,
 488         .stop                   = &op_amd_stop,
 489         .shutdown               = &op_amd_shutdown,
 490 };