arch/x86/oprofile/op_model_amd.c

   1 /*
   2  * @file op_model_amd.c
   3  * athlon / K7 / K8 / Family 10h model-specific MSR operations
   4  *
   5  * @remark Copyright 2002-2009 OProfile authors
   6  * @remark Read the file COPYING
   7  *
   8  * @author John Levon
   9  * @author Philippe Elie
  10  * @author Graydon Hoare
  11  * @author Robert Richter <robert.richter@amd.com>
  12  * @author Barry Kasindorf <barry.kasindorf@amd.com>
  13  * @author Jason Yeh <jason.yeh@amd.com>
  14  * @author Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
  15  */
  16
  17 #include <linux/oprofile.h>
  18 #include <linux/device.h>
  19 #include <linux/pci.h>
  20 #include <linux/percpu.h>
  21
  22 #include <asm/ptrace.h>
  23 #include <asm/msr.h>
  24 #include <asm/nmi.h>
  25 #include <asm/apic.h>
  26 #include <asm/processor.h>
  27 #include <asm/cpufeature.h>
  28
  29 #include "op_x86_model.h"
  30 #include "op_counter.h"
  31
  32 #define NUM_COUNTERS            4
  33 #define NUM_COUNTERS_F15H       6
  34 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
  35 #define NUM_VIRT_COUNTERS       32
  36 #else
  37 #define NUM_VIRT_COUNTERS       0
  38 #endif
  39
  40 #define OP_EVENT_MASK                   0x0FFF
  41 #define OP_CTR_OVERFLOW                 (1ULL<<31)
  42
  43 #define MSR_AMD_EVENTSEL_RESERVED       ((0xFFFFFCF0ULL<<32)|(1ULL<<21))
  44
  45 static int num_counters;
  46 static unsigned long reset_value[OP_MAX_COUNTER];
  47
  48 #define IBS_FETCH_SIZE                  6
  49 #define IBS_OP_SIZE                     12
  50
  51 static u32 ibs_caps;
  52
  53 struct ibs_config {
  54         unsigned long op_enabled;
  55         unsigned long fetch_enabled;
  56         unsigned long max_cnt_fetch;
  57         unsigned long max_cnt_op;
  58         unsigned long rand_en;
  59         unsigned long dispatched_ops;
  60         unsigned long branch_target;
  61 };
  62
  63 struct ibs_state {
  64         u64             ibs_op_ctl;
  65         int             branch_target;
  66         unsigned long   sample_size;
  67 };
  68
  69 static struct ibs_config ibs_config;
  70 static struct ibs_state ibs_state;
  71
  72 /*
  73  * IBS cpuid feature detection
  74  */
  75
  76 #define IBS_CPUID_FEATURES              0x8000001b
  77
  78 /*
  79  * Same bit mask as for IBS cpuid feature flags (Fn8000_001B_EAX), but
  80  * bit 0 is used to indicate the existence of IBS.
  81  */
  82 #define IBS_CAPS_AVAIL                  (1U<<0)
  83 #define IBS_CAPS_FETCHSAM               (1U<<1)
  84 #define IBS_CAPS_OPSAM                  (1U<<2)
  85 #define IBS_CAPS_RDWROPCNT              (1U<<3)
  86 #define IBS_CAPS_OPCNT                  (1U<<4)
  87 #define IBS_CAPS_BRNTRGT                (1U<<5)
  88 #define IBS_CAPS_OPCNTEXT               (1U<<6)
  89
  90 #define IBS_CAPS_DEFAULT                (IBS_CAPS_AVAIL         \
  91                                          | IBS_CAPS_FETCHSAM    \
  92                                          | IBS_CAPS_OPSAM)
  93
  94 /*
  95  * IBS APIC setup
  96  */
  97 #define IBSCTL                          0x1cc
  98 #define IBSCTL_LVT_OFFSET_VALID         (1ULL<<8)
  99 #define IBSCTL_LVT_OFFSET_MASK          0x0F
 100
 101 /*
 102  * IBS randomization macros
 103  */
 104 #define IBS_RANDOM_BITS                 12
 105 #define IBS_RANDOM_MASK                 ((1ULL << IBS_RANDOM_BITS) - 1)
 106 #define IBS_RANDOM_MAXCNT_OFFSET        (1ULL << (IBS_RANDOM_BITS - 5))
 107
 108 static u32 get_ibs_caps(void)
 109 {
 110         u32 ibs_caps;
 111         unsigned int max_level;
 112
 113         if (!boot_cpu_has(X86_FEATURE_IBS))
 114                 return 0;
 115
 116         /* check IBS cpuid feature flags */
 117         max_level = cpuid_eax(0x80000000);
 118         if (max_level < IBS_CPUID_FEATURES)
 119                 return IBS_CAPS_DEFAULT;
 120
 121         ibs_caps = cpuid_eax(IBS_CPUID_FEATURES);
 122         if (!(ibs_caps & IBS_CAPS_AVAIL))
 123                 /* cpuid flags not valid */
 124                 return IBS_CAPS_DEFAULT;
 125
 126         return ibs_caps;
 127 }
 128
 129 /*
 130  * 16-bit Linear Feedback Shift Register (LFSR)
 131  *
 132  *                       16   14   13    11
 133  * Feedback polynomial = X  + X  + X  +  X  + 1
 134  */
 135 static unsigned int lfsr_random(void)
 136 {
 137         static unsigned int lfsr_value = 0xF00D;
 138         unsigned int bit;
 139
 140         /* Compute next bit to shift in */
 141         bit = ((lfsr_value >> 0) ^
 142                (lfsr_value >> 2) ^
 143                (lfsr_value >> 3) ^
 144                (lfsr_value >> 5)) & 0x0001;
 145
 146         /* Advance to next register value */
 147         lfsr_value = (lfsr_value >> 1) | (bit << 15);
 148
 149         return lfsr_value;
 150 }
 151
 152 /*
 153  * IBS software randomization
 154  *
 155  * The IBS periodic op counter is randomized in software. The lower 12
 156  * bits of the 20 bit counter are randomized. IbsOpCurCnt is
 157  * initialized with a 12 bit random value.
 158  */
 159 static inline u64 op_amd_randomize_ibs_op(u64 val)
 160 {
 161         unsigned int random = lfsr_random();
 162
 163         if (!(ibs_caps & IBS_CAPS_RDWROPCNT))
 164                 /*
 165                  * Work around if the hw can not write to IbsOpCurCnt
 166                  *
 167                  * Randomize the lower 8 bits of the 16 bit
 168                  * IbsOpMaxCnt [15:0] value in the range of -128 to
 169                  * +127 by adding/subtracting an offset to the
 170                  * maximum count (IbsOpMaxCnt).
 171                  *
 172                  * To avoid over or underflows and protect upper bits
 173                  * starting at bit 16, the initial value for
 174                  * IbsOpMaxCnt must fit in the range from 0x0081 to
 175                  * 0xff80.
 176                  */
 177                 val += (s8)(random >> 4);
 178         else
 179                 val |= (u64)(random & IBS_RANDOM_MASK) << 32;
 180
 181         return val;
 182 }
 183
 184 static inline void
 185 op_amd_handle_ibs(struct pt_regs * const regs,
 186                   struct op_msrs const * const msrs)
 187 {
 188         u64 val, ctl;
 189         struct op_entry entry;
 190
 191         if (!ibs_caps)
 192                 return;
 193
 194         if (ibs_config.fetch_enabled) {
 195                 rdmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
 196                 if (ctl & IBS_FETCH_VAL) {
 197                         rdmsrl(MSR_AMD64_IBSFETCHLINAD, val);
 198                         oprofile_write_reserve(&entry, regs, val,
 199                                                IBS_FETCH_CODE, IBS_FETCH_SIZE);
 200                         oprofile_add_data64(&entry, val);
 201                         oprofile_add_data64(&entry, ctl);
 202                         rdmsrl(MSR_AMD64_IBSFETCHPHYSAD, val);
 203                         oprofile_add_data64(&entry, val);
 204                         oprofile_write_commit(&entry);
 205
 206                         /* reenable the IRQ */
 207                         ctl &= ~(IBS_FETCH_VAL | IBS_FETCH_CNT);
 208                         ctl |= IBS_FETCH_ENABLE;
 209                         wrmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
 210                 }
 211         }
 212
 213         if (ibs_config.op_enabled) {
 214                 rdmsrl(MSR_AMD64_IBSOPCTL, ctl);
 215                 if (ctl & IBS_OP_VAL) {
 216                         rdmsrl(MSR_AMD64_IBSOPRIP, val);
 217                         oprofile_write_reserve(&entry, regs, val, IBS_OP_CODE,
 218                                                ibs_state.sample_size);
 219                         oprofile_add_data64(&entry, val);
 220                         rdmsrl(MSR_AMD64_IBSOPDATA, val);
 221                         oprofile_add_data64(&entry, val);
 222                         rdmsrl(MSR_AMD64_IBSOPDATA2, val);
 223                         oprofile_add_data64(&entry, val);
 224                         rdmsrl(MSR_AMD64_IBSOPDATA3, val);
 225                         oprofile_add_data64(&entry, val);
 226                         rdmsrl(MSR_AMD64_IBSDCLINAD, val);
 227                         oprofile_add_data64(&entry, val);
 228                         rdmsrl(MSR_AMD64_IBSDCPHYSAD, val);
 229                         oprofile_add_data64(&entry, val);
 230                         if (ibs_state.branch_target) {
 231                                 rdmsrl(MSR_AMD64_IBSBRTARGET, val);
 232                                 oprofile_add_data(&entry, (unsigned long)val);
 233                         }
 234                         oprofile_write_commit(&entry);
 235
 236                         /* reenable the IRQ */
 237                         ctl = op_amd_randomize_ibs_op(ibs_state.ibs_op_ctl);
 238                         wrmsrl(MSR_AMD64_IBSOPCTL, ctl);
 239                 }
 240         }
 241 }
 242
 243 static inline void op_amd_start_ibs(void)
 244 {
 245         u64 val;
 246
 247         if (!ibs_caps)
 248                 return;
 249
 250         memset(&ibs_state, 0, sizeof(ibs_state));
 251
 252         /*
 253          * Note: Since the max count settings may out of range we
 254          * write back the actual used values so that userland can read
 255          * it.
 256          */
 257
 258         if (ibs_config.fetch_enabled) {
 259                 val = ibs_config.max_cnt_fetch >> 4;
 260                 val = min(val, IBS_FETCH_MAX_CNT);
 261                 ibs_config.max_cnt_fetch = val << 4;
 262                 val |= ibs_config.rand_en ? IBS_FETCH_RAND_EN : 0;
 263                 val |= IBS_FETCH_ENABLE;
 264                 wrmsrl(MSR_AMD64_IBSFETCHCTL, val);
 265         }
 266
 267         if (ibs_config.op_enabled) {
 268                 val = ibs_config.max_cnt_op >> 4;
 269                 if (!(ibs_caps & IBS_CAPS_RDWROPCNT)) {
 270                         /*
 271                          * IbsOpCurCnt not supported.  See
 272                          * op_amd_randomize_ibs_op() for details.
 273                          */
 274                         val = clamp(val, 0x0081ULL, 0xFF80ULL);
 275                         ibs_config.max_cnt_op = val << 4;
 276                 } else {
 277                         /*
 278                          * The start value is randomized with a
 279                          * positive offset, we need to compensate it
 280                          * with the half of the randomized range. Also
 281                          * avoid underflows.
 282                          */
 283                         val += IBS_RANDOM_MAXCNT_OFFSET;
 284                         if (ibs_caps & IBS_CAPS_OPCNTEXT)
 285                                 val = min(val, IBS_OP_MAX_CNT_EXT);
 286                         else
 287                                 val = min(val, IBS_OP_MAX_CNT);
 288                         ibs_config.max_cnt_op =
 289                                 (val - IBS_RANDOM_MAXCNT_OFFSET) << 4;
 290                 }
 291                 val = ((val & ~IBS_OP_MAX_CNT) << 4) | (val & IBS_OP_MAX_CNT);
 292                 val |= ibs_config.dispatched_ops ? IBS_OP_CNT_CTL : 0;
 293                 val |= IBS_OP_ENABLE;
 294                 ibs_state.ibs_op_ctl = val;
 295                 ibs_state.sample_size = IBS_OP_SIZE;
 296                 if (ibs_config.branch_target) {
 297                         ibs_state.branch_target = 1;
 298                         ibs_state.sample_size++;
 299                 }
 300                 val = op_amd_randomize_ibs_op(ibs_state.ibs_op_ctl);
 301                 wrmsrl(MSR_AMD64_IBSOPCTL, val);
 302         }
 303 }
 304
 305 static void op_amd_stop_ibs(void)
 306 {
 307         if (!ibs_caps)
 308                 return;
 309
 310         if (ibs_config.fetch_enabled)
 311                 /* clear max count and enable */
 312                 wrmsrl(MSR_AMD64_IBSFETCHCTL, 0);
 313
 314         if (ibs_config.op_enabled)
 315                 /* clear max count and enable */
 316                 wrmsrl(MSR_AMD64_IBSOPCTL, 0);
 317 }
 318
 319 static inline int get_eilvt(int offset)
 320 {
 321         return !setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 1);
 322 }
 323
 324 static inline int put_eilvt(int offset)
 325 {
 326         return !setup_APIC_eilvt(offset, 0, 0, 1);
 327 }
 328
 329 static inline int ibs_eilvt_valid(void)
 330 {
 331         int offset;
 332         u64 val;
 333         int valid = 0;
 334
 335         preempt_disable();
 336
 337         rdmsrl(MSR_AMD64_IBSCTL, val);
 338         offset = val & IBSCTL_LVT_OFFSET_MASK;
 339
 340         if (!(val & IBSCTL_LVT_OFFSET_VALID)) {
 341                 pr_err(FW_BUG "cpu %d, invalid IBS interrupt offset %d (MSR%08X=0x%016llx)\n",
 342                        smp_processor_id(), offset, MSR_AMD64_IBSCTL, val);
 343                 goto out;
 344         }
 345
 346         if (!get_eilvt(offset)) {
 347                 pr_err(FW_BUG "cpu %d, IBS interrupt offset %d not available (MSR%08X=0x%016llx)\n",
 348                        smp_processor_id(), offset, MSR_AMD64_IBSCTL, val);
 349                 goto out;
 350         }
 351
 352         valid = 1;
 353 out:
 354         preempt_enable();
 355
 356         return valid;
 357 }
 358
 359 static inline int get_ibs_offset(void)
 360 {
 361         u64 val;
 362
 363         rdmsrl(MSR_AMD64_IBSCTL, val);
 364         if (!(val & IBSCTL_LVT_OFFSET_VALID))
 365                 return -EINVAL;
 366
 367         return val & IBSCTL_LVT_OFFSET_MASK;
 368 }
 369
 370 static void setup_APIC_ibs(void)
 371 {
 372         int offset;
 373
 374         offset = get_ibs_offset();
 375         if (offset < 0)
 376                 goto failed;
 377
 378         if (!setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 0))
 379                 return;
 380 failed:
 381         pr_warn("oprofile: IBS APIC setup failed on cpu #%d\n",
 382                 smp_processor_id());
 383 }
 384
 385 static void clear_APIC_ibs(void)
 386 {
 387         int offset;
 388
 389         offset = get_ibs_offset();
 390         if (offset >= 0)
 391                 setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_FIX, 1);
 392 }
 393
 394 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
 395
 396 static void op_mux_switch_ctrl(struct op_x86_model_spec const *model,
 397                                struct op_msrs const * const msrs)
 398 {
 399         u64 val;
 400         int i;
 401
 402         /* enable active counters */
 403         for (i = 0; i < num_counters; ++i) {
 404                 int virt = op_x86_phys_to_virt(i);
 405                 if (!reset_value[virt])
 406                         continue;
 407                 rdmsrl(msrs->controls[i].addr, val);
 408                 val &= model->reserved;
 409                 val |= op_x86_get_ctrl(model, &counter_config[virt]);
 410                 wrmsrl(msrs->controls[i].addr, val);
 411         }
 412 }
 413
 414 #endif
 415
 416 /* functions for op_amd_spec */
 417
 418 static void op_amd_shutdown(struct op_msrs const * const msrs)
 419 {
 420         int i;
 421
 422         for (i = 0; i < num_counters; ++i) {
 423                 if (!msrs->counters[i].addr)
 424                         continue;
 425                 release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
 426                 release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
 427         }
 428 }
 429
 430 static int op_amd_fill_in_addresses(struct op_msrs * const msrs)
 431 {
 432         int i;
 433
 434         for (i = 0; i < num_counters; i++) {
 435                 if (!reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))
 436                         goto fail;
 437                 if (!reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i)) {
 438                         release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
 439                         goto fail;
 440                 }
 441                 /* both registers must be reserved */
 442                 if (num_counters == NUM_COUNTERS_F15H) {
 443                         msrs->counters[i].addr = MSR_F15H_PERF_CTR + (i << 1);
 444                         msrs->controls[i].addr = MSR_F15H_PERF_CTL + (i << 1);
 445                 } else {
 446                         msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i;
 447                         msrs->counters[i].addr = MSR_K7_PERFCTR0 + i;
 448                 }
 449                 continue;
 450         fail:
 451                 if (!counter_config[i].enabled)
 452                         continue;
 453                 op_x86_warn_reserved(i);
 454                 op_amd_shutdown(msrs);
 455                 return -EBUSY;
 456         }
 457
 458         return 0;
 459 }
 460
 461 static void op_amd_setup_ctrs(struct op_x86_model_spec const *model,
 462                               struct op_msrs const * const msrs)
 463 {
 464         u64 val;
 465         int i;
 466
 467         /* setup reset_value */
 468         for (i = 0; i < OP_MAX_COUNTER; ++i) {
 469                 if (counter_config[i].enabled
 470                     && msrs->counters[op_x86_virt_to_phys(i)].addr)
 471                         reset_value[i] = counter_config[i].count;
 472                 else
 473                         reset_value[i] = 0;
 474         }
 475
 476         /* clear all counters */
 477         for (i = 0; i < num_counters; ++i) {
 478                 if (!msrs->controls[i].addr)
 479                         continue;
 480                 rdmsrl(msrs->controls[i].addr, val);
 481                 if (val & ARCH_PERFMON_EVENTSEL_ENABLE)
 482                         op_x86_warn_in_use(i);
 483                 val &= model->reserved;
 484                 wrmsrl(msrs->controls[i].addr, val);
 485                 /*
 486                  * avoid a false detection of ctr overflows in NMI
 487                  * handler
 488                  */
 489                 wrmsrl(msrs->counters[i].addr, -1LL);
 490         }
 491
 492         /* enable active counters */
 493         for (i = 0; i < num_counters; ++i) {
 494                 int virt = op_x86_phys_to_virt(i);
 495                 if (!reset_value[virt])
 496                         continue;
 497
 498                 /* setup counter registers */
 499                 wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);
 500
 501                 /* setup control registers */
 502                 rdmsrl(msrs->controls[i].addr, val);
 503                 val &= model->reserved;
 504                 val |= op_x86_get_ctrl(model, &counter_config[virt]);
 505                 wrmsrl(msrs->controls[i].addr, val);
 506         }
 507
 508         if (ibs_caps)
 509                 setup_APIC_ibs();
 510 }
 511
 512 static void op_amd_cpu_shutdown(void)
 513 {
 514         if (ibs_caps)
 515                 clear_APIC_ibs();
 516 }
 517
 518 static int op_amd_check_ctrs(struct pt_regs * const regs,
 519                              struct op_msrs const * const msrs)
 520 {
 521         u64 val;
 522         int i;
 523
 524         for (i = 0; i < num_counters; ++i) {
 525                 int virt = op_x86_phys_to_virt(i);
 526                 if (!reset_value[virt])
 527                         continue;
 528                 rdmsrl(msrs->counters[i].addr, val);
 529                 /* bit is clear if overflowed: */
 530                 if (val & OP_CTR_OVERFLOW)
 531                         continue;
 532                 oprofile_add_sample(regs, virt);
 533                 wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);
 534         }
 535
 536         op_amd_handle_ibs(regs, msrs);
 537
 538         /* See op_model_ppro.c */
 539         return 1;
 540 }
 541
 542 static void op_amd_start(struct op_msrs const * const msrs)
 543 {
 544         u64 val;
 545         int i;
 546
 547         for (i = 0; i < num_counters; ++i) {
 548                 if (!reset_value[op_x86_phys_to_virt(i)])
 549                         continue;
 550                 rdmsrl(msrs->controls[i].addr, val);
 551                 val |= ARCH_PERFMON_EVENTSEL_ENABLE;
 552                 wrmsrl(msrs->controls[i].addr, val);
 553         }
 554
 555         op_amd_start_ibs();
 556 }
 557
 558 static void op_amd_stop(struct op_msrs const * const msrs)
 559 {
 560         u64 val;
 561         int i;
 562
 563         /*
 564          * Subtle: stop on all counters to avoid race with setting our
 565          * pm callback
 566          */
 567         for (i = 0; i < num_counters; ++i) {
 568                 if (!reset_value[op_x86_phys_to_virt(i)])
 569                         continue;
 570                 rdmsrl(msrs->controls[i].addr, val);
 571                 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
 572                 wrmsrl(msrs->controls[i].addr, val);
 573         }
 574
 575         op_amd_stop_ibs();
 576 }
 577
 578 static int setup_ibs_ctl(int ibs_eilvt_off)
 579 {
 580         struct pci_dev *cpu_cfg;
 581         int nodes;
 582         u32 value = 0;
 583
 584         nodes = 0;
 585         cpu_cfg = NULL;
 586         do {
 587                 cpu_cfg = pci_get_device(PCI_VENDOR_ID_AMD,
 588                                          PCI_DEVICE_ID_AMD_10H_NB_MISC,
 589                                          cpu_cfg);
 590                 if (!cpu_cfg)
 591                         break;
 592                 ++nodes;
 593                 pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off
 594                                        | IBSCTL_LVT_OFFSET_VALID);
 595                 pci_read_config_dword(cpu_cfg, IBSCTL, &value);
 596                 if (value != (ibs_eilvt_off | IBSCTL_LVT_OFFSET_VALID)) {
 597                         pci_dev_put(cpu_cfg);
 598                         printk(KERN_DEBUG "Failed to setup IBS LVT offset, "
 599                                "IBSCTL = 0x%08x\n", value);
 600                         return -EINVAL;
 601                 }
 602         } while (1);
 603
 604         if (!nodes) {
 605                 printk(KERN_DEBUG "No CPU node configured for IBS\n");
 606                 return -ENODEV;
 607         }
 608
 609         return 0;
 610 }
 611
 612 /*
 613  * This runs only on the current cpu. We try to find an LVT offset and
 614  * setup the local APIC. For this we must disable preemption. On
 615  * success we initialize all nodes with this offset. This updates then
 616  * the offset in the IBS_CTL per-node msr. The per-core APIC setup of
 617  * the IBS interrupt vector is called from op_amd_setup_ctrs()/op_-
 618  * amd_cpu_shutdown() using the new offset.
 619  */
 620 static int force_ibs_eilvt_setup(void)
 621 {
 622         int offset;
 623         int ret;
 624
 625         preempt_disable();
 626         /* find the next free available EILVT entry, skip offset 0 */
 627         for (offset = 1; offset < APIC_EILVT_NR_MAX; offset++) {
 628                 if (get_eilvt(offset))
 629                         break;
 630         }
 631         preempt_enable();
 632
 633         if (offset == APIC_EILVT_NR_MAX) {
 634                 printk(KERN_DEBUG "No EILVT entry available\n");
 635                 return -EBUSY;
 636         }
 637
 638         ret = setup_ibs_ctl(offset);
 639         if (ret)
 640                 goto out;
 641
 642         if (!ibs_eilvt_valid()) {
 643                 ret = -EFAULT;
 644                 goto out;
 645         }
 646
 647         pr_err(FW_BUG "using offset %d for IBS interrupts\n", offset);
 648         pr_err(FW_BUG "workaround enabled for IBS LVT offset\n");
 649
 650         return 0;
 651 out:
 652         preempt_disable();
 653         put_eilvt(offset);
 654         preempt_enable();
 655         return ret;
 656 }
 657
 658 /*
 659  * check and reserve APIC extended interrupt LVT offset for IBS if
 660  * available
 661  */
 662
 663 static void init_ibs(void)
 664 {
 665         ibs_caps = get_ibs_caps();
 666
 667         if (!ibs_caps)
 668                 return;
 669
 670         if (ibs_eilvt_valid())
 671                 goto out;
 672
 673         if (!force_ibs_eilvt_setup())
 674                 goto out;
 675
 676         /* Failed to setup ibs */
 677         ibs_caps = 0;
 678         return;
 679
 680 out:
 681         printk(KERN_INFO "oprofile: AMD IBS detected (0x%08x)\n", ibs_caps);
 682 }
 683
 684 static int (*create_arch_files)(struct super_block *sb, struct dentry *root);
 685
 686 static int setup_ibs_files(struct super_block *sb, struct dentry *root)
 687 {
 688         struct dentry *dir;
 689         int ret = 0;
 690
 691         /* architecture specific files */
 692         if (create_arch_files)
 693                 ret = create_arch_files(sb, root);
 694
 695         if (ret)
 696                 return ret;
 697
 698         if (!ibs_caps)
 699                 return ret;
 700
 701         /* model specific files */
 702
 703         /* setup some reasonable defaults */
 704         memset(&ibs_config, 0, sizeof(ibs_config));
 705         ibs_config.max_cnt_fetch = 250000;
 706         ibs_config.max_cnt_op = 250000;
 707
 708         if (ibs_caps & IBS_CAPS_FETCHSAM) {
 709                 dir = oprofilefs_mkdir(sb, root, "ibs_fetch");
 710                 oprofilefs_create_ulong(sb, dir, "enable",
 711                                         &ibs_config.fetch_enabled);
 712                 oprofilefs_create_ulong(sb, dir, "max_count",
 713                                         &ibs_config.max_cnt_fetch);
 714                 oprofilefs_create_ulong(sb, dir, "rand_enable",
 715                                         &ibs_config.rand_en);
 716         }
 717
 718         if (ibs_caps & IBS_CAPS_OPSAM) {
 719                 dir = oprofilefs_mkdir(sb, root, "ibs_op");
 720                 oprofilefs_create_ulong(sb, dir, "enable",
 721                                         &ibs_config.op_enabled);
 722                 oprofilefs_create_ulong(sb, dir, "max_count",
 723                                         &ibs_config.max_cnt_op);
 724                 if (ibs_caps & IBS_CAPS_OPCNT)
 725                         oprofilefs_create_ulong(sb, dir, "dispatched_ops",
 726                                                 &ibs_config.dispatched_ops);
 727                 if (ibs_caps & IBS_CAPS_BRNTRGT)
 728                         oprofilefs_create_ulong(sb, dir, "branch_target",
 729                                                 &ibs_config.branch_target);
 730         }
 731
 732         return 0;
 733 }
 734
 735 struct op_x86_model_spec op_amd_spec;
 736
 737 static int op_amd_init(struct oprofile_operations *ops)
 738 {
 739         init_ibs();
 740         create_arch_files = ops->create_files;
 741         ops->create_files = setup_ibs_files;
 742
 743         if (boot_cpu_data.x86 == 0x15) {
 744                 num_counters = NUM_COUNTERS_F15H;
 745         } else {
 746                 num_counters = NUM_COUNTERS;
 747         }
 748
 749         op_amd_spec.num_counters = num_counters;
 750         op_amd_spec.num_controls = num_counters;
 751         op_amd_spec.num_virt_counters = max(num_counters, NUM_VIRT_COUNTERS);
 752
 753         return 0;
 754 }
 755
 756 struct op_x86_model_spec op_amd_spec = {
 757         /* num_counters/num_controls filled in at runtime */
 758         .reserved               = MSR_AMD_EVENTSEL_RESERVED,
 759         .event_mask             = OP_EVENT_MASK,
 760         .init                   = op_amd_init,
 761         .fill_in_addresses      = &op_amd_fill_in_addresses,
 762         .setup_ctrs             = &op_amd_setup_ctrs,
 763         .cpu_down               = &op_amd_cpu_shutdown,
 764         .check_ctrs             = &op_amd_check_ctrs,
 765         .start                  = &op_amd_start,
 766         .stop                   = &op_amd_stop,
 767         .shutdown               = &op_amd_shutdown,
 768 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
 769         .switch_ctrl            = &op_mux_switch_ctrl,
 770 #endif
 771 };