arch/x86/kernel/cpu/perf_event_intel.c

   1 #ifdef CONFIG_CPU_SUP_INTEL
   2
   3 #define MAX_EXTRA_REGS 2
   4
   5 /*
   6  * Per register state.
   7  */
   8 struct er_account {
   9         int                     ref;            /* reference count */
  10         unsigned int            extra_reg;      /* extra MSR number */
  11         u64                     extra_config;   /* extra MSR config */
  12 };
  13
  14 /*
  15  * Per core state
  16  * This used to coordinate shared registers for HT threads.
  17  */
  18 struct intel_percore {
  19         raw_spinlock_t          lock;           /* protect structure */
  20         struct er_account       regs[MAX_EXTRA_REGS];
  21         int                     refcnt;         /* number of threads */
  22         unsigned                core_id;
  23 };
  24
  25 /*
  26  * Intel PerfMon, used on Core and later.
  27  */
  28 static const u64 intel_perfmon_event_map[] =
  29 {
  30   [PERF_COUNT_HW_CPU_CYCLES]            = 0x003c,
  31   [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
  32   [PERF_COUNT_HW_CACHE_REFERENCES]      = 0x4f2e,
  33   [PERF_COUNT_HW_CACHE_MISSES]          = 0x412e,
  34   [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]   = 0x00c4,
  35   [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
  36   [PERF_COUNT_HW_BUS_CYCLES]            = 0x013c,
  37 };
  38
  39 static struct event_constraint intel_core_event_constraints[] =
  40 {
  41         INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */
  42         INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
  43         INTEL_EVENT_CONSTRAINT(0x13, 0x2), /* DIV */
  44         INTEL_EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */
  45         INTEL_EVENT_CONSTRAINT(0x19, 0x2), /* DELAYED_BYPASS */
  46         INTEL_EVENT_CONSTRAINT(0xc1, 0x1), /* FP_COMP_INSTR_RET */
  47         EVENT_CONSTRAINT_END
  48 };
  49
  50 static struct event_constraint intel_core2_event_constraints[] =
  51 {
  52         FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  53         FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  54         /*
  55          * Core2 has Fixed Counter 2 listed as CPU_CLK_UNHALTED.REF and event
  56          * 0x013c as CPU_CLK_UNHALTED.BUS and specifies there is a fixed
  57          * ratio between these counters.
  58          */
  59         /* FIXED_EVENT_CONSTRAINT(0x013c, 2),  CPU_CLK_UNHALTED.REF */
  60         INTEL_EVENT_CONSTRAINT(0x10, 0x1), /* FP_COMP_OPS_EXE */
  61         INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */
  62         INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
  63         INTEL_EVENT_CONSTRAINT(0x13, 0x2), /* DIV */
  64         INTEL_EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */
  65         INTEL_EVENT_CONSTRAINT(0x18, 0x1), /* IDLE_DURING_DIV */
  66         INTEL_EVENT_CONSTRAINT(0x19, 0x2), /* DELAYED_BYPASS */
  67         INTEL_EVENT_CONSTRAINT(0xa1, 0x1), /* RS_UOPS_DISPATCH_CYCLES */
  68         INTEL_EVENT_CONSTRAINT(0xc9, 0x1), /* ITLB_MISS_RETIRED (T30-9) */
  69         INTEL_EVENT_CONSTRAINT(0xcb, 0x1), /* MEM_LOAD_RETIRED */
  70         EVENT_CONSTRAINT_END
  71 };
  72
  73 static struct event_constraint intel_nehalem_event_constraints[] =
  74 {
  75         FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  76         FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  77         /* FIXED_EVENT_CONSTRAINT(0x013c, 2), CPU_CLK_UNHALTED.REF */
  78         INTEL_EVENT_CONSTRAINT(0x40, 0x3), /* L1D_CACHE_LD */
  79         INTEL_EVENT_CONSTRAINT(0x41, 0x3), /* L1D_CACHE_ST */
  80         INTEL_EVENT_CONSTRAINT(0x42, 0x3), /* L1D_CACHE_LOCK */
  81         INTEL_EVENT_CONSTRAINT(0x43, 0x3), /* L1D_ALL_REF */
  82         INTEL_EVENT_CONSTRAINT(0x48, 0x3), /* L1D_PEND_MISS */
  83         INTEL_EVENT_CONSTRAINT(0x4e, 0x3), /* L1D_PREFETCH */
  84         INTEL_EVENT_CONSTRAINT(0x51, 0x3), /* L1D */
  85         INTEL_EVENT_CONSTRAINT(0x63, 0x3), /* CACHE_LOCK_CYCLES */
  86         EVENT_CONSTRAINT_END
  87 };
  88
  89 static struct extra_reg intel_nehalem_extra_regs[] =
  90 {
  91         INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff),
  92         EVENT_EXTRA_END
  93 };
  94
  95 static struct event_constraint intel_nehalem_percore_constraints[] =
  96 {
  97         INTEL_EVENT_CONSTRAINT(0xb7, 0),
  98         EVENT_CONSTRAINT_END
  99 };
 100
 101 static struct event_constraint intel_westmere_event_constraints[] =
 102 {
 103         FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
 104         FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
 105         /* FIXED_EVENT_CONSTRAINT(0x013c, 2), CPU_CLK_UNHALTED.REF */
 106         INTEL_EVENT_CONSTRAINT(0x51, 0x3), /* L1D */
 107         INTEL_EVENT_CONSTRAINT(0x60, 0x1), /* OFFCORE_REQUESTS_OUTSTANDING */
 108         INTEL_EVENT_CONSTRAINT(0x63, 0x3), /* CACHE_LOCK_CYCLES */
 109         INTEL_EVENT_CONSTRAINT(0xb3, 0x1), /* SNOOPQ_REQUEST_OUTSTANDING */
 110         EVENT_CONSTRAINT_END
 111 };
 112
 113 static struct event_constraint intel_snb_event_constraints[] =
 114 {
 115         FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
 116         FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
 117         /* FIXED_EVENT_CONSTRAINT(0x013c, 2), CPU_CLK_UNHALTED.REF */
 118         INTEL_EVENT_CONSTRAINT(0x48, 0x4), /* L1D_PEND_MISS.PENDING */
 119         INTEL_EVENT_CONSTRAINT(0xb7, 0x1), /* OFF_CORE_RESPONSE_0 */
 120         INTEL_EVENT_CONSTRAINT(0xbb, 0x8), /* OFF_CORE_RESPONSE_1 */
 121         INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */
 122         INTEL_EVENT_CONSTRAINT(0xcd, 0x8), /* MEM_TRANS_RETIRED.LOAD_LATENCY */
 123         EVENT_CONSTRAINT_END
 124 };
 125
 126 static struct extra_reg intel_westmere_extra_regs[] =
 127 {
 128         INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff),
 129         INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0xffff),
 130         EVENT_EXTRA_END
 131 };
 132
 133 static struct event_constraint intel_westmere_percore_constraints[] =
 134 {
 135         INTEL_EVENT_CONSTRAINT(0xb7, 0),
 136         INTEL_EVENT_CONSTRAINT(0xbb, 0),
 137         EVENT_CONSTRAINT_END
 138 };
 139
 140 static struct event_constraint intel_gen_event_constraints[] =
 141 {
 142         FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
 143         FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
 144         /* FIXED_EVENT_CONSTRAINT(0x013c, 2), CPU_CLK_UNHALTED.REF */
 145         EVENT_CONSTRAINT_END
 146 };
 147
 148 static u64 intel_pmu_event_map(int hw_event)
 149 {
 150         return intel_perfmon_event_map[hw_event];
 151 }
 152
 153 static __initconst const u64 snb_hw_cache_event_ids
 154                                 [PERF_COUNT_HW_CACHE_MAX]
 155                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 156                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 157 {
 158  [ C(L1D) ] = {
 159         [ C(OP_READ) ] = {
 160                 [ C(RESULT_ACCESS) ] = 0xf1d0, /* MEM_UOP_RETIRED.LOADS        */
 161                 [ C(RESULT_MISS)   ] = 0x0151, /* L1D.REPLACEMENT              */
 162         },
 163         [ C(OP_WRITE) ] = {
 164                 [ C(RESULT_ACCESS) ] = 0xf2d0, /* MEM_UOP_RETIRED.STORES       */
 165                 [ C(RESULT_MISS)   ] = 0x0851, /* L1D.ALL_M_REPLACEMENT        */
 166         },
 167         [ C(OP_PREFETCH) ] = {
 168                 [ C(RESULT_ACCESS) ] = 0x0,
 169                 [ C(RESULT_MISS)   ] = 0x024e, /* HW_PRE_REQ.DL1_MISS          */
 170         },
 171  },
 172  [ C(L1I ) ] = {
 173         [ C(OP_READ) ] = {
 174                 [ C(RESULT_ACCESS) ] = 0x0,
 175                 [ C(RESULT_MISS)   ] = 0x0280, /* ICACHE.MISSES */
 176         },
 177         [ C(OP_WRITE) ] = {
 178                 [ C(RESULT_ACCESS) ] = -1,
 179                 [ C(RESULT_MISS)   ] = -1,
 180         },
 181         [ C(OP_PREFETCH) ] = {
 182                 [ C(RESULT_ACCESS) ] = 0x0,
 183                 [ C(RESULT_MISS)   ] = 0x0,
 184         },
 185  },
 186  [ C(LL  ) ] = {
 187         /*
 188          * TBD: Need Off-core Response Performance Monitoring support
 189          */
 190         [ C(OP_READ) ] = {
 191                 /* OFFCORE_RESPONSE_0.ANY_DATA.LOCAL_CACHE */
 192                 [ C(RESULT_ACCESS) ] = 0x01b7,
 193                 /* OFFCORE_RESPONSE_1.ANY_DATA.ANY_LLC_MISS */
 194                 [ C(RESULT_MISS)   ] = 0x01bb,
 195         },
 196         [ C(OP_WRITE) ] = {
 197                 /* OFFCORE_RESPONSE_0.ANY_RFO.LOCAL_CACHE */
 198                 [ C(RESULT_ACCESS) ] = 0x01b7,
 199                 /* OFFCORE_RESPONSE_1.ANY_RFO.ANY_LLC_MISS */
 200                 [ C(RESULT_MISS)   ] = 0x01bb,
 201         },
 202         [ C(OP_PREFETCH) ] = {
 203                 /* OFFCORE_RESPONSE_0.PREFETCH.LOCAL_CACHE */
 204                 [ C(RESULT_ACCESS) ] = 0x01b7,
 205                 /* OFFCORE_RESPONSE_1.PREFETCH.ANY_LLC_MISS */
 206                 [ C(RESULT_MISS)   ] = 0x01bb,
 207         },
 208  },
 209  [ C(DTLB) ] = {
 210         [ C(OP_READ) ] = {
 211                 [ C(RESULT_ACCESS) ] = 0x81d0, /* MEM_UOP_RETIRED.ALL_LOADS */
 212                 [ C(RESULT_MISS)   ] = 0x0108, /* DTLB_LOAD_MISSES.CAUSES_A_WALK */
 213         },
 214         [ C(OP_WRITE) ] = {
 215                 [ C(RESULT_ACCESS) ] = 0x82d0, /* MEM_UOP_RETIRED.ALL_STORES */
 216                 [ C(RESULT_MISS)   ] = 0x0149, /* DTLB_STORE_MISSES.MISS_CAUSES_A_WALK */
 217         },
 218         [ C(OP_PREFETCH) ] = {
 219                 [ C(RESULT_ACCESS) ] = 0x0,
 220                 [ C(RESULT_MISS)   ] = 0x0,
 221         },
 222  },
 223  [ C(ITLB) ] = {
 224         [ C(OP_READ) ] = {
 225                 [ C(RESULT_ACCESS) ] = 0x1085, /* ITLB_MISSES.STLB_HIT         */
 226                 [ C(RESULT_MISS)   ] = 0x0185, /* ITLB_MISSES.CAUSES_A_WALK    */
 227         },
 228         [ C(OP_WRITE) ] = {
 229                 [ C(RESULT_ACCESS) ] = -1,
 230                 [ C(RESULT_MISS)   ] = -1,
 231         },
 232         [ C(OP_PREFETCH) ] = {
 233                 [ C(RESULT_ACCESS) ] = -1,
 234                 [ C(RESULT_MISS)   ] = -1,
 235         },
 236  },
 237  [ C(BPU ) ] = {
 238         [ C(OP_READ) ] = {
 239                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
 240                 [ C(RESULT_MISS)   ] = 0x00c5, /* BR_MISP_RETIRED.ALL_BRANCHES */
 241         },
 242         [ C(OP_WRITE) ] = {
 243                 [ C(RESULT_ACCESS) ] = -1,
 244                 [ C(RESULT_MISS)   ] = -1,
 245         },
 246         [ C(OP_PREFETCH) ] = {
 247                 [ C(RESULT_ACCESS) ] = -1,
 248                 [ C(RESULT_MISS)   ] = -1,
 249         },
 250  },
 251 };
 252
 253 static __initconst const u64 westmere_hw_cache_event_ids
 254                                 [PERF_COUNT_HW_CACHE_MAX]
 255                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 256                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 257 {
 258  [ C(L1D) ] = {
 259         [ C(OP_READ) ] = {
 260                 [ C(RESULT_ACCESS) ] = 0x010b, /* MEM_INST_RETIRED.LOADS       */
 261                 [ C(RESULT_MISS)   ] = 0x0151, /* L1D.REPL                     */
 262         },
 263         [ C(OP_WRITE) ] = {
 264                 [ C(RESULT_ACCESS) ] = 0x020b, /* MEM_INST_RETURED.STORES      */
 265                 [ C(RESULT_MISS)   ] = 0x0251, /* L1D.M_REPL                   */
 266         },
 267         [ C(OP_PREFETCH) ] = {
 268                 [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS        */
 269                 [ C(RESULT_MISS)   ] = 0x024e, /* L1D_PREFETCH.MISS            */
 270         },
 271  },
 272  [ C(L1I ) ] = {
 273         [ C(OP_READ) ] = {
 274                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                    */
 275                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                   */
 276         },
 277         [ C(OP_WRITE) ] = {
 278                 [ C(RESULT_ACCESS) ] = -1,
 279                 [ C(RESULT_MISS)   ] = -1,
 280         },
 281         [ C(OP_PREFETCH) ] = {
 282                 [ C(RESULT_ACCESS) ] = 0x0,
 283                 [ C(RESULT_MISS)   ] = 0x0,
 284         },
 285  },
 286  [ C(LL  ) ] = {
 287         [ C(OP_READ) ] = {
 288                 /* OFFCORE_RESPONSE_0.ANY_DATA.LOCAL_CACHE */
 289                 [ C(RESULT_ACCESS) ] = 0x01b7,
 290                 /* OFFCORE_RESPONSE_1.ANY_DATA.ANY_LLC_MISS */
 291                 [ C(RESULT_MISS)   ] = 0x01bb,
 292         },
 293         /*
 294          * Use RFO, not WRITEBACK, because a write miss would typically occur
 295          * on RFO.
 296          */
 297         [ C(OP_WRITE) ] = {
 298                 /* OFFCORE_RESPONSE_1.ANY_RFO.LOCAL_CACHE */
 299                 [ C(RESULT_ACCESS) ] = 0x01bb,
 300                 /* OFFCORE_RESPONSE_0.ANY_RFO.ANY_LLC_MISS */
 301                 [ C(RESULT_MISS)   ] = 0x01b7,
 302         },
 303         [ C(OP_PREFETCH) ] = {
 304                 /* OFFCORE_RESPONSE_0.PREFETCH.LOCAL_CACHE */
 305                 [ C(RESULT_ACCESS) ] = 0x01b7,
 306                 /* OFFCORE_RESPONSE_1.PREFETCH.ANY_LLC_MISS */
 307                 [ C(RESULT_MISS)   ] = 0x01bb,
 308         },
 309  },
 310  [ C(DTLB) ] = {
 311         [ C(OP_READ) ] = {
 312                 [ C(RESULT_ACCESS) ] = 0x010b, /* MEM_INST_RETIRED.LOADS       */
 313                 [ C(RESULT_MISS)   ] = 0x0108, /* DTLB_LOAD_MISSES.ANY         */
 314         },
 315         [ C(OP_WRITE) ] = {
 316                 [ C(RESULT_ACCESS) ] = 0x020b, /* MEM_INST_RETURED.STORES      */
 317                 [ C(RESULT_MISS)   ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS  */
 318         },
 319         [ C(OP_PREFETCH) ] = {
 320                 [ C(RESULT_ACCESS) ] = 0x0,
 321                 [ C(RESULT_MISS)   ] = 0x0,
 322         },
 323  },
 324  [ C(ITLB) ] = {
 325         [ C(OP_READ) ] = {
 326                 [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P           */
 327                 [ C(RESULT_MISS)   ] = 0x0185, /* ITLB_MISSES.ANY              */
 328         },
 329         [ C(OP_WRITE) ] = {
 330                 [ C(RESULT_ACCESS) ] = -1,
 331                 [ C(RESULT_MISS)   ] = -1,
 332         },
 333         [ C(OP_PREFETCH) ] = {
 334                 [ C(RESULT_ACCESS) ] = -1,
 335                 [ C(RESULT_MISS)   ] = -1,
 336         },
 337  },
 338  [ C(BPU ) ] = {
 339         [ C(OP_READ) ] = {
 340                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
 341                 [ C(RESULT_MISS)   ] = 0x03e8, /* BPU_CLEARS.ANY               */
 342         },
 343         [ C(OP_WRITE) ] = {
 344                 [ C(RESULT_ACCESS) ] = -1,
 345                 [ C(RESULT_MISS)   ] = -1,
 346         },
 347         [ C(OP_PREFETCH) ] = {
 348                 [ C(RESULT_ACCESS) ] = -1,
 349                 [ C(RESULT_MISS)   ] = -1,
 350         },
 351  },
 352 };
 353
 354 /*
 355  * OFFCORE_RESPONSE MSR bits (subset), See IA32 SDM Vol 3 30.6.1.3
 356  */
 357
 358 #define DMND_DATA_RD     (1 << 0)
 359 #define DMND_RFO         (1 << 1)
 360 #define DMND_WB          (1 << 3)
 361 #define PF_DATA_RD       (1 << 4)
 362 #define PF_DATA_RFO      (1 << 5)
 363 #define RESP_UNCORE_HIT  (1 << 8)
 364 #define RESP_MISS        (0xf600) /* non uncore hit */
 365
 366 static __initconst const u64 nehalem_hw_cache_extra_regs
 367                                 [PERF_COUNT_HW_CACHE_MAX]
 368                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 369                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 370 {
 371  [ C(LL  ) ] = {
 372         [ C(OP_READ) ] = {
 373                 [ C(RESULT_ACCESS) ] = DMND_DATA_RD|RESP_UNCORE_HIT,
 374                 [ C(RESULT_MISS)   ] = DMND_DATA_RD|RESP_MISS,
 375         },
 376         [ C(OP_WRITE) ] = {
 377                 [ C(RESULT_ACCESS) ] = DMND_RFO|DMND_WB|RESP_UNCORE_HIT,
 378                 [ C(RESULT_MISS)   ] = DMND_RFO|DMND_WB|RESP_MISS,
 379         },
 380         [ C(OP_PREFETCH) ] = {
 381                 [ C(RESULT_ACCESS) ] = PF_DATA_RD|PF_DATA_RFO|RESP_UNCORE_HIT,
 382                 [ C(RESULT_MISS)   ] = PF_DATA_RD|PF_DATA_RFO|RESP_MISS,
 383         },
 384  }
 385 };
 386
 387 static __initconst const u64 nehalem_hw_cache_event_ids
 388                                 [PERF_COUNT_HW_CACHE_MAX]
 389                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 390                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 391 {
 392  [ C(L1D) ] = {
 393         [ C(OP_READ) ] = {
 394                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI            */
 395                 [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE         */
 396         },
 397         [ C(OP_WRITE) ] = {
 398                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI            */
 399                 [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE         */
 400         },
 401         [ C(OP_PREFETCH) ] = {
 402                 [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS        */
 403                 [ C(RESULT_MISS)   ] = 0x024e, /* L1D_PREFETCH.MISS            */
 404         },
 405  },
 406  [ C(L1I ) ] = {
 407         [ C(OP_READ) ] = {
 408                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                    */
 409                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                   */
 410         },
 411         [ C(OP_WRITE) ] = {
 412                 [ C(RESULT_ACCESS) ] = -1,
 413                 [ C(RESULT_MISS)   ] = -1,
 414         },
 415         [ C(OP_PREFETCH) ] = {
 416                 [ C(RESULT_ACCESS) ] = 0x0,
 417                 [ C(RESULT_MISS)   ] = 0x0,
 418         },
 419  },
 420  [ C(LL  ) ] = {
 421         [ C(OP_READ) ] = {
 422                 /* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */
 423                 [ C(RESULT_ACCESS) ] = 0x01b7,
 424                 /* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */
 425                 [ C(RESULT_MISS)   ] = 0x01b7,
 426         },
 427         /*
 428          * Use RFO, not WRITEBACK, because a write miss would typically occur
 429          * on RFO.
 430          */
 431         [ C(OP_WRITE) ] = {
 432                 /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */
 433                 [ C(RESULT_ACCESS) ] = 0x01b7,
 434                 /* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */
 435                 [ C(RESULT_MISS)   ] = 0x01b7,
 436         },
 437         [ C(OP_PREFETCH) ] = {
 438                 /* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */
 439                 [ C(RESULT_ACCESS) ] = 0x01b7,
 440                 /* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */
 441                 [ C(RESULT_MISS)   ] = 0x01b7,
 442         },
 443  },
 444  [ C(DTLB) ] = {
 445         [ C(OP_READ) ] = {
 446                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI   (alias)  */
 447                 [ C(RESULT_MISS)   ] = 0x0108, /* DTLB_LOAD_MISSES.ANY         */
 448         },
 449         [ C(OP_WRITE) ] = {
 450                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI   (alias)  */
 451                 [ C(RESULT_MISS)   ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS  */
 452         },
 453         [ C(OP_PREFETCH) ] = {
 454                 [ C(RESULT_ACCESS) ] = 0x0,
 455                 [ C(RESULT_MISS)   ] = 0x0,
 456         },
 457  },
 458  [ C(ITLB) ] = {
 459         [ C(OP_READ) ] = {
 460                 [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P           */
 461                 [ C(RESULT_MISS)   ] = 0x20c8, /* ITLB_MISS_RETIRED            */
 462         },
 463         [ C(OP_WRITE) ] = {
 464                 [ C(RESULT_ACCESS) ] = -1,
 465                 [ C(RESULT_MISS)   ] = -1,
 466         },
 467         [ C(OP_PREFETCH) ] = {
 468                 [ C(RESULT_ACCESS) ] = -1,
 469                 [ C(RESULT_MISS)   ] = -1,
 470         },
 471  },
 472  [ C(BPU ) ] = {
 473         [ C(OP_READ) ] = {
 474                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
 475                 [ C(RESULT_MISS)   ] = 0x03e8, /* BPU_CLEARS.ANY               */
 476         },
 477         [ C(OP_WRITE) ] = {
 478                 [ C(RESULT_ACCESS) ] = -1,
 479                 [ C(RESULT_MISS)   ] = -1,
 480         },
 481         [ C(OP_PREFETCH) ] = {
 482                 [ C(RESULT_ACCESS) ] = -1,
 483                 [ C(RESULT_MISS)   ] = -1,
 484         },
 485  },
 486 };
 487
 488 static __initconst const u64 core2_hw_cache_event_ids
 489                                 [PERF_COUNT_HW_CACHE_MAX]
 490                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 491                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 492 {
 493  [ C(L1D) ] = {
 494         [ C(OP_READ) ] = {
 495                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI          */
 496                 [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE       */
 497         },
 498         [ C(OP_WRITE) ] = {
 499                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI          */
 500                 [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE       */
 501         },
 502         [ C(OP_PREFETCH) ] = {
 503                 [ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS      */
 504                 [ C(RESULT_MISS)   ] = 0,
 505         },
 506  },
 507  [ C(L1I ) ] = {
 508         [ C(OP_READ) ] = {
 509                 [ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS                  */
 510                 [ C(RESULT_MISS)   ] = 0x0081, /* L1I.MISSES                 */
 511         },
 512         [ C(OP_WRITE) ] = {
 513                 [ C(RESULT_ACCESS) ] = -1,
 514                 [ C(RESULT_MISS)   ] = -1,
 515         },
 516         [ C(OP_PREFETCH) ] = {
 517                 [ C(RESULT_ACCESS) ] = 0,
 518                 [ C(RESULT_MISS)   ] = 0,
 519         },
 520  },
 521  [ C(LL  ) ] = {
 522         [ C(OP_READ) ] = {
 523                 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
 524                 [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
 525         },
 526         [ C(OP_WRITE) ] = {
 527                 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
 528                 [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
 529         },
 530         [ C(OP_PREFETCH) ] = {
 531                 [ C(RESULT_ACCESS) ] = 0,
 532                 [ C(RESULT_MISS)   ] = 0,
 533         },
 534  },
 535  [ C(DTLB) ] = {
 536         [ C(OP_READ) ] = {
 537                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI  (alias) */
 538                 [ C(RESULT_MISS)   ] = 0x0208, /* DTLB_MISSES.MISS_LD        */
 539         },
 540         [ C(OP_WRITE) ] = {
 541                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI  (alias) */
 542                 [ C(RESULT_MISS)   ] = 0x0808, /* DTLB_MISSES.MISS_ST        */
 543         },
 544         [ C(OP_PREFETCH) ] = {
 545                 [ C(RESULT_ACCESS) ] = 0,
 546                 [ C(RESULT_MISS)   ] = 0,
 547         },
 548  },
 549  [ C(ITLB) ] = {
 550         [ C(OP_READ) ] = {
 551                 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
 552                 [ C(RESULT_MISS)   ] = 0x1282, /* ITLBMISSES                 */
 553         },
 554         [ C(OP_WRITE) ] = {
 555                 [ C(RESULT_ACCESS) ] = -1,
 556                 [ C(RESULT_MISS)   ] = -1,
 557         },
 558         [ C(OP_PREFETCH) ] = {
 559                 [ C(RESULT_ACCESS) ] = -1,
 560                 [ C(RESULT_MISS)   ] = -1,
 561         },
 562  },
 563  [ C(BPU ) ] = {
 564         [ C(OP_READ) ] = {
 565                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
 566                 [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
 567         },
 568         [ C(OP_WRITE) ] = {
 569                 [ C(RESULT_ACCESS) ] = -1,
 570                 [ C(RESULT_MISS)   ] = -1,
 571         },
 572         [ C(OP_PREFETCH) ] = {
 573                 [ C(RESULT_ACCESS) ] = -1,
 574                 [ C(RESULT_MISS)   ] = -1,
 575         },
 576  },
 577 };
 578
 579 static __initconst const u64 atom_hw_cache_event_ids
 580                                 [PERF_COUNT_HW_CACHE_MAX]
 581                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 582                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 583 {
 584  [ C(L1D) ] = {
 585         [ C(OP_READ) ] = {
 586                 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD               */
 587                 [ C(RESULT_MISS)   ] = 0,
 588         },
 589         [ C(OP_WRITE) ] = {
 590                 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST               */
 591                 [ C(RESULT_MISS)   ] = 0,
 592         },
 593         [ C(OP_PREFETCH) ] = {
 594                 [ C(RESULT_ACCESS) ] = 0x0,
 595                 [ C(RESULT_MISS)   ] = 0,
 596         },
 597  },
 598  [ C(L1I ) ] = {
 599         [ C(OP_READ) ] = {
 600                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                  */
 601                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                 */
 602         },
 603         [ C(OP_WRITE) ] = {
 604                 [ C(RESULT_ACCESS) ] = -1,
 605                 [ C(RESULT_MISS)   ] = -1,
 606         },
 607         [ C(OP_PREFETCH) ] = {
 608                 [ C(RESULT_ACCESS) ] = 0,
 609                 [ C(RESULT_MISS)   ] = 0,
 610         },
 611  },
 612  [ C(LL  ) ] = {
 613         [ C(OP_READ) ] = {
 614                 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
 615                 [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
 616         },
 617         [ C(OP_WRITE) ] = {
 618                 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
 619                 [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
 620         },
 621         [ C(OP_PREFETCH) ] = {
 622                 [ C(RESULT_ACCESS) ] = 0,
 623                 [ C(RESULT_MISS)   ] = 0,
 624         },
 625  },
 626  [ C(DTLB) ] = {
 627         [ C(OP_READ) ] = {
 628                 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI  (alias) */
 629                 [ C(RESULT_MISS)   ] = 0x0508, /* DTLB_MISSES.MISS_LD        */
 630         },
 631         [ C(OP_WRITE) ] = {
 632                 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI  (alias) */
 633                 [ C(RESULT_MISS)   ] = 0x0608, /* DTLB_MISSES.MISS_ST        */
 634         },
 635         [ C(OP_PREFETCH) ] = {
 636                 [ C(RESULT_ACCESS) ] = 0,
 637                 [ C(RESULT_MISS)   ] = 0,
 638         },
 639  },
 640  [ C(ITLB) ] = {
 641         [ C(OP_READ) ] = {
 642                 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
 643                 [ C(RESULT_MISS)   ] = 0x0282, /* ITLB.MISSES                */
 644         },
 645         [ C(OP_WRITE) ] = {
 646                 [ C(RESULT_ACCESS) ] = -1,
 647                 [ C(RESULT_MISS)   ] = -1,
 648         },
 649         [ C(OP_PREFETCH) ] = {
 650                 [ C(RESULT_ACCESS) ] = -1,
 651                 [ C(RESULT_MISS)   ] = -1,
 652         },
 653  },
 654  [ C(BPU ) ] = {
 655         [ C(OP_READ) ] = {
 656                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
 657                 [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
 658         },
 659         [ C(OP_WRITE) ] = {
 660                 [ C(RESULT_ACCESS) ] = -1,
 661                 [ C(RESULT_MISS)   ] = -1,
 662         },
 663         [ C(OP_PREFETCH) ] = {
 664                 [ C(RESULT_ACCESS) ] = -1,
 665                 [ C(RESULT_MISS)   ] = -1,
 666         },
 667  },
 668 };
 669
 670 static void intel_pmu_disable_all(void)
 671 {
 672         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 673
 674         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
 675
 676         if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask))
 677                 intel_pmu_disable_bts();
 678
 679         intel_pmu_pebs_disable_all();
 680         intel_pmu_lbr_disable_all();
 681 }
 682
 683 static void intel_pmu_enable_all(int added)
 684 {
 685         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 686
 687         intel_pmu_pebs_enable_all();
 688         intel_pmu_lbr_enable_all();
 689         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
 690
 691         if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
 692                 struct perf_event *event =
 693                         cpuc->events[X86_PMC_IDX_FIXED_BTS];
 694
 695                 if (WARN_ON_ONCE(!event))
 696                         return;
 697
 698                 intel_pmu_enable_bts(event->hw.config);
 699         }
 700 }
 701
 702 /*
 703  * Workaround for:
 704  *   Intel Errata AAK100 (model 26)
 705  *   Intel Errata AAP53  (model 30)
 706  *   Intel Errata BD53   (model 44)
 707  *
 708  * The official story:
 709  *   These chips need to be 'reset' when adding counters by programming the
 710  *   magic three (non-counting) events 0x4300B5, 0x4300D2, and 0x4300B1 either
 711  *   in sequence on the same PMC or on different PMCs.
 712  *
 713  * In practise it appears some of these events do in fact count, and
 714  * we need to programm all 4 events.
 715  */
 716 static void intel_pmu_nhm_workaround(void)
 717 {
 718         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 719         static const unsigned long nhm_magic[4] = {
 720                 0x4300B5,
 721                 0x4300D2,
 722                 0x4300B1,
 723                 0x4300B1
 724         };
 725         struct perf_event *event;
 726         int i;
 727
 728         /*
 729          * The Errata requires below steps:
 730          * 1) Clear MSR_IA32_PEBS_ENABLE and MSR_CORE_PERF_GLOBAL_CTRL;
 731          * 2) Configure 4 PERFEVTSELx with the magic events and clear
 732          *    the corresponding PMCx;
 733          * 3) set bit0~bit3 of MSR_CORE_PERF_GLOBAL_CTRL;
 734          * 4) Clear MSR_CORE_PERF_GLOBAL_CTRL;
 735          * 5) Clear 4 pairs of ERFEVTSELx and PMCx;
 736          */
 737
 738         /*
 739          * The real steps we choose are a little different from above.
 740          * A) To reduce MSR operations, we don't run step 1) as they
 741          *    are already cleared before this function is called;
 742          * B) Call x86_perf_event_update to save PMCx before configuring
 743          *    PERFEVTSELx with magic number;
 744          * C) With step 5), we do clear only when the PERFEVTSELx is
 745          *    not used currently.
 746          * D) Call x86_perf_event_set_period to restore PMCx;
 747          */
 748
 749         /* We always operate 4 pairs of PERF Counters */
 750         for (i = 0; i < 4; i++) {
 751                 event = cpuc->events[i];
 752                 if (event)
 753                         x86_perf_event_update(event);
 754         }
 755
 756         for (i = 0; i < 4; i++) {
 757                 wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + i, nhm_magic[i]);
 758                 wrmsrl(MSR_ARCH_PERFMON_PERFCTR0 + i, 0x0);
 759         }
 760
 761         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0xf);
 762         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0x0);
 763
 764         for (i = 0; i < 4; i++) {
 765                 event = cpuc->events[i];
 766
 767                 if (event) {
 768                         x86_perf_event_set_period(event);
 769                         __x86_pmu_enable_event(&event->hw,
 770                                         ARCH_PERFMON_EVENTSEL_ENABLE);
 771                 } else
 772                         wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + i, 0x0);
 773         }
 774 }
 775
 776 static void intel_pmu_nhm_enable_all(int added)
 777 {
 778         if (added)
 779                 intel_pmu_nhm_workaround();
 780         intel_pmu_enable_all(added);
 781 }
 782
 783 static inline u64 intel_pmu_get_status(void)
 784 {
 785         u64 status;
 786
 787         rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
 788
 789         return status;
 790 }
 791
 792 static inline void intel_pmu_ack_status(u64 ack)
 793 {
 794         wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
 795 }
 796
 797 static void intel_pmu_disable_fixed(struct hw_perf_event *hwc)
 798 {
 799         int idx = hwc->idx - X86_PMC_IDX_FIXED;
 800         u64 ctrl_val, mask;
 801
 802         mask = 0xfULL << (idx * 4);
 803
 804         rdmsrl(hwc->config_base, ctrl_val);
 805         ctrl_val &= ~mask;
 806         wrmsrl(hwc->config_base, ctrl_val);
 807 }
 808
 809 static void intel_pmu_disable_event(struct perf_event *event)
 810 {
 811         struct hw_perf_event *hwc = &event->hw;
 812
 813         if (unlikely(hwc->idx == X86_PMC_IDX_FIXED_BTS)) {
 814                 intel_pmu_disable_bts();
 815                 intel_pmu_drain_bts_buffer();
 816                 return;
 817         }
 818
 819         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
 820                 intel_pmu_disable_fixed(hwc);
 821                 return;
 822         }
 823
 824         x86_pmu_disable_event(event);
 825
 826         if (unlikely(event->attr.precise_ip))
 827                 intel_pmu_pebs_disable(event);
 828 }
 829
 830 static void intel_pmu_enable_fixed(struct hw_perf_event *hwc)
 831 {
 832         int idx = hwc->idx - X86_PMC_IDX_FIXED;
 833         u64 ctrl_val, bits, mask;
 834
 835         /*
 836          * Enable IRQ generation (0x8),
 837          * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
 838          * if requested:
 839          */
 840         bits = 0x8ULL;
 841         if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
 842                 bits |= 0x2;
 843         if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
 844                 bits |= 0x1;
 845
 846         /*
 847          * ANY bit is supported in v3 and up
 848          */
 849         if (x86_pmu.version > 2 && hwc->config & ARCH_PERFMON_EVENTSEL_ANY)
 850                 bits |= 0x4;
 851
 852         bits <<= (idx * 4);
 853         mask = 0xfULL << (idx * 4);
 854
 855         rdmsrl(hwc->config_base, ctrl_val);
 856         ctrl_val &= ~mask;
 857         ctrl_val |= bits;
 858         wrmsrl(hwc->config_base, ctrl_val);
 859 }
 860
 861 static void intel_pmu_enable_event(struct perf_event *event)
 862 {
 863         struct hw_perf_event *hwc = &event->hw;
 864
 865         if (unlikely(hwc->idx == X86_PMC_IDX_FIXED_BTS)) {
 866                 if (!__this_cpu_read(cpu_hw_events.enabled))
 867                         return;
 868
 869                 intel_pmu_enable_bts(hwc->config);
 870                 return;
 871         }
 872
 873         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
 874                 intel_pmu_enable_fixed(hwc);
 875                 return;
 876         }
 877
 878         if (unlikely(event->attr.precise_ip))
 879                 intel_pmu_pebs_enable(event);
 880
 881         __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
 882 }
 883
 884 /*
 885  * Save and restart an expired event. Called by NMI contexts,
 886  * so it has to be careful about preempting normal event ops:
 887  */
 888 static int intel_pmu_save_and_restart(struct perf_event *event)
 889 {
 890         x86_perf_event_update(event);
 891         return x86_perf_event_set_period(event);
 892 }
 893
 894 static void intel_pmu_reset(void)
 895 {
 896         struct debug_store *ds = __this_cpu_read(cpu_hw_events.ds);
 897         unsigned long flags;
 898         int idx;
 899
 900         if (!x86_pmu.num_counters)
 901                 return;
 902
 903         local_irq_save(flags);
 904
 905         printk("clearing PMU state on CPU#%d\n", smp_processor_id());
 906
 907         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 908                 checking_wrmsrl(x86_pmu_config_addr(idx), 0ull);
 909                 checking_wrmsrl(x86_pmu_event_addr(idx),  0ull);
 910         }
 911         for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++)
 912                 checking_wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
 913
 914         if (ds)
 915                 ds->bts_index = ds->bts_buffer_base;
 916
 917         local_irq_restore(flags);
 918 }
 919
 920 /*
 921  * This handler is triggered by the local APIC, so the APIC IRQ handling
 922  * rules apply:
 923  */
 924 static int intel_pmu_handle_irq(struct pt_regs *regs)
 925 {
 926         struct perf_sample_data data;
 927         struct cpu_hw_events *cpuc;
 928         int bit, loops;
 929         u64 status;
 930         int handled;
 931
 932         perf_sample_data_init(&data, 0);
 933
 934         cpuc = &__get_cpu_var(cpu_hw_events);
 935
 936         intel_pmu_disable_all();
 937         handled = intel_pmu_drain_bts_buffer();
 938         status = intel_pmu_get_status();
 939         if (!status) {
 940                 intel_pmu_enable_all(0);
 941                 return handled;
 942         }
 943
 944         loops = 0;
 945 again:
 946         intel_pmu_ack_status(status);
 947         if (++loops > 100) {
 948                 WARN_ONCE(1, "perfevents: irq loop stuck!\n");
 949                 perf_event_print_debug();
 950                 intel_pmu_reset();
 951                 goto done;
 952         }
 953
 954         inc_irq_stat(apic_perf_irqs);
 955
 956         intel_pmu_lbr_read();
 957
 958         /*
 959          * PEBS overflow sets bit 62 in the global status register
 960          */
 961         if (__test_and_clear_bit(62, (unsigned long *)&status)) {
 962                 handled++;
 963                 x86_pmu.drain_pebs(regs);
 964         }
 965
 966         for_each_set_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
 967                 struct perf_event *event = cpuc->events[bit];
 968
 969                 handled++;
 970
 971                 if (!test_bit(bit, cpuc->active_mask))
 972                         continue;
 973
 974                 if (!intel_pmu_save_and_restart(event))
 975                         continue;
 976
 977                 data.period = event->hw.last_period;
 978
 979                 if (perf_event_overflow(event, 1, &data, regs))
 980                         x86_pmu_stop(event, 0);
 981         }
 982
 983         /*
 984          * Repeat if there is more work to be done:
 985          */
 986         status = intel_pmu_get_status();
 987         if (status)
 988                 goto again;
 989
 990 done:
 991         intel_pmu_enable_all(0);
 992         return handled;
 993 }
 994
 995 static struct event_constraint *
 996 intel_bts_constraints(struct perf_event *event)
 997 {
 998         struct hw_perf_event *hwc = &event->hw;
 999         unsigned int hw_event, bts_event;
1000
1001         hw_event = hwc->config & INTEL_ARCH_EVENT_MASK;
1002         bts_event = x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS);
1003
1004         if (unlikely(hw_event == bts_event && hwc->sample_period == 1))
1005                 return &bts_constraint;
1006
1007         return NULL;
1008 }
1009
1010 static struct event_constraint *
1011 intel_percore_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
1012 {
1013         struct hw_perf_event *hwc = &event->hw;
1014         unsigned int e = hwc->config & ARCH_PERFMON_EVENTSEL_EVENT;
1015         struct event_constraint *c;
1016         struct intel_percore *pc;
1017         struct er_account *era;
1018         int i;
1019         int free_slot;
1020         int found;
1021
1022         if (!x86_pmu.percore_constraints || hwc->extra_alloc)
1023                 return NULL;
1024
1025         for (c = x86_pmu.percore_constraints; c->cmask; c++) {
1026                 if (e != c->code)
1027                         continue;
1028
1029                 /*
1030                  * Allocate resource per core.
1031                  */
1032                 pc = cpuc->per_core;
1033                 if (!pc)
1034                         break;
1035                 c = &emptyconstraint;
1036                 raw_spin_lock(&pc->lock);
1037                 free_slot = -1;
1038                 found = 0;
1039                 for (i = 0; i < MAX_EXTRA_REGS; i++) {
1040                         era = &pc->regs[i];
1041                         if (era->ref > 0 && hwc->extra_reg == era->extra_reg) {
1042                                 /* Allow sharing same config */
1043                                 if (hwc->extra_config == era->extra_config) {
1044                                         era->ref++;
1045                                         cpuc->percore_used = 1;
1046                                         hwc->extra_alloc = 1;
1047                                         c = NULL;
1048                                 }
1049                                 /* else conflict */
1050                                 found = 1;
1051                                 break;
1052                         } else if (era->ref == 0 && free_slot == -1)
1053                                 free_slot = i;
1054                 }
1055                 if (!found && free_slot != -1) {
1056                         era = &pc->regs[free_slot];
1057                         era->ref = 1;
1058                         era->extra_reg = hwc->extra_reg;
1059                         era->extra_config = hwc->extra_config;
1060                         cpuc->percore_used = 1;
1061                         hwc->extra_alloc = 1;
1062                         c = NULL;
1063                 }
1064                 raw_spin_unlock(&pc->lock);
1065                 return c;
1066         }
1067
1068         return NULL;
1069 }
1070
1071 static struct event_constraint *
1072 intel_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
1073 {
1074         struct event_constraint *c;
1075
1076         c = intel_bts_constraints(event);
1077         if (c)
1078                 return c;
1079
1080         c = intel_pebs_constraints(event);
1081         if (c)
1082                 return c;
1083
1084         c = intel_percore_constraints(cpuc, event);
1085         if (c)
1086                 return c;
1087
1088         return x86_get_event_constraints(cpuc, event);
1089 }
1090
1091 static void intel_put_event_constraints(struct cpu_hw_events *cpuc,
1092                                         struct perf_event *event)
1093 {
1094         struct extra_reg *er;
1095         struct intel_percore *pc;
1096         struct er_account *era;
1097         struct hw_perf_event *hwc = &event->hw;
1098         int i, allref;
1099
1100         if (!cpuc->percore_used)
1101                 return;
1102
1103         for (er = x86_pmu.extra_regs; er->msr; er++) {
1104                 if (er->event != (hwc->config & er->config_mask))
1105                         continue;
1106
1107                 pc = cpuc->per_core;
1108                 raw_spin_lock(&pc->lock);
1109                 for (i = 0; i < MAX_EXTRA_REGS; i++) {
1110                         era = &pc->regs[i];
1111                         if (era->ref > 0 &&
1112                             era->extra_config == hwc->extra_config &&
1113                             era->extra_reg == er->msr) {
1114                                 era->ref--;
1115                                 hwc->extra_alloc = 0;
1116                                 break;
1117                         }
1118                 }
1119                 allref = 0;
1120                 for (i = 0; i < MAX_EXTRA_REGS; i++)
1121                         allref += pc->regs[i].ref;
1122                 if (allref == 0)
1123                         cpuc->percore_used = 0;
1124                 raw_spin_unlock(&pc->lock);
1125                 break;
1126         }
1127 }
1128
1129 static int intel_pmu_hw_config(struct perf_event *event)
1130 {
1131         int ret = x86_pmu_hw_config(event);
1132
1133         if (ret)
1134                 return ret;
1135
1136         if (event->attr.precise_ip &&
1137             (event->hw.config & X86_RAW_EVENT_MASK) == 0x003c) {
1138                 /*
1139                  * Use an alternative encoding for CPU_CLK_UNHALTED.THREAD_P
1140                  * (0x003c) so that we can use it with PEBS.
1141                  *
1142                  * The regular CPU_CLK_UNHALTED.THREAD_P event (0x003c) isn't
1143                  * PEBS capable. However we can use INST_RETIRED.ANY_P
1144                  * (0x00c0), which is a PEBS capable event, to get the same
1145                  * count.
1146                  *
1147                  * INST_RETIRED.ANY_P counts the number of cycles that retires
1148                  * CNTMASK instructions. By setting CNTMASK to a value (16)
1149                  * larger than the maximum number of instructions that can be
1150                  * retired per cycle (4) and then inverting the condition, we
1151                  * count all cycles that retire 16 or less instructions, which
1152                  * is every cycle.
1153                  *
1154                  * Thereby we gain a PEBS capable cycle counter.
1155                  */
1156                 u64 alt_config = 0x108000c0; /* INST_RETIRED.TOTAL_CYCLES */
1157
1158                 alt_config |= (event->hw.config & ~X86_RAW_EVENT_MASK);
1159                 event->hw.config = alt_config;
1160         }
1161
1162         if (event->attr.type != PERF_TYPE_RAW)
1163                 return 0;
1164
1165         if (!(event->attr.config & ARCH_PERFMON_EVENTSEL_ANY))
1166                 return 0;
1167
1168         if (x86_pmu.version < 3)
1169                 return -EINVAL;
1170
1171         if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
1172                 return -EACCES;
1173
1174         event->hw.config |= ARCH_PERFMON_EVENTSEL_ANY;
1175
1176         return 0;
1177 }
1178
1179 static __initconst const struct x86_pmu core_pmu = {
1180         .name                   = "core",
1181         .handle_irq             = x86_pmu_handle_irq,
1182         .disable_all            = x86_pmu_disable_all,
1183         .enable_all             = x86_pmu_enable_all,
1184         .enable                 = x86_pmu_enable_event,
1185         .disable                = x86_pmu_disable_event,
1186         .hw_config              = x86_pmu_hw_config,
1187         .schedule_events        = x86_schedule_events,
1188         .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
1189         .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
1190         .event_map              = intel_pmu_event_map,
1191         .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
1192         .apic                   = 1,
1193         /*
1194          * Intel PMCs cannot be accessed sanely above 32 bit width,
1195          * so we install an artificial 1<<31 period regardless of
1196          * the generic event period:
1197          */
1198         .max_period             = (1ULL << 31) - 1,
1199         .get_event_constraints  = intel_get_event_constraints,
1200         .put_event_constraints  = intel_put_event_constraints,
1201         .event_constraints      = intel_core_event_constraints,
1202 };
1203
1204 static int intel_pmu_cpu_prepare(int cpu)
1205 {
1206         struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
1207
1208         if (!cpu_has_ht_siblings())
1209                 return NOTIFY_OK;
1210
1211         cpuc->per_core = kzalloc_node(sizeof(struct intel_percore),
1212                                       GFP_KERNEL, cpu_to_node(cpu));
1213         if (!cpuc->per_core)
1214                 return NOTIFY_BAD;
1215
1216         raw_spin_lock_init(&cpuc->per_core->lock);
1217         cpuc->per_core->core_id = -1;
1218         return NOTIFY_OK;
1219 }
1220
1221 static void intel_pmu_cpu_starting(int cpu)
1222 {
1223         struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
1224         int core_id = topology_core_id(cpu);
1225         int i;
1226
1227         init_debug_store_on_cpu(cpu);
1228         /*
1229          * Deal with CPUs that don't clear their LBRs on power-up.
1230          */
1231         intel_pmu_lbr_reset();
1232
1233         if (!cpu_has_ht_siblings())
1234                 return;
1235
1236         for_each_cpu(i, topology_thread_cpumask(cpu)) {
1237                 struct intel_percore *pc = per_cpu(cpu_hw_events, i).per_core;
1238
1239                 if (pc && pc->core_id == core_id) {
1240                         kfree(cpuc->per_core);
1241                         cpuc->per_core = pc;
1242                         break;
1243                 }
1244         }
1245
1246         cpuc->per_core->core_id = core_id;
1247         cpuc->per_core->refcnt++;
1248 }
1249
1250 static void intel_pmu_cpu_dying(int cpu)
1251 {
1252         struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
1253         struct intel_percore *pc = cpuc->per_core;
1254
1255         if (pc) {
1256                 if (pc->core_id == -1 || --pc->refcnt == 0)
1257                         kfree(pc);
1258                 cpuc->per_core = NULL;
1259         }
1260
1261         fini_debug_store_on_cpu(cpu);
1262 }
1263
1264 static __initconst const struct x86_pmu intel_pmu = {
1265         .name                   = "Intel",
1266         .handle_irq             = intel_pmu_handle_irq,
1267         .disable_all            = intel_pmu_disable_all,
1268         .enable_all             = intel_pmu_enable_all,
1269         .enable                 = intel_pmu_enable_event,
1270         .disable                = intel_pmu_disable_event,
1271         .hw_config              = intel_pmu_hw_config,
1272         .schedule_events        = x86_schedule_events,
1273         .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
1274         .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
1275         .event_map              = intel_pmu_event_map,
1276         .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
1277         .apic                   = 1,
1278         /*
1279          * Intel PMCs cannot be accessed sanely above 32 bit width,
1280          * so we install an artificial 1<<31 period regardless of
1281          * the generic event period:
1282          */
1283         .max_period             = (1ULL << 31) - 1,
1284         .get_event_constraints  = intel_get_event_constraints,
1285         .put_event_constraints  = intel_put_event_constraints,
1286
1287         .cpu_prepare            = intel_pmu_cpu_prepare,
1288         .cpu_starting           = intel_pmu_cpu_starting,
1289         .cpu_dying              = intel_pmu_cpu_dying,
1290 };
1291
1292 static void intel_clovertown_quirks(void)
1293 {
1294         /*
1295          * PEBS is unreliable due to:
1296          *
1297          *   AJ67  - PEBS may experience CPL leaks
1298          *   AJ68  - PEBS PMI may be delayed by one event
1299          *   AJ69  - GLOBAL_STATUS[62] will only be set when DEBUGCTL[12]
1300          *   AJ106 - FREEZE_LBRS_ON_PMI doesn't work in combination with PEBS
1301          *
1302          * AJ67 could be worked around by restricting the OS/USR flags.
1303          * AJ69 could be worked around by setting PMU_FREEZE_ON_PMI.
1304          *
1305          * AJ106 could possibly be worked around by not allowing LBR
1306          *       usage from PEBS, including the fixup.
1307          * AJ68  could possibly be worked around by always programming
1308          *       a pebs_event_reset[0] value and coping with the lost events.
1309          *
1310          * But taken together it might just make sense to not enable PEBS on
1311          * these chips.
1312          */
1313         printk(KERN_WARNING "PEBS disabled due to CPU errata.\n");
1314         x86_pmu.pebs = 0;
1315         x86_pmu.pebs_constraints = NULL;
1316 }
1317
1318 static __init int intel_pmu_init(void)
1319 {
1320         union cpuid10_edx edx;
1321         union cpuid10_eax eax;
1322         unsigned int unused;
1323         unsigned int ebx;
1324         int version;
1325
1326         if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
1327                 switch (boot_cpu_data.x86) {
1328                 case 0x6:
1329                         return p6_pmu_init();
1330                 case 0xf:
1331                         return p4_pmu_init();
1332                 }
1333                 return -ENODEV;
1334         }
1335
1336         /*
1337          * Check whether the Architectural PerfMon supports
1338          * Branch Misses Retired hw_event or not.
1339          */
1340         cpuid(10, &eax.full, &ebx, &unused, &edx.full);
1341         if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
1342                 return -ENODEV;
1343
1344         version = eax.split.version_id;
1345         if (version < 2)
1346                 x86_pmu = core_pmu;
1347         else
1348                 x86_pmu = intel_pmu;
1349
1350         x86_pmu.version                 = version;
1351         x86_pmu.num_counters            = eax.split.num_counters;
1352         x86_pmu.cntval_bits             = eax.split.bit_width;
1353         x86_pmu.cntval_mask             = (1ULL << eax.split.bit_width) - 1;
1354
1355         /*
1356          * Quirk: v2 perfmon does not report fixed-purpose events, so
1357          * assume at least 3 events:
1358          */
1359         if (version > 1)
1360                 x86_pmu.num_counters_fixed = max((int)edx.split.num_counters_fixed, 3);
1361
1362         /*
1363          * v2 and above have a perf capabilities MSR
1364          */
1365         if (version > 1) {
1366                 u64 capabilities;
1367
1368                 rdmsrl(MSR_IA32_PERF_CAPABILITIES, capabilities);
1369                 x86_pmu.intel_cap.capabilities = capabilities;
1370         }
1371
1372         intel_ds_init();
1373
1374         /*
1375          * Install the hw-cache-events table:
1376          */
1377         switch (boot_cpu_data.x86_model) {
1378         case 14: /* 65 nm core solo/duo, "Yonah" */
1379                 pr_cont("Core events, ");
1380                 break;
1381
1382         case 15: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
1383                 x86_pmu.quirks = intel_clovertown_quirks;
1384         case 22: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
1385         case 23: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
1386         case 29: /* six-core 45 nm xeon "Dunnington" */
1387                 memcpy(hw_cache_event_ids, core2_hw_cache_event_ids,
1388                        sizeof(hw_cache_event_ids));
1389
1390                 intel_pmu_lbr_init_core();
1391
1392                 x86_pmu.event_constraints = intel_core2_event_constraints;
1393                 x86_pmu.pebs_constraints = intel_core2_pebs_event_constraints;
1394                 pr_cont("Core2 events, ");
1395                 break;
1396
1397         case 26: /* 45 nm nehalem, "Bloomfield" */
1398         case 30: /* 45 nm nehalem, "Lynnfield" */
1399         case 46: /* 45 nm nehalem-ex, "Beckton" */
1400                 memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
1401                        sizeof(hw_cache_event_ids));
1402                 memcpy(hw_cache_extra_regs, nehalem_hw_cache_extra_regs,
1403                        sizeof(hw_cache_extra_regs));
1404
1405                 intel_pmu_lbr_init_nhm();
1406
1407                 x86_pmu.event_constraints = intel_nehalem_event_constraints;
1408                 x86_pmu.pebs_constraints = intel_nehalem_pebs_event_constraints;
1409                 x86_pmu.percore_constraints = intel_nehalem_percore_constraints;
1410                 x86_pmu.enable_all = intel_pmu_nhm_enable_all;
1411                 x86_pmu.extra_regs = intel_nehalem_extra_regs;
1412                 pr_cont("Nehalem events, ");
1413                 break;
1414
1415         case 28: /* Atom */
1416                 memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
1417                        sizeof(hw_cache_event_ids));
1418
1419                 intel_pmu_lbr_init_atom();
1420
1421                 x86_pmu.event_constraints = intel_gen_event_constraints;
1422                 x86_pmu.pebs_constraints = intel_atom_pebs_event_constraints;
1423                 pr_cont("Atom events, ");
1424                 break;
1425
1426         case 37: /* 32 nm nehalem, "Clarkdale" */
1427         case 44: /* 32 nm nehalem, "Gulftown" */
1428                 memcpy(hw_cache_event_ids, westmere_hw_cache_event_ids,
1429                        sizeof(hw_cache_event_ids));
1430                 memcpy(hw_cache_extra_regs, nehalem_hw_cache_extra_regs,
1431                        sizeof(hw_cache_extra_regs));
1432
1433                 intel_pmu_lbr_init_nhm();
1434
1435                 x86_pmu.event_constraints = intel_westmere_event_constraints;
1436                 x86_pmu.percore_constraints = intel_westmere_percore_constraints;
1437                 x86_pmu.enable_all = intel_pmu_nhm_enable_all;
1438                 x86_pmu.pebs_constraints = intel_westmere_pebs_event_constraints;
1439                 x86_pmu.extra_regs = intel_westmere_extra_regs;
1440                 pr_cont("Westmere events, ");
1441                 break;
1442
1443         case 42: /* SandyBridge */
1444                 memcpy(hw_cache_event_ids, snb_hw_cache_event_ids,
1445                        sizeof(hw_cache_event_ids));
1446
1447                 intel_pmu_lbr_init_nhm();
1448
1449                 x86_pmu.event_constraints = intel_snb_event_constraints;
1450                 x86_pmu.pebs_constraints = intel_snb_pebs_events;
1451                 pr_cont("SandyBridge events, ");
1452                 break;
1453
1454         default:
1455                 /*
1456                  * default constraints for v2 and up
1457                  */
1458                 x86_pmu.event_constraints = intel_gen_event_constraints;
1459                 pr_cont("generic architected perfmon, ");
1460         }
1461         return 0;
1462 }
1463
1464 #else /* CONFIG_CPU_SUP_INTEL */
1465
1466 static int intel_pmu_init(void)
1467 {
1468         return 0;
1469 }
1470
1471 #endif /* CONFIG_CPU_SUP_INTEL */