arch/x86/kernel/cpu/perf_event_intel_lbr.c

   1 #include <linux/perf_event.h>
   2 #include <linux/types.h>
   3
   4 #include <asm/perf_event.h>
   5 #include <asm/msr.h>
   6
   7 #include "perf_event.h"
   8
   9 enum {
  10         LBR_FORMAT_32           = 0x00,
  11         LBR_FORMAT_LIP          = 0x01,
  12         LBR_FORMAT_EIP          = 0x02,
  13         LBR_FORMAT_EIP_FLAGS    = 0x03,
  14 };
  15
  16 /*
  17  * We only support LBR implementations that have FREEZE_LBRS_ON_PMI
  18  * otherwise it becomes near impossible to get a reliable stack.
  19  */
  20
  21 static void __intel_pmu_lbr_enable(void)
  22 {
  23         u64 debugctl;
  24
  25         rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  26         debugctl |= (DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
  27         wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  28 }
  29
  30 static void __intel_pmu_lbr_disable(void)
  31 {
  32         u64 debugctl;
  33
  34         rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  35         debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
  36         wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  37 }
  38
  39 static void intel_pmu_lbr_reset_32(void)
  40 {
  41         int i;
  42
  43         for (i = 0; i < x86_pmu.lbr_nr; i++)
  44                 wrmsrl(x86_pmu.lbr_from + i, 0);
  45 }
  46
  47 static void intel_pmu_lbr_reset_64(void)
  48 {
  49         int i;
  50
  51         for (i = 0; i < x86_pmu.lbr_nr; i++) {
  52                 wrmsrl(x86_pmu.lbr_from + i, 0);
  53                 wrmsrl(x86_pmu.lbr_to   + i, 0);
  54         }
  55 }
  56
  57 void intel_pmu_lbr_reset(void)
  58 {
  59         if (!x86_pmu.lbr_nr)
  60                 return;
  61
  62         if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
  63                 intel_pmu_lbr_reset_32();
  64         else
  65                 intel_pmu_lbr_reset_64();
  66 }
  67
  68 void intel_pmu_lbr_enable(struct perf_event *event)
  69 {
  70         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  71
  72         if (!x86_pmu.lbr_nr)
  73                 return;
  74
  75         WARN_ON_ONCE(cpuc->enabled);
  76
  77         /*
  78          * Reset the LBR stack if we changed task context to
  79          * avoid data leaks.
  80          */
  81
  82         if (event->ctx->task && cpuc->lbr_context != event->ctx) {
  83                 intel_pmu_lbr_reset();
  84                 cpuc->lbr_context = event->ctx;
  85         }
  86
  87         cpuc->lbr_users++;
  88 }
  89
  90 void intel_pmu_lbr_disable(struct perf_event *event)
  91 {
  92         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  93
  94         if (!x86_pmu.lbr_nr)
  95                 return;
  96
  97         cpuc->lbr_users--;
  98         WARN_ON_ONCE(cpuc->lbr_users < 0);
  99
 100         if (cpuc->enabled && !cpuc->lbr_users)
 101                 __intel_pmu_lbr_disable();
 102 }
 103
 104 void intel_pmu_lbr_enable_all(void)
 105 {
 106         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 107
 108         if (cpuc->lbr_users)
 109                 __intel_pmu_lbr_enable();
 110 }
 111
 112 void intel_pmu_lbr_disable_all(void)
 113 {
 114         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 115
 116         if (cpuc->lbr_users)
 117                 __intel_pmu_lbr_disable();
 118 }
 119
 120 static inline u64 intel_pmu_lbr_tos(void)
 121 {
 122         u64 tos;
 123
 124         rdmsrl(x86_pmu.lbr_tos, tos);
 125
 126         return tos;
 127 }
 128
 129 static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
 130 {
 131         unsigned long mask = x86_pmu.lbr_nr - 1;
 132         u64 tos = intel_pmu_lbr_tos();
 133         int i;
 134
 135         for (i = 0; i < x86_pmu.lbr_nr; i++) {
 136                 unsigned long lbr_idx = (tos - i) & mask;
 137                 union {
 138                         struct {
 139                                 u32 from;
 140                                 u32 to;
 141                         };
 142                         u64     lbr;
 143                 } msr_lastbranch;
 144
 145                 rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr);
 146
 147                 cpuc->lbr_entries[i].from  = msr_lastbranch.from;
 148                 cpuc->lbr_entries[i].to    = msr_lastbranch.to;
 149                 cpuc->lbr_entries[i].flags = 0;
 150         }
 151         cpuc->lbr_stack.nr = i;
 152 }
 153
 154 #define LBR_FROM_FLAG_MISPRED  (1ULL << 63)
 155
 156 /*
 157  * Due to lack of segmentation in Linux the effective address (offset)
 158  * is the same as the linear address, allowing us to merge the LIP and EIP
 159  * LBR formats.
 160  */
 161 static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
 162 {
 163         unsigned long mask = x86_pmu.lbr_nr - 1;
 164         int lbr_format = x86_pmu.intel_cap.lbr_format;
 165         u64 tos = intel_pmu_lbr_tos();
 166         int i;
 167
 168         for (i = 0; i < x86_pmu.lbr_nr; i++) {
 169                 unsigned long lbr_idx = (tos - i) & mask;
 170                 u64 from, to, flags = 0;
 171
 172                 rdmsrl(x86_pmu.lbr_from + lbr_idx, from);
 173                 rdmsrl(x86_pmu.lbr_to   + lbr_idx, to);
 174
 175                 if (lbr_format == LBR_FORMAT_EIP_FLAGS) {
 176                         flags = !!(from & LBR_FROM_FLAG_MISPRED);
 177                         from = (u64)((((s64)from) << 1) >> 1);
 178                 }
 179
 180                 cpuc->lbr_entries[i].from  = from;
 181                 cpuc->lbr_entries[i].to    = to;
 182                 cpuc->lbr_entries[i].flags = flags;
 183         }
 184         cpuc->lbr_stack.nr = i;
 185 }
 186
 187 void intel_pmu_lbr_read(void)
 188 {
 189         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 190
 191         if (!cpuc->lbr_users)
 192                 return;
 193
 194         if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
 195                 intel_pmu_lbr_read_32(cpuc);
 196         else
 197                 intel_pmu_lbr_read_64(cpuc);
 198 }
 199
 200 void intel_pmu_lbr_init_core(void)
 201 {
 202         x86_pmu.lbr_nr     = 4;
 203         x86_pmu.lbr_tos    = 0x01c9;
 204         x86_pmu.lbr_from   = 0x40;
 205         x86_pmu.lbr_to     = 0x60;
 206 }
 207
 208 void intel_pmu_lbr_init_nhm(void)
 209 {
 210         x86_pmu.lbr_nr     = 16;
 211         x86_pmu.lbr_tos    = 0x01c9;
 212         x86_pmu.lbr_from   = 0x680;
 213         x86_pmu.lbr_to     = 0x6c0;
 214 }
 215
 216 void intel_pmu_lbr_init_atom(void)
 217 {
 218         x86_pmu.lbr_nr     = 8;
 219         x86_pmu.lbr_tos    = 0x01c9;
 220         x86_pmu.lbr_from   = 0x40;
 221         x86_pmu.lbr_to     = 0x60;
 222 }