3 * athlon / K7 / K8 / Family 10h model-specific MSR operations
5 * @remark Copyright 2002-2009 OProfile authors
6 * @remark Read the file COPYING
9 * @author Philippe Elie
10 * @author Graydon Hoare
11 * @author Robert Richter <robert.richter@amd.com>
12 * @author Barry Kasindorf
15 #include <linux/oprofile.h>
16 #include <linux/device.h>
17 #include <linux/pci.h>
19 #include <asm/ptrace.h>
23 #include "op_x86_model.h"
24 #include "op_counter.h"
26 #define NUM_COUNTERS 4
27 #define NUM_CONTROLS 4
28 #define OP_EVENT_MASK 0x0FFF
29 #define OP_CTR_OVERFLOW (1ULL<<31)
31 #define MSR_AMD_EVENTSEL_RESERVED ((0xFFFFFCF0ULL<<32)|(1ULL<<21))
33 static unsigned long reset_value[NUM_COUNTERS];
35 #ifdef CONFIG_OPROFILE_IBS
37 /* IbsFetchCtl bits/masks */
38 #define IBS_FETCH_RAND_EN (1ULL<<57)
39 #define IBS_FETCH_VAL (1ULL<<49)
40 #define IBS_FETCH_ENABLE (1ULL<<48)
41 #define IBS_FETCH_CNT_MASK 0xFFFF0000ULL
44 #define IBS_OP_CNT_CTL (1ULL<<19)
45 #define IBS_OP_VAL (1ULL<<18)
46 #define IBS_OP_ENABLE (1ULL<<17)
48 #define IBS_FETCH_SIZE 6
49 #define IBS_OP_SIZE 12
51 static int has_ibs; /* AMD Family10h and later */
53 struct op_ibs_config {
54 unsigned long op_enabled;
55 unsigned long fetch_enabled;
56 unsigned long max_cnt_fetch;
57 unsigned long max_cnt_op;
58 unsigned long rand_en;
59 unsigned long dispatched_ops;
62 static struct op_ibs_config ibs_config;
66 /* functions for op_amd_spec */
68 static void op_amd_fill_in_addresses(struct op_msrs * const msrs)
72 for (i = 0; i < NUM_COUNTERS; i++) {
73 if (reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))
74 msrs->counters[i].addr = MSR_K7_PERFCTR0 + i;
76 msrs->counters[i].addr = 0;
79 for (i = 0; i < NUM_CONTROLS; i++) {
80 if (reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i))
81 msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i;
83 msrs->controls[i].addr = 0;
87 static void op_amd_setup_ctrs(struct op_x86_model_spec const *model,
88 struct op_msrs const * const msrs)
93 /* clear all counters */
94 for (i = 0 ; i < NUM_CONTROLS; ++i) {
95 if (unlikely(!msrs->controls[i].addr))
97 rdmsrl(msrs->controls[i].addr, val);
98 val &= model->reserved;
99 wrmsrl(msrs->controls[i].addr, val);
102 /* avoid a false detection of ctr overflows in NMI handler */
103 for (i = 0; i < NUM_COUNTERS; ++i) {
104 if (unlikely(!msrs->counters[i].addr))
106 wrmsrl(msrs->counters[i].addr, -1LL);
109 /* enable active counters */
110 for (i = 0; i < NUM_COUNTERS; ++i) {
111 if (counter_config[i].enabled && msrs->counters[i].addr) {
112 reset_value[i] = counter_config[i].count;
113 wrmsrl(msrs->counters[i].addr,
114 -(s64)counter_config[i].count);
115 rdmsrl(msrs->controls[i].addr, val);
116 val &= model->reserved;
117 val |= op_x86_get_ctrl(model, &counter_config[i]);
118 wrmsrl(msrs->controls[i].addr, val);
125 #ifdef CONFIG_OPROFILE_IBS
128 op_amd_handle_ibs(struct pt_regs * const regs,
129 struct op_msrs const * const msrs)
132 struct op_entry entry;
137 if (ibs_config.fetch_enabled) {
138 rdmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
139 if (ctl & IBS_FETCH_VAL) {
140 rdmsrl(MSR_AMD64_IBSFETCHLINAD, val);
141 oprofile_write_reserve(&entry, regs, val,
142 IBS_FETCH_CODE, IBS_FETCH_SIZE);
143 oprofile_add_data64(&entry, val);
144 oprofile_add_data64(&entry, ctl);
145 rdmsrl(MSR_AMD64_IBSFETCHPHYSAD, val);
146 oprofile_add_data64(&entry, val);
147 oprofile_write_commit(&entry);
149 /* reenable the IRQ */
150 ctl &= ~(IBS_FETCH_VAL | IBS_FETCH_CNT_MASK);
151 ctl |= IBS_FETCH_ENABLE;
152 wrmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
156 if (ibs_config.op_enabled) {
157 rdmsrl(MSR_AMD64_IBSOPCTL, ctl);
158 if (ctl & IBS_OP_VAL) {
159 rdmsrl(MSR_AMD64_IBSOPRIP, val);
160 oprofile_write_reserve(&entry, regs, val,
161 IBS_OP_CODE, IBS_OP_SIZE);
162 oprofile_add_data64(&entry, val);
163 rdmsrl(MSR_AMD64_IBSOPDATA, val);
164 oprofile_add_data64(&entry, val);
165 rdmsrl(MSR_AMD64_IBSOPDATA2, val);
166 oprofile_add_data64(&entry, val);
167 rdmsrl(MSR_AMD64_IBSOPDATA3, val);
168 oprofile_add_data64(&entry, val);
169 rdmsrl(MSR_AMD64_IBSDCLINAD, val);
170 oprofile_add_data64(&entry, val);
171 rdmsrl(MSR_AMD64_IBSDCPHYSAD, val);
172 oprofile_add_data64(&entry, val);
173 oprofile_write_commit(&entry);
175 /* reenable the IRQ */
176 ctl &= ~IBS_OP_VAL & 0xFFFFFFFF;
177 ctl |= IBS_OP_ENABLE;
178 wrmsrl(MSR_AMD64_IBSOPCTL, ctl);
185 static inline void op_amd_start_ibs(void)
188 if (has_ibs && ibs_config.fetch_enabled) {
189 val = (ibs_config.max_cnt_fetch >> 4) & 0xFFFF;
190 val |= ibs_config.rand_en ? IBS_FETCH_RAND_EN : 0;
191 val |= IBS_FETCH_ENABLE;
192 wrmsrl(MSR_AMD64_IBSFETCHCTL, val);
195 if (has_ibs && ibs_config.op_enabled) {
196 val = (ibs_config.max_cnt_op >> 4) & 0xFFFF;
197 val |= ibs_config.dispatched_ops ? IBS_OP_CNT_CTL : 0;
198 val |= IBS_OP_ENABLE;
199 wrmsrl(MSR_AMD64_IBSOPCTL, val);
203 static void op_amd_stop_ibs(void)
205 if (has_ibs && ibs_config.fetch_enabled)
206 /* clear max count and enable */
207 wrmsrl(MSR_AMD64_IBSFETCHCTL, 0);
209 if (has_ibs && ibs_config.op_enabled)
210 /* clear max count and enable */
211 wrmsrl(MSR_AMD64_IBSOPCTL, 0);
216 static inline int op_amd_handle_ibs(struct pt_regs * const regs,
217 struct op_msrs const * const msrs) { }
218 static inline void op_amd_start_ibs(void) { }
219 static inline void op_amd_stop_ibs(void) { }
223 static int op_amd_check_ctrs(struct pt_regs * const regs,
224 struct op_msrs const * const msrs)
229 for (i = 0 ; i < NUM_COUNTERS; ++i) {
232 rdmsrl(msrs->counters[i].addr, val);
233 /* bit is clear if overflowed: */
234 if (val & OP_CTR_OVERFLOW)
236 oprofile_add_sample(regs, i);
237 wrmsrl(msrs->counters[i].addr, -(s64)reset_value[i]);
240 op_amd_handle_ibs(regs, msrs);
242 /* See op_model_ppro.c */
246 static void op_amd_start(struct op_msrs const * const msrs)
250 for (i = 0 ; i < NUM_COUNTERS ; ++i) {
251 if (reset_value[i]) {
252 rdmsrl(msrs->controls[i].addr, val);
253 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
254 wrmsrl(msrs->controls[i].addr, val);
261 static void op_amd_stop(struct op_msrs const * const msrs)
267 * Subtle: stop on all counters to avoid race with setting our
270 for (i = 0 ; i < NUM_COUNTERS ; ++i) {
273 rdmsrl(msrs->controls[i].addr, val);
274 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
275 wrmsrl(msrs->controls[i].addr, val);
281 static void op_amd_shutdown(struct op_msrs const * const msrs)
285 for (i = 0 ; i < NUM_COUNTERS ; ++i) {
286 if (msrs->counters[i].addr)
287 release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
289 for (i = 0 ; i < NUM_CONTROLS ; ++i) {
290 if (msrs->controls[i].addr)
291 release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
295 #ifdef CONFIG_OPROFILE_IBS
297 static u8 ibs_eilvt_off;
299 static inline void apic_init_ibs_nmi_per_cpu(void *arg)
301 ibs_eilvt_off = setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_NMI, 0);
304 static inline void apic_clear_ibs_nmi_per_cpu(void *arg)
306 setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_FIX, 1);
309 static int init_ibs_nmi(void)
311 #define IBSCTL_LVTOFFSETVAL (1 << 8)
313 struct pci_dev *cpu_cfg;
318 on_each_cpu(apic_init_ibs_nmi_per_cpu, NULL, 1);
323 cpu_cfg = pci_get_device(PCI_VENDOR_ID_AMD,
324 PCI_DEVICE_ID_AMD_10H_NB_MISC,
329 pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off
330 | IBSCTL_LVTOFFSETVAL);
331 pci_read_config_dword(cpu_cfg, IBSCTL, &value);
332 if (value != (ibs_eilvt_off | IBSCTL_LVTOFFSETVAL)) {
333 pci_dev_put(cpu_cfg);
334 printk(KERN_DEBUG "Failed to setup IBS LVT offset, "
335 "IBSCTL = 0x%08x", value);
341 printk(KERN_DEBUG "No CPU node configured for IBS");
347 /* Works only for 64bit with proper numa implementation. */
348 if (nodes != num_possible_nodes()) {
349 printk(KERN_DEBUG "Failed to setup CPU node(s) for IBS, "
350 "found: %d, expected %d",
351 nodes, num_possible_nodes());
358 /* uninitialize the APIC for the IBS interrupts if needed */
359 static void clear_ibs_nmi(void)
362 on_each_cpu(apic_clear_ibs_nmi_per_cpu, NULL, 1);
365 /* initialize the APIC for the IBS interrupts if available */
366 static void ibs_init(void)
368 has_ibs = boot_cpu_has(X86_FEATURE_IBS);
373 if (init_ibs_nmi()) {
378 printk(KERN_INFO "oprofile: AMD IBS detected\n");
381 static void ibs_exit(void)
389 static int (*create_arch_files)(struct super_block *sb, struct dentry *root);
391 static int setup_ibs_files(struct super_block *sb, struct dentry *root)
396 /* architecture specific files */
397 if (create_arch_files)
398 ret = create_arch_files(sb, root);
406 /* model specific files */
408 /* setup some reasonable defaults */
409 ibs_config.max_cnt_fetch = 250000;
410 ibs_config.fetch_enabled = 0;
411 ibs_config.max_cnt_op = 250000;
412 ibs_config.op_enabled = 0;
413 ibs_config.dispatched_ops = 1;
415 dir = oprofilefs_mkdir(sb, root, "ibs_fetch");
416 oprofilefs_create_ulong(sb, dir, "enable",
417 &ibs_config.fetch_enabled);
418 oprofilefs_create_ulong(sb, dir, "max_count",
419 &ibs_config.max_cnt_fetch);
420 oprofilefs_create_ulong(sb, dir, "rand_enable",
421 &ibs_config.rand_en);
423 dir = oprofilefs_mkdir(sb, root, "ibs_op");
424 oprofilefs_create_ulong(sb, dir, "enable",
425 &ibs_config.op_enabled);
426 oprofilefs_create_ulong(sb, dir, "max_count",
427 &ibs_config.max_cnt_op);
428 oprofilefs_create_ulong(sb, dir, "dispatched_ops",
429 &ibs_config.dispatched_ops);
434 static int op_amd_init(struct oprofile_operations *ops)
437 create_arch_files = ops->create_files;
438 ops->create_files = setup_ibs_files;
442 static void op_amd_exit(void)
451 static int op_amd_init(struct oprofile_operations *ops)
456 static void op_amd_exit(void) {}
458 #endif /* CONFIG_OPROFILE_IBS */
460 struct op_x86_model_spec const op_amd_spec = {
461 .num_counters = NUM_COUNTERS,
462 .num_controls = NUM_CONTROLS,
463 .reserved = MSR_AMD_EVENTSEL_RESERVED,
464 .event_mask = OP_EVENT_MASK,
467 .fill_in_addresses = &op_amd_fill_in_addresses,
468 .setup_ctrs = &op_amd_setup_ctrs,
469 .check_ctrs = &op_amd_check_ctrs,
470 .start = &op_amd_start,
471 .stop = &op_amd_stop,
472 .shutdown = &op_amd_shutdown,