2 * Performance counter support - powerpc architecture code
4 * Copyright 2008-2009 Paul Mackerras, IBM Corporation.
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
11 #include <linux/kernel.h>
12 #include <linux/sched.h>
13 #include <linux/perf_counter.h>
14 #include <linux/percpu.h>
15 #include <linux/hardirq.h>
18 #include <asm/machdep.h>
19 #include <asm/firmware.h>
21 struct cpu_hw_counters {
26 struct perf_counter *counter[MAX_HWCOUNTERS];
27 unsigned int events[MAX_HWCOUNTERS];
31 DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters);
33 struct power_pmu *ppmu;
36 * Normally, to ignore kernel events we set the FCS (freeze counters
37 * in supervisor mode) bit in MMCR0, but if the kernel runs with the
38 * hypervisor bit set in the MSR, or if we are running on a processor
39 * where the hypervisor bit is forced to 1 (as on Apple G5 processors),
40 * then we need to use the FCHV bit to ignore kernel events.
42 static unsigned int freeze_counters_kernel = MMCR0_FCS;
44 static void perf_counter_interrupt(struct pt_regs *regs);
46 void perf_counter_print_debug(void)
51 * Read one performance monitor counter (PMC).
53 static unsigned long read_pmc(int idx)
59 val = mfspr(SPRN_PMC1);
62 val = mfspr(SPRN_PMC2);
65 val = mfspr(SPRN_PMC3);
68 val = mfspr(SPRN_PMC4);
71 val = mfspr(SPRN_PMC5);
74 val = mfspr(SPRN_PMC6);
77 val = mfspr(SPRN_PMC7);
80 val = mfspr(SPRN_PMC8);
83 printk(KERN_ERR "oops trying to read PMC%d\n", idx);
92 static void write_pmc(int idx, unsigned long val)
96 mtspr(SPRN_PMC1, val);
99 mtspr(SPRN_PMC2, val);
102 mtspr(SPRN_PMC3, val);
105 mtspr(SPRN_PMC4, val);
108 mtspr(SPRN_PMC5, val);
111 mtspr(SPRN_PMC6, val);
114 mtspr(SPRN_PMC7, val);
117 mtspr(SPRN_PMC8, val);
120 printk(KERN_ERR "oops trying to write PMC%d\n", idx);
125 * Check if a set of events can all go on the PMU at once.
126 * If they can't, this will look at alternative codes for the events
127 * and see if any combination of alternative codes is feasible.
128 * The feasible set is returned in event[].
130 static int power_check_constraints(unsigned int event[], int n_ev)
133 unsigned int alternatives[MAX_HWCOUNTERS][MAX_EVENT_ALTERNATIVES];
134 u64 amasks[MAX_HWCOUNTERS][MAX_EVENT_ALTERNATIVES];
135 u64 avalues[MAX_HWCOUNTERS][MAX_EVENT_ALTERNATIVES];
136 u64 smasks[MAX_HWCOUNTERS], svalues[MAX_HWCOUNTERS];
137 int n_alt[MAX_HWCOUNTERS], choice[MAX_HWCOUNTERS];
139 u64 addf = ppmu->add_fields;
140 u64 tadd = ppmu->test_adder;
142 if (n_ev > ppmu->n_counter)
145 /* First see if the events will go on as-is */
146 for (i = 0; i < n_ev; ++i) {
147 alternatives[i][0] = event[i];
148 if (ppmu->get_constraint(event[i], &amasks[i][0],
154 for (i = 0; i < n_ev; ++i) {
155 nv = (value | avalues[i][0]) + (value & avalues[i][0] & addf);
156 if ((((nv + tadd) ^ value) & mask) != 0 ||
157 (((nv + tadd) ^ avalues[i][0]) & amasks[i][0]) != 0)
160 mask |= amasks[i][0];
163 return 0; /* all OK */
165 /* doesn't work, gather alternatives... */
166 if (!ppmu->get_alternatives)
168 for (i = 0; i < n_ev; ++i) {
169 n_alt[i] = ppmu->get_alternatives(event[i], alternatives[i]);
170 for (j = 1; j < n_alt[i]; ++j)
171 ppmu->get_constraint(alternatives[i][j],
172 &amasks[i][j], &avalues[i][j]);
175 /* enumerate all possibilities and see if any will work */
178 value = mask = nv = 0;
181 /* we're backtracking, restore context */
187 * See if any alternative k for event i,
188 * where k > j, will satisfy the constraints.
190 while (++j < n_alt[i]) {
191 nv = (value | avalues[i][j]) +
192 (value & avalues[i][j] & addf);
193 if ((((nv + tadd) ^ value) & mask) == 0 &&
194 (((nv + tadd) ^ avalues[i][j])
195 & amasks[i][j]) == 0)
200 * No feasible alternative, backtrack
201 * to event i-1 and continue enumerating its
202 * alternatives from where we got up to.
208 * Found a feasible alternative for event i,
209 * remember where we got up to with this event,
210 * go on to the next event, and start with
211 * the first alternative for it.
217 mask |= amasks[i][j];
223 /* OK, we have a feasible combination, tell the caller the solution */
224 for (i = 0; i < n_ev; ++i)
225 event[i] = alternatives[i][choice[i]];
230 * Check if newly-added counters have consistent settings for
231 * exclude_{user,kernel,hv} with each other and any previously
234 static int check_excludes(struct perf_counter **ctrs, int n_prev, int n_new)
238 struct perf_counter *counter;
244 eu = ctrs[0]->hw_event.exclude_user;
245 ek = ctrs[0]->hw_event.exclude_kernel;
246 eh = ctrs[0]->hw_event.exclude_hv;
249 for (i = n_prev; i < n; ++i) {
251 if (counter->hw_event.exclude_user != eu ||
252 counter->hw_event.exclude_kernel != ek ||
253 counter->hw_event.exclude_hv != eh)
259 static void power_pmu_read(struct perf_counter *counter)
261 long val, delta, prev;
263 if (!counter->hw.idx)
266 * Performance monitor interrupts come even when interrupts
267 * are soft-disabled, as long as interrupts are hard-enabled.
268 * Therefore we treat them like NMIs.
271 prev = atomic64_read(&counter->hw.prev_count);
273 val = read_pmc(counter->hw.idx);
274 } while (atomic64_cmpxchg(&counter->hw.prev_count, prev, val) != prev);
276 /* The counters are only 32 bits wide */
277 delta = (val - prev) & 0xfffffffful;
278 atomic64_add(delta, &counter->count);
279 atomic64_sub(delta, &counter->hw.period_left);
283 * Disable all counters to prevent PMU interrupts and to allow
284 * counters to be added or removed.
286 u64 hw_perf_save_disable(void)
288 struct cpu_hw_counters *cpuhw;
292 local_irq_save(flags);
293 cpuhw = &__get_cpu_var(cpu_hw_counters);
295 ret = cpuhw->disabled;
301 * Check if we ever enabled the PMU on this cpu.
303 if (!cpuhw->pmcs_enabled) {
304 if (ppc_md.enable_pmcs)
305 ppc_md.enable_pmcs();
306 cpuhw->pmcs_enabled = 1;
310 * Disable instruction sampling if it was enabled
312 if (cpuhw->mmcr[2] & MMCRA_SAMPLE_ENABLE) {
314 cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
319 * Set the 'freeze counters' bit.
320 * The barrier is to make sure the mtspr has been
321 * executed and the PMU has frozen the counters
324 mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) | MMCR0_FC);
327 local_irq_restore(flags);
332 * Re-enable all counters if disable == 0.
333 * If we were previously disabled and counters were added, then
334 * put the new config on the PMU.
336 void hw_perf_restore(u64 disable)
338 struct perf_counter *counter;
339 struct cpu_hw_counters *cpuhw;
344 unsigned int hwc_index[MAX_HWCOUNTERS];
348 local_irq_save(flags);
349 cpuhw = &__get_cpu_var(cpu_hw_counters);
353 * If we didn't change anything, or only removed counters,
354 * no need to recalculate MMCR* settings and reset the PMCs.
355 * Just reenable the PMU with the current MMCR* settings
356 * (possibly updated for removal of counters).
358 if (!cpuhw->n_added) {
359 mtspr(SPRN_MMCRA, cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
360 mtspr(SPRN_MMCR1, cpuhw->mmcr[1]);
361 if (cpuhw->n_counters == 0)
362 get_lppaca()->pmcregs_in_use = 0;
367 * Compute MMCR* values for the new set of counters
369 if (ppmu->compute_mmcr(cpuhw->events, cpuhw->n_counters, hwc_index,
371 /* shouldn't ever get here */
372 printk(KERN_ERR "oops compute_mmcr failed\n");
377 * Add in MMCR0 freeze bits corresponding to the
378 * hw_event.exclude_* bits for the first counter.
379 * We have already checked that all counters have the
380 * same values for these bits as the first counter.
382 counter = cpuhw->counter[0];
383 if (counter->hw_event.exclude_user)
384 cpuhw->mmcr[0] |= MMCR0_FCP;
385 if (counter->hw_event.exclude_kernel)
386 cpuhw->mmcr[0] |= freeze_counters_kernel;
387 if (counter->hw_event.exclude_hv)
388 cpuhw->mmcr[0] |= MMCR0_FCHV;
391 * Write the new configuration to MMCR* with the freeze
392 * bit set and set the hardware counters to their initial values.
393 * Then unfreeze the counters.
395 get_lppaca()->pmcregs_in_use = 1;
396 mtspr(SPRN_MMCRA, cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
397 mtspr(SPRN_MMCR1, cpuhw->mmcr[1]);
398 mtspr(SPRN_MMCR0, (cpuhw->mmcr[0] & ~(MMCR0_PMC1CE | MMCR0_PMCjCE))
402 * Read off any pre-existing counters that need to move
405 for (i = 0; i < cpuhw->n_counters; ++i) {
406 counter = cpuhw->counter[i];
407 if (counter->hw.idx && counter->hw.idx != hwc_index[i] + 1) {
408 power_pmu_read(counter);
409 write_pmc(counter->hw.idx, 0);
415 * Initialize the PMCs for all the new and moved counters.
417 for (i = 0; i < cpuhw->n_counters; ++i) {
418 counter = cpuhw->counter[i];
422 if (counter->hw_event.irq_period) {
423 left = atomic64_read(&counter->hw.period_left);
424 if (left < 0x80000000L)
425 val = 0x80000000L - left;
427 atomic64_set(&counter->hw.prev_count, val);
428 counter->hw.idx = hwc_index[i] + 1;
429 write_pmc(counter->hw.idx, val);
430 perf_counter_update_userpage(counter);
432 cpuhw->mmcr[0] |= MMCR0_PMXE | MMCR0_FCECE;
436 mtspr(SPRN_MMCR0, cpuhw->mmcr[0]);
439 * Enable instruction sampling if necessary
441 if (cpuhw->mmcr[2] & MMCRA_SAMPLE_ENABLE) {
443 mtspr(SPRN_MMCRA, cpuhw->mmcr[2]);
447 local_irq_restore(flags);
450 static int collect_events(struct perf_counter *group, int max_count,
451 struct perf_counter *ctrs[], unsigned int *events)
454 struct perf_counter *counter;
456 if (!is_software_counter(group)) {
460 events[n++] = group->hw.config;
462 list_for_each_entry(counter, &group->sibling_list, list_entry) {
463 if (!is_software_counter(counter) &&
464 counter->state != PERF_COUNTER_STATE_OFF) {
468 events[n++] = counter->hw.config;
474 static void counter_sched_in(struct perf_counter *counter, int cpu)
476 counter->state = PERF_COUNTER_STATE_ACTIVE;
477 counter->oncpu = cpu;
478 counter->tstamp_running += counter->ctx->time - counter->tstamp_stopped;
479 if (is_software_counter(counter))
480 counter->pmu->enable(counter);
484 * Called to enable a whole group of counters.
485 * Returns 1 if the group was enabled, or -EAGAIN if it could not be.
486 * Assumes the caller has disabled interrupts and has
487 * frozen the PMU with hw_perf_save_disable.
489 int hw_perf_group_sched_in(struct perf_counter *group_leader,
490 struct perf_cpu_context *cpuctx,
491 struct perf_counter_context *ctx, int cpu)
493 struct cpu_hw_counters *cpuhw;
495 struct perf_counter *sub;
497 cpuhw = &__get_cpu_var(cpu_hw_counters);
498 n0 = cpuhw->n_counters;
499 n = collect_events(group_leader, ppmu->n_counter - n0,
500 &cpuhw->counter[n0], &cpuhw->events[n0]);
503 if (check_excludes(cpuhw->counter, n0, n))
505 if (power_check_constraints(cpuhw->events, n + n0))
507 cpuhw->n_counters = n0 + n;
511 * OK, this group can go on; update counter states etc.,
512 * and enable any software counters
514 for (i = n0; i < n0 + n; ++i)
515 cpuhw->counter[i]->hw.config = cpuhw->events[i];
516 cpuctx->active_oncpu += n;
518 counter_sched_in(group_leader, cpu);
519 list_for_each_entry(sub, &group_leader->sibling_list, list_entry) {
520 if (sub->state != PERF_COUNTER_STATE_OFF) {
521 counter_sched_in(sub, cpu);
531 * Add a counter to the PMU.
532 * If all counters are not already frozen, then we disable and
533 * re-enable the PMU in order to get hw_perf_restore to do the
534 * actual work of reconfiguring the PMU.
536 static int power_pmu_enable(struct perf_counter *counter)
538 struct cpu_hw_counters *cpuhw;
544 local_irq_save(flags);
545 pmudis = hw_perf_save_disable();
548 * Add the counter to the list (if there is room)
549 * and check whether the total set is still feasible.
551 cpuhw = &__get_cpu_var(cpu_hw_counters);
552 n0 = cpuhw->n_counters;
553 if (n0 >= ppmu->n_counter)
555 cpuhw->counter[n0] = counter;
556 cpuhw->events[n0] = counter->hw.config;
557 if (check_excludes(cpuhw->counter, n0, 1))
559 if (power_check_constraints(cpuhw->events, n0 + 1))
562 counter->hw.config = cpuhw->events[n0];
568 hw_perf_restore(pmudis);
569 local_irq_restore(flags);
574 * Remove a counter from the PMU.
576 static void power_pmu_disable(struct perf_counter *counter)
578 struct cpu_hw_counters *cpuhw;
583 local_irq_save(flags);
584 pmudis = hw_perf_save_disable();
586 power_pmu_read(counter);
588 cpuhw = &__get_cpu_var(cpu_hw_counters);
589 for (i = 0; i < cpuhw->n_counters; ++i) {
590 if (counter == cpuhw->counter[i]) {
591 while (++i < cpuhw->n_counters)
592 cpuhw->counter[i-1] = cpuhw->counter[i];
594 ppmu->disable_pmc(counter->hw.idx - 1, cpuhw->mmcr);
595 write_pmc(counter->hw.idx, 0);
597 perf_counter_update_userpage(counter);
601 if (cpuhw->n_counters == 0) {
602 /* disable exceptions if no counters are running */
603 cpuhw->mmcr[0] &= ~(MMCR0_PMXE | MMCR0_FCECE);
606 hw_perf_restore(pmudis);
607 local_irq_restore(flags);
610 struct pmu power_pmu = {
611 .enable = power_pmu_enable,
612 .disable = power_pmu_disable,
613 .read = power_pmu_read,
616 /* Number of perf_counters counting hardware events */
617 static atomic_t num_counters;
618 /* Used to avoid races in calling reserve/release_pmc_hardware */
619 static DEFINE_MUTEX(pmc_reserve_mutex);
622 * Release the PMU if this is the last perf_counter.
624 static void hw_perf_counter_destroy(struct perf_counter *counter)
626 if (!atomic_add_unless(&num_counters, -1, 1)) {
627 mutex_lock(&pmc_reserve_mutex);
628 if (atomic_dec_return(&num_counters) == 0)
629 release_pmc_hardware();
630 mutex_unlock(&pmc_reserve_mutex);
634 const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
637 struct perf_counter *ctrs[MAX_HWCOUNTERS];
638 unsigned int events[MAX_HWCOUNTERS];
643 return ERR_PTR(-ENXIO);
644 if ((s64)counter->hw_event.irq_period < 0)
645 return ERR_PTR(-EINVAL);
646 if (!perf_event_raw(&counter->hw_event)) {
647 ev = perf_event_id(&counter->hw_event);
648 if (ev >= ppmu->n_generic || ppmu->generic_events[ev] == 0)
649 return ERR_PTR(-EOPNOTSUPP);
650 ev = ppmu->generic_events[ev];
652 ev = perf_event_config(&counter->hw_event);
654 counter->hw.config_base = ev;
658 * If we are not running on a hypervisor, force the
659 * exclude_hv bit to 0 so that we don't care what
660 * the user set it to.
662 if (!firmware_has_feature(FW_FEATURE_LPAR))
663 counter->hw_event.exclude_hv = 0;
666 * If this is in a group, check if it can go on with all the
667 * other hardware counters in the group. We assume the counter
668 * hasn't been linked into its leader's sibling list at this point.
671 if (counter->group_leader != counter) {
672 n = collect_events(counter->group_leader, ppmu->n_counter - 1,
675 return ERR_PTR(-EINVAL);
679 if (check_excludes(ctrs, n, 1))
680 return ERR_PTR(-EINVAL);
681 if (power_check_constraints(events, n + 1))
682 return ERR_PTR(-EINVAL);
684 counter->hw.config = events[n];
685 atomic64_set(&counter->hw.period_left, counter->hw_event.irq_period);
688 * See if we need to reserve the PMU.
689 * If no counters are currently in use, then we have to take a
690 * mutex to ensure that we don't race with another task doing
691 * reserve_pmc_hardware or release_pmc_hardware.
694 if (!atomic_inc_not_zero(&num_counters)) {
695 mutex_lock(&pmc_reserve_mutex);
696 if (atomic_read(&num_counters) == 0 &&
697 reserve_pmc_hardware(perf_counter_interrupt))
700 atomic_inc(&num_counters);
701 mutex_unlock(&pmc_reserve_mutex);
703 counter->destroy = hw_perf_counter_destroy;
711 * A counter has overflowed; update its count and record
712 * things if requested. Note that interrupts are hard-disabled
713 * here so there is no possibility of being interrupted.
715 static void record_and_restart(struct perf_counter *counter, long val,
716 struct pt_regs *regs, int nmi)
718 s64 prev, delta, left;
721 /* we don't have to worry about interrupts here */
722 prev = atomic64_read(&counter->hw.prev_count);
723 delta = (val - prev) & 0xfffffffful;
724 atomic64_add(delta, &counter->count);
727 * See if the total period for this counter has expired,
728 * and update for the next period.
731 left = atomic64_read(&counter->hw.period_left) - delta;
732 if (counter->hw_event.irq_period) {
734 left += counter->hw_event.irq_period;
736 left = counter->hw_event.irq_period;
739 if (left < 0x80000000L)
740 val = 0x80000000L - left;
742 write_pmc(counter->hw.idx, val);
743 atomic64_set(&counter->hw.prev_count, val);
744 atomic64_set(&counter->hw.period_left, left);
745 perf_counter_update_userpage(counter);
748 * Finally record data if requested.
751 perf_counter_overflow(counter, nmi, regs, 0);
755 * Performance monitor interrupt stuff
757 static void perf_counter_interrupt(struct pt_regs *regs)
760 struct cpu_hw_counters *cpuhw = &__get_cpu_var(cpu_hw_counters);
761 struct perf_counter *counter;
767 * If interrupts were soft-disabled when this PMU interrupt
768 * occurred, treat it as an NMI.
776 for (i = 0; i < cpuhw->n_counters; ++i) {
777 counter = cpuhw->counter[i];
778 val = read_pmc(counter->hw.idx);
780 /* counter has overflowed */
782 record_and_restart(counter, val, regs, nmi);
787 * In case we didn't find and reset the counter that caused
788 * the interrupt, scan all counters and reset any that are
789 * negative, to avoid getting continual interrupts.
790 * Any that we processed in the previous loop will not be negative.
793 for (i = 0; i < ppmu->n_counter; ++i) {
794 val = read_pmc(i + 1);
801 * Reset MMCR0 to its normal value. This will set PMXE and
802 * clear FC (freeze counters) and PMAO (perf mon alert occurred)
803 * and thus allow interrupts to occur again.
804 * XXX might want to use MSR.PM to keep the counters frozen until
805 * we get back out of this interrupt.
807 mtspr(SPRN_MMCR0, cpuhw->mmcr[0]);
815 void hw_perf_counter_setup(int cpu)
817 struct cpu_hw_counters *cpuhw = &per_cpu(cpu_hw_counters, cpu);
819 memset(cpuhw, 0, sizeof(*cpuhw));
820 cpuhw->mmcr[0] = MMCR0_FC;
823 extern struct power_pmu power4_pmu;
824 extern struct power_pmu ppc970_pmu;
825 extern struct power_pmu power5_pmu;
826 extern struct power_pmu power5p_pmu;
827 extern struct power_pmu power6_pmu;
829 static int init_perf_counters(void)
833 /* XXX should get this from cputable */
834 pvr = mfspr(SPRN_PVR);
835 switch (PVR_VER(pvr)) {
857 * Use FCHV to ignore kernel events if MSR.HV is set.
859 if (mfmsr() & MSR_HV)
860 freeze_counters_kernel = MMCR0_FCHV;
865 arch_initcall(init_perf_counters);