arch/powerpc/kernel/perf_counter.c

   1 /*
   2  * Performance counter support - powerpc architecture code
   3  *
   4  * Copyright 2008-2009 Paul Mackerras, IBM Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version
   9  * 2 of the License, or (at your option) any later version.
  10  */
  11 #include <linux/kernel.h>
  12 #include <linux/sched.h>
  13 #include <linux/perf_counter.h>
  14 #include <linux/percpu.h>
  15 #include <linux/hardirq.h>
  16 #include <asm/reg.h>
  17 #include <asm/pmc.h>
  18 #include <asm/machdep.h>
  19 #include <asm/firmware.h>
  20 #include <asm/ptrace.h>
  21
  22 struct cpu_hw_counters {
  23         int n_counters;
  24         int n_percpu;
  25         int disabled;
  26         int n_added;
  27         int n_limited;
  28         u8  pmcs_enabled;
  29         struct perf_counter *counter[MAX_HWCOUNTERS];
  30         u64 events[MAX_HWCOUNTERS];
  31         unsigned int flags[MAX_HWCOUNTERS];
  32         unsigned long mmcr[3];
  33         struct perf_counter *limited_counter[MAX_LIMITED_HWCOUNTERS];
  34         u8  limited_hwidx[MAX_LIMITED_HWCOUNTERS];
  35 };
  36 DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters);
  37
  38 struct power_pmu *ppmu;
  39
  40 /*
  41  * Normally, to ignore kernel events we set the FCS (freeze counters
  42  * in supervisor mode) bit in MMCR0, but if the kernel runs with the
  43  * hypervisor bit set in the MSR, or if we are running on a processor
  44  * where the hypervisor bit is forced to 1 (as on Apple G5 processors),
  45  * then we need to use the FCHV bit to ignore kernel events.
  46  */
  47 static unsigned int freeze_counters_kernel = MMCR0_FCS;
  48
  49 /*
  50  * 32-bit doesn't have MMCRA but does have an MMCR2,
  51  * and a few other names are different.
  52  */
  53 #ifdef CONFIG_PPC32
  54
  55 #define MMCR0_FCHV              0
  56 #define MMCR0_PMCjCE            MMCR0_PMCnCE
  57
  58 #define SPRN_MMCRA              SPRN_MMCR2
  59 #define MMCRA_SAMPLE_ENABLE     0
  60
  61 static inline unsigned long perf_ip_adjust(struct pt_regs *regs)
  62 {
  63         return 0;
  64 }
  65 static inline void perf_set_pmu_inuse(int inuse) { }
  66 static inline void perf_get_data_addr(struct pt_regs *regs, u64 *addrp) { }
  67 static inline u32 perf_get_misc_flags(struct pt_regs *regs)
  68 {
  69         return 0;
  70 }
  71 static inline void perf_read_regs(struct pt_regs *regs) { }
  72 static inline int perf_intr_is_nmi(struct pt_regs *regs)
  73 {
  74         return 0;
  75 }
  76
  77 #endif /* CONFIG_PPC32 */
  78
  79 /*
  80  * Things that are specific to 64-bit implementations.
  81  */
  82 #ifdef CONFIG_PPC64
  83
  84 static inline unsigned long perf_ip_adjust(struct pt_regs *regs)
  85 {
  86         unsigned long mmcra = regs->dsisr;
  87
  88         if ((mmcra & MMCRA_SAMPLE_ENABLE) && !(ppmu->flags & PPMU_ALT_SIPR)) {
  89                 unsigned long slot = (mmcra & MMCRA_SLOT) >> MMCRA_SLOT_SHIFT;
  90                 if (slot > 1)
  91                         return 4 * (slot - 1);
  92         }
  93         return 0;
  94 }
  95
  96 static inline void perf_set_pmu_inuse(int inuse)
  97 {
  98         get_lppaca()->pmcregs_in_use = inuse;
  99 }
 100
 101 /*
 102  * The user wants a data address recorded.
 103  * If we're not doing instruction sampling, give them the SDAR
 104  * (sampled data address).  If we are doing instruction sampling, then
 105  * only give them the SDAR if it corresponds to the instruction
 106  * pointed to by SIAR; this is indicated by the [POWER6_]MMCRA_SDSYNC
 107  * bit in MMCRA.
 108  */
 109 static inline void perf_get_data_addr(struct pt_regs *regs, u64 *addrp)
 110 {
 111         unsigned long mmcra = regs->dsisr;
 112         unsigned long sdsync = (ppmu->flags & PPMU_ALT_SIPR) ?
 113                 POWER6_MMCRA_SDSYNC : MMCRA_SDSYNC;
 114
 115         if (!(mmcra & MMCRA_SAMPLE_ENABLE) || (mmcra & sdsync))
 116                 *addrp = mfspr(SPRN_SDAR);
 117 }
 118
 119 static inline u32 perf_get_misc_flags(struct pt_regs *regs)
 120 {
 121         unsigned long mmcra = regs->dsisr;
 122
 123         if (TRAP(regs) != 0xf00)
 124                 return 0;       /* not a PMU interrupt */
 125
 126         if (ppmu->flags & PPMU_ALT_SIPR) {
 127                 if (mmcra & POWER6_MMCRA_SIHV)
 128                         return PERF_EVENT_MISC_HYPERVISOR;
 129                 return (mmcra & POWER6_MMCRA_SIPR) ?
 130                         PERF_EVENT_MISC_USER : PERF_EVENT_MISC_KERNEL;
 131         }
 132         if (mmcra & MMCRA_SIHV)
 133                 return PERF_EVENT_MISC_HYPERVISOR;
 134         return (mmcra & MMCRA_SIPR) ? PERF_EVENT_MISC_USER :
 135                 PERF_EVENT_MISC_KERNEL;
 136 }
 137
 138 /*
 139  * Overload regs->dsisr to store MMCRA so we only need to read it once
 140  * on each interrupt.
 141  */
 142 static inline void perf_read_regs(struct pt_regs *regs)
 143 {
 144         regs->dsisr = mfspr(SPRN_MMCRA);
 145 }
 146
 147 /*
 148  * If interrupts were soft-disabled when a PMU interrupt occurs, treat
 149  * it as an NMI.
 150  */
 151 static inline int perf_intr_is_nmi(struct pt_regs *regs)
 152 {
 153         return !regs->softe;
 154 }
 155
 156 #endif /* CONFIG_PPC64 */
 157
 158 static void perf_counter_interrupt(struct pt_regs *regs);
 159
 160 void perf_counter_print_debug(void)
 161 {
 162 }
 163
 164 /*
 165  * Read one performance monitor counter (PMC).
 166  */
 167 static unsigned long read_pmc(int idx)
 168 {
 169         unsigned long val;
 170
 171         switch (idx) {
 172         case 1:
 173                 val = mfspr(SPRN_PMC1);
 174                 break;
 175         case 2:
 176                 val = mfspr(SPRN_PMC2);
 177                 break;
 178         case 3:
 179                 val = mfspr(SPRN_PMC3);
 180                 break;
 181         case 4:
 182                 val = mfspr(SPRN_PMC4);
 183                 break;
 184         case 5:
 185                 val = mfspr(SPRN_PMC5);
 186                 break;
 187         case 6:
 188                 val = mfspr(SPRN_PMC6);
 189                 break;
 190 #ifdef CONFIG_PPC64
 191         case 7:
 192                 val = mfspr(SPRN_PMC7);
 193                 break;
 194         case 8:
 195                 val = mfspr(SPRN_PMC8);
 196                 break;
 197 #endif /* CONFIG_PPC64 */
 198         default:
 199                 printk(KERN_ERR "oops trying to read PMC%d\n", idx);
 200                 val = 0;
 201         }
 202         return val;
 203 }
 204
 205 /*
 206  * Write one PMC.
 207  */
 208 static void write_pmc(int idx, unsigned long val)
 209 {
 210         switch (idx) {
 211         case 1:
 212                 mtspr(SPRN_PMC1, val);
 213                 break;
 214         case 2:
 215                 mtspr(SPRN_PMC2, val);
 216                 break;
 217         case 3:
 218                 mtspr(SPRN_PMC3, val);
 219                 break;
 220         case 4:
 221                 mtspr(SPRN_PMC4, val);
 222                 break;
 223         case 5:
 224                 mtspr(SPRN_PMC5, val);
 225                 break;
 226         case 6:
 227                 mtspr(SPRN_PMC6, val);
 228                 break;
 229 #ifdef CONFIG_PPC64
 230         case 7:
 231                 mtspr(SPRN_PMC7, val);
 232                 break;
 233         case 8:
 234                 mtspr(SPRN_PMC8, val);
 235                 break;
 236 #endif /* CONFIG_PPC64 */
 237         default:
 238                 printk(KERN_ERR "oops trying to write PMC%d\n", idx);
 239         }
 240 }
 241
 242 /*
 243  * Check if a set of events can all go on the PMU at once.
 244  * If they can't, this will look at alternative codes for the events
 245  * and see if any combination of alternative codes is feasible.
 246  * The feasible set is returned in event[].
 247  */
 248 static int power_check_constraints(u64 event[], unsigned int cflags[],
 249                                    int n_ev)
 250 {
 251         unsigned long mask, value, nv;
 252         u64 alternatives[MAX_HWCOUNTERS][MAX_EVENT_ALTERNATIVES];
 253         unsigned long amasks[MAX_HWCOUNTERS][MAX_EVENT_ALTERNATIVES];
 254         unsigned long avalues[MAX_HWCOUNTERS][MAX_EVENT_ALTERNATIVES];
 255         unsigned long smasks[MAX_HWCOUNTERS], svalues[MAX_HWCOUNTERS];
 256         int n_alt[MAX_HWCOUNTERS], choice[MAX_HWCOUNTERS];
 257         int i, j;
 258         unsigned long addf = ppmu->add_fields;
 259         unsigned long tadd = ppmu->test_adder;
 260
 261         if (n_ev > ppmu->n_counter)
 262                 return -1;
 263
 264         /* First see if the events will go on as-is */
 265         for (i = 0; i < n_ev; ++i) {
 266                 if ((cflags[i] & PPMU_LIMITED_PMC_REQD)
 267                     && !ppmu->limited_pmc_event(event[i])) {
 268                         ppmu->get_alternatives(event[i], cflags[i],
 269                                                alternatives[i]);
 270                         event[i] = alternatives[i][0];
 271                 }
 272                 if (ppmu->get_constraint(event[i], &amasks[i][0],
 273                                          &avalues[i][0]))
 274                         return -1;
 275         }
 276         value = mask = 0;
 277         for (i = 0; i < n_ev; ++i) {
 278                 nv = (value | avalues[i][0]) + (value & avalues[i][0] & addf);
 279                 if ((((nv + tadd) ^ value) & mask) != 0 ||
 280                     (((nv + tadd) ^ avalues[i][0]) & amasks[i][0]) != 0)
 281                         break;
 282                 value = nv;
 283                 mask |= amasks[i][0];
 284         }
 285         if (i == n_ev)
 286                 return 0;       /* all OK */
 287
 288         /* doesn't work, gather alternatives... */
 289         if (!ppmu->get_alternatives)
 290                 return -1;
 291         for (i = 0; i < n_ev; ++i) {
 292                 choice[i] = 0;
 293                 n_alt[i] = ppmu->get_alternatives(event[i], cflags[i],
 294                                                   alternatives[i]);
 295                 for (j = 1; j < n_alt[i]; ++j)
 296                         ppmu->get_constraint(alternatives[i][j],
 297                                              &amasks[i][j], &avalues[i][j]);
 298         }
 299
 300         /* enumerate all possibilities and see if any will work */
 301         i = 0;
 302         j = -1;
 303         value = mask = nv = 0;
 304         while (i < n_ev) {
 305                 if (j >= 0) {
 306                         /* we're backtracking, restore context */
 307                         value = svalues[i];
 308                         mask = smasks[i];
 309                         j = choice[i];
 310                 }
 311                 /*
 312                  * See if any alternative k for event i,
 313                  * where k > j, will satisfy the constraints.
 314                  */
 315                 while (++j < n_alt[i]) {
 316                         nv = (value | avalues[i][j]) +
 317                                 (value & avalues[i][j] & addf);
 318                         if ((((nv + tadd) ^ value) & mask) == 0 &&
 319                             (((nv + tadd) ^ avalues[i][j])
 320                              & amasks[i][j]) == 0)
 321                                 break;
 322                 }
 323                 if (j >= n_alt[i]) {
 324                         /*
 325                          * No feasible alternative, backtrack
 326                          * to event i-1 and continue enumerating its
 327                          * alternatives from where we got up to.
 328                          */
 329                         if (--i < 0)
 330                                 return -1;
 331                 } else {
 332                         /*
 333                          * Found a feasible alternative for event i,
 334                          * remember where we got up to with this event,
 335                          * go on to the next event, and start with
 336                          * the first alternative for it.
 337                          */
 338                         choice[i] = j;
 339                         svalues[i] = value;
 340                         smasks[i] = mask;
 341                         value = nv;
 342                         mask |= amasks[i][j];
 343                         ++i;
 344                         j = -1;
 345                 }
 346         }
 347
 348         /* OK, we have a feasible combination, tell the caller the solution */
 349         for (i = 0; i < n_ev; ++i)
 350                 event[i] = alternatives[i][choice[i]];
 351         return 0;
 352 }
 353
 354 /*
 355  * Check if newly-added counters have consistent settings for
 356  * exclude_{user,kernel,hv} with each other and any previously
 357  * added counters.
 358  */
 359 static int check_excludes(struct perf_counter **ctrs, unsigned int cflags[],
 360                           int n_prev, int n_new)
 361 {
 362         int eu = 0, ek = 0, eh = 0;
 363         int i, n, first;
 364         struct perf_counter *counter;
 365
 366         n = n_prev + n_new;
 367         if (n <= 1)
 368                 return 0;
 369
 370         first = 1;
 371         for (i = 0; i < n; ++i) {
 372                 if (cflags[i] & PPMU_LIMITED_PMC_OK) {
 373                         cflags[i] &= ~PPMU_LIMITED_PMC_REQD;
 374                         continue;
 375                 }
 376                 counter = ctrs[i];
 377                 if (first) {
 378                         eu = counter->attr.exclude_user;
 379                         ek = counter->attr.exclude_kernel;
 380                         eh = counter->attr.exclude_hv;
 381                         first = 0;
 382                 } else if (counter->attr.exclude_user != eu ||
 383                            counter->attr.exclude_kernel != ek ||
 384                            counter->attr.exclude_hv != eh) {
 385                         return -EAGAIN;
 386                 }
 387         }
 388
 389         if (eu || ek || eh)
 390                 for (i = 0; i < n; ++i)
 391                         if (cflags[i] & PPMU_LIMITED_PMC_OK)
 392                                 cflags[i] |= PPMU_LIMITED_PMC_REQD;
 393
 394         return 0;
 395 }
 396
 397 static void power_pmu_read(struct perf_counter *counter)
 398 {
 399         s64 val, delta, prev;
 400
 401         if (!counter->hw.idx)
 402                 return;
 403         /*
 404          * Performance monitor interrupts come even when interrupts
 405          * are soft-disabled, as long as interrupts are hard-enabled.
 406          * Therefore we treat them like NMIs.
 407          */
 408         do {
 409                 prev = atomic64_read(&counter->hw.prev_count);
 410                 barrier();
 411                 val = read_pmc(counter->hw.idx);
 412         } while (atomic64_cmpxchg(&counter->hw.prev_count, prev, val) != prev);
 413
 414         /* The counters are only 32 bits wide */
 415         delta = (val - prev) & 0xfffffffful;
 416         atomic64_add(delta, &counter->count);
 417         atomic64_sub(delta, &counter->hw.period_left);
 418 }
 419
 420 /*
 421  * On some machines, PMC5 and PMC6 can't be written, don't respect
 422  * the freeze conditions, and don't generate interrupts.  This tells
 423  * us if `counter' is using such a PMC.
 424  */
 425 static int is_limited_pmc(int pmcnum)
 426 {
 427         return (ppmu->flags & PPMU_LIMITED_PMC5_6)
 428                 && (pmcnum == 5 || pmcnum == 6);
 429 }
 430
 431 static void freeze_limited_counters(struct cpu_hw_counters *cpuhw,
 432                                     unsigned long pmc5, unsigned long pmc6)
 433 {
 434         struct perf_counter *counter;
 435         u64 val, prev, delta;
 436         int i;
 437
 438         for (i = 0; i < cpuhw->n_limited; ++i) {
 439                 counter = cpuhw->limited_counter[i];
 440                 if (!counter->hw.idx)
 441                         continue;
 442                 val = (counter->hw.idx == 5) ? pmc5 : pmc6;
 443                 prev = atomic64_read(&counter->hw.prev_count);
 444                 counter->hw.idx = 0;
 445                 delta = (val - prev) & 0xfffffffful;
 446                 atomic64_add(delta, &counter->count);
 447         }
 448 }
 449
 450 static void thaw_limited_counters(struct cpu_hw_counters *cpuhw,
 451                                   unsigned long pmc5, unsigned long pmc6)
 452 {
 453         struct perf_counter *counter;
 454         u64 val;
 455         int i;
 456
 457         for (i = 0; i < cpuhw->n_limited; ++i) {
 458                 counter = cpuhw->limited_counter[i];
 459                 counter->hw.idx = cpuhw->limited_hwidx[i];
 460                 val = (counter->hw.idx == 5) ? pmc5 : pmc6;
 461                 atomic64_set(&counter->hw.prev_count, val);
 462                 perf_counter_update_userpage(counter);
 463         }
 464 }
 465
 466 /*
 467  * Since limited counters don't respect the freeze conditions, we
 468  * have to read them immediately after freezing or unfreezing the
 469  * other counters.  We try to keep the values from the limited
 470  * counters as consistent as possible by keeping the delay (in
 471  * cycles and instructions) between freezing/unfreezing and reading
 472  * the limited counters as small and consistent as possible.
 473  * Therefore, if any limited counters are in use, we read them
 474  * both, and always in the same order, to minimize variability,
 475  * and do it inside the same asm that writes MMCR0.
 476  */
 477 static void write_mmcr0(struct cpu_hw_counters *cpuhw, unsigned long mmcr0)
 478 {
 479         unsigned long pmc5, pmc6;
 480
 481         if (!cpuhw->n_limited) {
 482                 mtspr(SPRN_MMCR0, mmcr0);
 483                 return;
 484         }
 485
 486         /*
 487          * Write MMCR0, then read PMC5 and PMC6 immediately.
 488          * To ensure we don't get a performance monitor interrupt
 489          * between writing MMCR0 and freezing/thawing the limited
 490          * counters, we first write MMCR0 with the counter overflow
 491          * interrupt enable bits turned off.
 492          */
 493         asm volatile("mtspr %3,%2; mfspr %0,%4; mfspr %1,%5"
 494                      : "=&r" (pmc5), "=&r" (pmc6)
 495                      : "r" (mmcr0 & ~(MMCR0_PMC1CE | MMCR0_PMCjCE)),
 496                        "i" (SPRN_MMCR0),
 497                        "i" (SPRN_PMC5), "i" (SPRN_PMC6));
 498
 499         if (mmcr0 & MMCR0_FC)
 500                 freeze_limited_counters(cpuhw, pmc5, pmc6);
 501         else
 502                 thaw_limited_counters(cpuhw, pmc5, pmc6);
 503
 504         /*
 505          * Write the full MMCR0 including the counter overflow interrupt
 506          * enable bits, if necessary.
 507          */
 508         if (mmcr0 & (MMCR0_PMC1CE | MMCR0_PMCjCE))
 509                 mtspr(SPRN_MMCR0, mmcr0);
 510 }
 511
 512 /*
 513  * Disable all counters to prevent PMU interrupts and to allow
 514  * counters to be added or removed.
 515  */
 516 void hw_perf_disable(void)
 517 {
 518         struct cpu_hw_counters *cpuhw;
 519         unsigned long flags;
 520
 521         local_irq_save(flags);
 522         cpuhw = &__get_cpu_var(cpu_hw_counters);
 523
 524         if (!cpuhw->disabled) {
 525                 cpuhw->disabled = 1;
 526                 cpuhw->n_added = 0;
 527
 528                 /*
 529                  * Check if we ever enabled the PMU on this cpu.
 530                  */
 531                 if (!cpuhw->pmcs_enabled) {
 532                         if (ppc_md.enable_pmcs)
 533                                 ppc_md.enable_pmcs();
 534                         cpuhw->pmcs_enabled = 1;
 535                 }
 536
 537                 /*
 538                  * Disable instruction sampling if it was enabled
 539                  */
 540                 if (cpuhw->mmcr[2] & MMCRA_SAMPLE_ENABLE) {
 541                         mtspr(SPRN_MMCRA,
 542                               cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
 543                         mb();
 544                 }
 545
 546                 /*
 547                  * Set the 'freeze counters' bit.
 548                  * The barrier is to make sure the mtspr has been
 549                  * executed and the PMU has frozen the counters
 550                  * before we return.
 551                  */
 552                 write_mmcr0(cpuhw, mfspr(SPRN_MMCR0) | MMCR0_FC);
 553                 mb();
 554         }
 555         local_irq_restore(flags);
 556 }
 557
 558 /*
 559  * Re-enable all counters if disable == 0.
 560  * If we were previously disabled and counters were added, then
 561  * put the new config on the PMU.
 562  */
 563 void hw_perf_enable(void)
 564 {
 565         struct perf_counter *counter;
 566         struct cpu_hw_counters *cpuhw;
 567         unsigned long flags;
 568         long i;
 569         unsigned long val;
 570         s64 left;
 571         unsigned int hwc_index[MAX_HWCOUNTERS];
 572         int n_lim;
 573         int idx;
 574
 575         local_irq_save(flags);
 576         cpuhw = &__get_cpu_var(cpu_hw_counters);
 577         if (!cpuhw->disabled) {
 578                 local_irq_restore(flags);
 579                 return;
 580         }
 581         cpuhw->disabled = 0;
 582
 583         /*
 584          * If we didn't change anything, or only removed counters,
 585          * no need to recalculate MMCR* settings and reset the PMCs.
 586          * Just reenable the PMU with the current MMCR* settings
 587          * (possibly updated for removal of counters).
 588          */
 589         if (!cpuhw->n_added) {
 590                 mtspr(SPRN_MMCRA, cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
 591                 mtspr(SPRN_MMCR1, cpuhw->mmcr[1]);
 592                 if (cpuhw->n_counters == 0)
 593                         perf_set_pmu_inuse(0);
 594                 goto out_enable;
 595         }
 596
 597         /*
 598          * Compute MMCR* values for the new set of counters
 599          */
 600         if (ppmu->compute_mmcr(cpuhw->events, cpuhw->n_counters, hwc_index,
 601                                cpuhw->mmcr)) {
 602                 /* shouldn't ever get here */
 603                 printk(KERN_ERR "oops compute_mmcr failed\n");
 604                 goto out;
 605         }
 606
 607         /*
 608          * Add in MMCR0 freeze bits corresponding to the
 609          * attr.exclude_* bits for the first counter.
 610          * We have already checked that all counters have the
 611          * same values for these bits as the first counter.
 612          */
 613         counter = cpuhw->counter[0];
 614         if (counter->attr.exclude_user)
 615                 cpuhw->mmcr[0] |= MMCR0_FCP;
 616         if (counter->attr.exclude_kernel)
 617                 cpuhw->mmcr[0] |= freeze_counters_kernel;
 618         if (counter->attr.exclude_hv)
 619                 cpuhw->mmcr[0] |= MMCR0_FCHV;
 620
 621         /*
 622          * Write the new configuration to MMCR* with the freeze
 623          * bit set and set the hardware counters to their initial values.
 624          * Then unfreeze the counters.
 625          */
 626         perf_set_pmu_inuse(1);
 627         mtspr(SPRN_MMCRA, cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
 628         mtspr(SPRN_MMCR1, cpuhw->mmcr[1]);
 629         mtspr(SPRN_MMCR0, (cpuhw->mmcr[0] & ~(MMCR0_PMC1CE | MMCR0_PMCjCE))
 630                                 | MMCR0_FC);
 631
 632         /*
 633          * Read off any pre-existing counters that need to move
 634          * to another PMC.
 635          */
 636         for (i = 0; i < cpuhw->n_counters; ++i) {
 637                 counter = cpuhw->counter[i];
 638                 if (counter->hw.idx && counter->hw.idx != hwc_index[i] + 1) {
 639                         power_pmu_read(counter);
 640                         write_pmc(counter->hw.idx, 0);
 641                         counter->hw.idx = 0;
 642                 }
 643         }
 644
 645         /*
 646          * Initialize the PMCs for all the new and moved counters.
 647          */
 648         cpuhw->n_limited = n_lim = 0;
 649         for (i = 0; i < cpuhw->n_counters; ++i) {
 650                 counter = cpuhw->counter[i];
 651                 if (counter->hw.idx)
 652                         continue;
 653                 idx = hwc_index[i] + 1;
 654                 if (is_limited_pmc(idx)) {
 655                         cpuhw->limited_counter[n_lim] = counter;
 656                         cpuhw->limited_hwidx[n_lim] = idx;
 657                         ++n_lim;
 658                         continue;
 659                 }
 660                 val = 0;
 661                 if (counter->hw.sample_period) {
 662                         left = atomic64_read(&counter->hw.period_left);
 663                         if (left < 0x80000000L)
 664                                 val = 0x80000000L - left;
 665                 }
 666                 atomic64_set(&counter->hw.prev_count, val);
 667                 counter->hw.idx = idx;
 668                 write_pmc(idx, val);
 669                 perf_counter_update_userpage(counter);
 670         }
 671         cpuhw->n_limited = n_lim;
 672         cpuhw->mmcr[0] |= MMCR0_PMXE | MMCR0_FCECE;
 673
 674  out_enable:
 675         mb();
 676         write_mmcr0(cpuhw, cpuhw->mmcr[0]);
 677
 678         /*
 679          * Enable instruction sampling if necessary
 680          */
 681         if (cpuhw->mmcr[2] & MMCRA_SAMPLE_ENABLE) {
 682                 mb();
 683                 mtspr(SPRN_MMCRA, cpuhw->mmcr[2]);
 684         }
 685
 686  out:
 687         local_irq_restore(flags);
 688 }
 689
 690 static int collect_events(struct perf_counter *group, int max_count,
 691                           struct perf_counter *ctrs[], u64 *events,
 692                           unsigned int *flags)
 693 {
 694         int n = 0;
 695         struct perf_counter *counter;
 696
 697         if (!is_software_counter(group)) {
 698                 if (n >= max_count)
 699                         return -1;
 700                 ctrs[n] = group;
 701                 flags[n] = group->hw.counter_base;
 702                 events[n++] = group->hw.config;
 703         }
 704         list_for_each_entry(counter, &group->sibling_list, list_entry) {
 705                 if (!is_software_counter(counter) &&
 706                     counter->state != PERF_COUNTER_STATE_OFF) {
 707                         if (n >= max_count)
 708                                 return -1;
 709                         ctrs[n] = counter;
 710                         flags[n] = counter->hw.counter_base;
 711                         events[n++] = counter->hw.config;
 712                 }
 713         }
 714         return n;
 715 }
 716
 717 static void counter_sched_in(struct perf_counter *counter, int cpu)
 718 {
 719         counter->state = PERF_COUNTER_STATE_ACTIVE;
 720         counter->oncpu = cpu;
 721         counter->tstamp_running += counter->ctx->time - counter->tstamp_stopped;
 722         if (is_software_counter(counter))
 723                 counter->pmu->enable(counter);
 724 }
 725
 726 /*
 727  * Called to enable a whole group of counters.
 728  * Returns 1 if the group was enabled, or -EAGAIN if it could not be.
 729  * Assumes the caller has disabled interrupts and has
 730  * frozen the PMU with hw_perf_save_disable.
 731  */
 732 int hw_perf_group_sched_in(struct perf_counter *group_leader,
 733                struct perf_cpu_context *cpuctx,
 734                struct perf_counter_context *ctx, int cpu)
 735 {
 736         struct cpu_hw_counters *cpuhw;
 737         long i, n, n0;
 738         struct perf_counter *sub;
 739
 740         cpuhw = &__get_cpu_var(cpu_hw_counters);
 741         n0 = cpuhw->n_counters;
 742         n = collect_events(group_leader, ppmu->n_counter - n0,
 743                            &cpuhw->counter[n0], &cpuhw->events[n0],
 744                            &cpuhw->flags[n0]);
 745         if (n < 0)
 746                 return -EAGAIN;
 747         if (check_excludes(cpuhw->counter, cpuhw->flags, n0, n))
 748                 return -EAGAIN;
 749         i = power_check_constraints(cpuhw->events, cpuhw->flags, n + n0);
 750         if (i < 0)
 751                 return -EAGAIN;
 752         cpuhw->n_counters = n0 + n;
 753         cpuhw->n_added += n;
 754
 755         /*
 756          * OK, this group can go on; update counter states etc.,
 757          * and enable any software counters
 758          */
 759         for (i = n0; i < n0 + n; ++i)
 760                 cpuhw->counter[i]->hw.config = cpuhw->events[i];
 761         cpuctx->active_oncpu += n;
 762         n = 1;
 763         counter_sched_in(group_leader, cpu);
 764         list_for_each_entry(sub, &group_leader->sibling_list, list_entry) {
 765                 if (sub->state != PERF_COUNTER_STATE_OFF) {
 766                         counter_sched_in(sub, cpu);
 767                         ++n;
 768                 }
 769         }
 770         ctx->nr_active += n;
 771
 772         return 1;
 773 }
 774
 775 /*
 776  * Add a counter to the PMU.
 777  * If all counters are not already frozen, then we disable and
 778  * re-enable the PMU in order to get hw_perf_enable to do the
 779  * actual work of reconfiguring the PMU.
 780  */
 781 static int power_pmu_enable(struct perf_counter *counter)
 782 {
 783         struct cpu_hw_counters *cpuhw;
 784         unsigned long flags;
 785         int n0;
 786         int ret = -EAGAIN;
 787
 788         local_irq_save(flags);
 789         perf_disable();
 790
 791         /*
 792          * Add the counter to the list (if there is room)
 793          * and check whether the total set is still feasible.
 794          */
 795         cpuhw = &__get_cpu_var(cpu_hw_counters);
 796         n0 = cpuhw->n_counters;
 797         if (n0 >= ppmu->n_counter)
 798                 goto out;
 799         cpuhw->counter[n0] = counter;
 800         cpuhw->events[n0] = counter->hw.config;
 801         cpuhw->flags[n0] = counter->hw.counter_base;
 802         if (check_excludes(cpuhw->counter, cpuhw->flags, n0, 1))
 803                 goto out;
 804         if (power_check_constraints(cpuhw->events, cpuhw->flags, n0 + 1))
 805                 goto out;
 806
 807         counter->hw.config = cpuhw->events[n0];
 808         ++cpuhw->n_counters;
 809         ++cpuhw->n_added;
 810
 811         ret = 0;
 812  out:
 813         perf_enable();
 814         local_irq_restore(flags);
 815         return ret;
 816 }
 817
 818 /*
 819  * Remove a counter from the PMU.
 820  */
 821 static void power_pmu_disable(struct perf_counter *counter)
 822 {
 823         struct cpu_hw_counters *cpuhw;
 824         long i;
 825         unsigned long flags;
 826
 827         local_irq_save(flags);
 828         perf_disable();
 829
 830         power_pmu_read(counter);
 831
 832         cpuhw = &__get_cpu_var(cpu_hw_counters);
 833         for (i = 0; i < cpuhw->n_counters; ++i) {
 834                 if (counter == cpuhw->counter[i]) {
 835                         while (++i < cpuhw->n_counters)
 836                                 cpuhw->counter[i-1] = cpuhw->counter[i];
 837                         --cpuhw->n_counters;
 838                         ppmu->disable_pmc(counter->hw.idx - 1, cpuhw->mmcr);
 839                         if (counter->hw.idx) {
 840                                 write_pmc(counter->hw.idx, 0);
 841                                 counter->hw.idx = 0;
 842                         }
 843                         perf_counter_update_userpage(counter);
 844                         break;
 845                 }
 846         }
 847         for (i = 0; i < cpuhw->n_limited; ++i)
 848                 if (counter == cpuhw->limited_counter[i])
 849                         break;
 850         if (i < cpuhw->n_limited) {
 851                 while (++i < cpuhw->n_limited) {
 852                         cpuhw->limited_counter[i-1] = cpuhw->limited_counter[i];
 853                         cpuhw->limited_hwidx[i-1] = cpuhw->limited_hwidx[i];
 854                 }
 855                 --cpuhw->n_limited;
 856         }
 857         if (cpuhw->n_counters == 0) {
 858                 /* disable exceptions if no counters are running */
 859                 cpuhw->mmcr[0] &= ~(MMCR0_PMXE | MMCR0_FCECE);
 860         }
 861
 862         perf_enable();
 863         local_irq_restore(flags);
 864 }
 865
 866 /*
 867  * Re-enable interrupts on a counter after they were throttled
 868  * because they were coming too fast.
 869  */
 870 static void power_pmu_unthrottle(struct perf_counter *counter)
 871 {
 872         s64 val, left;
 873         unsigned long flags;
 874
 875         if (!counter->hw.idx || !counter->hw.sample_period)
 876                 return;
 877         local_irq_save(flags);
 878         perf_disable();
 879         power_pmu_read(counter);
 880         left = counter->hw.sample_period;
 881         counter->hw.last_period = left;
 882         val = 0;
 883         if (left < 0x80000000L)
 884                 val = 0x80000000L - left;
 885         write_pmc(counter->hw.idx, val);
 886         atomic64_set(&counter->hw.prev_count, val);
 887         atomic64_set(&counter->hw.period_left, left);
 888         perf_counter_update_userpage(counter);
 889         perf_enable();
 890         local_irq_restore(flags);
 891 }
 892
 893 struct pmu power_pmu = {
 894         .enable         = power_pmu_enable,
 895         .disable        = power_pmu_disable,
 896         .read           = power_pmu_read,
 897         .unthrottle     = power_pmu_unthrottle,
 898 };
 899
 900 /*
 901  * Return 1 if we might be able to put counter on a limited PMC,
 902  * or 0 if not.
 903  * A counter can only go on a limited PMC if it counts something
 904  * that a limited PMC can count, doesn't require interrupts, and
 905  * doesn't exclude any processor mode.
 906  */
 907 static int can_go_on_limited_pmc(struct perf_counter *counter, u64 ev,
 908                                  unsigned int flags)
 909 {
 910         int n;
 911         u64 alt[MAX_EVENT_ALTERNATIVES];
 912
 913         if (counter->attr.exclude_user
 914             || counter->attr.exclude_kernel
 915             || counter->attr.exclude_hv
 916             || counter->attr.sample_period)
 917                 return 0;
 918
 919         if (ppmu->limited_pmc_event(ev))
 920                 return 1;
 921
 922         /*
 923          * The requested event isn't on a limited PMC already;
 924          * see if any alternative code goes on a limited PMC.
 925          */
 926         if (!ppmu->get_alternatives)
 927                 return 0;
 928
 929         flags |= PPMU_LIMITED_PMC_OK | PPMU_LIMITED_PMC_REQD;
 930         n = ppmu->get_alternatives(ev, flags, alt);
 931
 932         return n > 0;
 933 }
 934
 935 /*
 936  * Find an alternative event that goes on a normal PMC, if possible,
 937  * and return the event code, or 0 if there is no such alternative.
 938  * (Note: event code 0 is "don't count" on all machines.)
 939  */
 940 static u64 normal_pmc_alternative(u64 ev, unsigned long flags)
 941 {
 942         u64 alt[MAX_EVENT_ALTERNATIVES];
 943         int n;
 944
 945         flags &= ~(PPMU_LIMITED_PMC_OK | PPMU_LIMITED_PMC_REQD);
 946         n = ppmu->get_alternatives(ev, flags, alt);
 947         if (!n)
 948                 return 0;
 949         return alt[0];
 950 }
 951
 952 /* Number of perf_counters counting hardware events */
 953 static atomic_t num_counters;
 954 /* Used to avoid races in calling reserve/release_pmc_hardware */
 955 static DEFINE_MUTEX(pmc_reserve_mutex);
 956
 957 /*
 958  * Release the PMU if this is the last perf_counter.
 959  */
 960 static void hw_perf_counter_destroy(struct perf_counter *counter)
 961 {
 962         if (!atomic_add_unless(&num_counters, -1, 1)) {
 963                 mutex_lock(&pmc_reserve_mutex);
 964                 if (atomic_dec_return(&num_counters) == 0)
 965                         release_pmc_hardware();
 966                 mutex_unlock(&pmc_reserve_mutex);
 967         }
 968 }
 969
 970 /*
 971  * Translate a generic cache event config to a raw event code.
 972  */
 973 static int hw_perf_cache_event(u64 config, u64 *eventp)
 974 {
 975         unsigned long type, op, result;
 976         int ev;
 977
 978         if (!ppmu->cache_events)
 979                 return -EINVAL;
 980
 981         /* unpack config */
 982         type = config & 0xff;
 983         op = (config >> 8) & 0xff;
 984         result = (config >> 16) & 0xff;
 985
 986         if (type >= PERF_COUNT_HW_CACHE_MAX ||
 987             op >= PERF_COUNT_HW_CACHE_OP_MAX ||
 988             result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
 989                 return -EINVAL;
 990
 991         ev = (*ppmu->cache_events)[type][op][result];
 992         if (ev == 0)
 993                 return -EOPNOTSUPP;
 994         if (ev == -1)
 995                 return -EINVAL;
 996         *eventp = ev;
 997         return 0;
 998 }
 999
1000 const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
1001 {
1002         u64 ev;
1003         unsigned long flags;
1004         struct perf_counter *ctrs[MAX_HWCOUNTERS];
1005         u64 events[MAX_HWCOUNTERS];
1006         unsigned int cflags[MAX_HWCOUNTERS];
1007         int n;
1008         int err;
1009
1010         if (!ppmu)
1011                 return ERR_PTR(-ENXIO);
1012         switch (counter->attr.type) {
1013         case PERF_TYPE_HARDWARE:
1014                 ev = counter->attr.config;
1015                 if (ev >= ppmu->n_generic || ppmu->generic_events[ev] == 0)
1016                         return ERR_PTR(-EOPNOTSUPP);
1017                 ev = ppmu->generic_events[ev];
1018                 break;
1019         case PERF_TYPE_HW_CACHE:
1020                 err = hw_perf_cache_event(counter->attr.config, &ev);
1021                 if (err)
1022                         return ERR_PTR(err);
1023                 break;
1024         case PERF_TYPE_RAW:
1025                 ev = counter->attr.config;
1026                 break;
1027         default:
1028                 return ERR_PTR(-EINVAL);
1029         }
1030         counter->hw.config_base = ev;
1031         counter->hw.idx = 0;
1032
1033         /*
1034          * If we are not running on a hypervisor, force the
1035          * exclude_hv bit to 0 so that we don't care what
1036          * the user set it to.
1037          */
1038         if (!firmware_has_feature(FW_FEATURE_LPAR))
1039                 counter->attr.exclude_hv = 0;
1040
1041         /*
1042          * If this is a per-task counter, then we can use
1043          * PM_RUN_* events interchangeably with their non RUN_*
1044          * equivalents, e.g. PM_RUN_CYC instead of PM_CYC.
1045          * XXX we should check if the task is an idle task.
1046          */
1047         flags = 0;
1048         if (counter->ctx->task)
1049                 flags |= PPMU_ONLY_COUNT_RUN;
1050
1051         /*
1052          * If this machine has limited counters, check whether this
1053          * event could go on a limited counter.
1054          */
1055         if (ppmu->flags & PPMU_LIMITED_PMC5_6) {
1056                 if (can_go_on_limited_pmc(counter, ev, flags)) {
1057                         flags |= PPMU_LIMITED_PMC_OK;
1058                 } else if (ppmu->limited_pmc_event(ev)) {
1059                         /*
1060                          * The requested event is on a limited PMC,
1061                          * but we can't use a limited PMC; see if any
1062                          * alternative goes on a normal PMC.
1063                          */
1064                         ev = normal_pmc_alternative(ev, flags);
1065                         if (!ev)
1066                                 return ERR_PTR(-EINVAL);
1067                 }
1068         }
1069
1070         /*
1071          * If this is in a group, check if it can go on with all the
1072          * other hardware counters in the group.  We assume the counter
1073          * hasn't been linked into its leader's sibling list at this point.
1074          */
1075         n = 0;
1076         if (counter->group_leader != counter) {
1077                 n = collect_events(counter->group_leader, ppmu->n_counter - 1,
1078                                    ctrs, events, cflags);
1079                 if (n < 0)
1080                         return ERR_PTR(-EINVAL);
1081         }
1082         events[n] = ev;
1083         ctrs[n] = counter;
1084         cflags[n] = flags;
1085         if (check_excludes(ctrs, cflags, n, 1))
1086                 return ERR_PTR(-EINVAL);
1087         if (power_check_constraints(events, cflags, n + 1))
1088                 return ERR_PTR(-EINVAL);
1089
1090         counter->hw.config = events[n];
1091         counter->hw.counter_base = cflags[n];
1092         counter->hw.last_period = counter->hw.sample_period;
1093         atomic64_set(&counter->hw.period_left, counter->hw.last_period);
1094
1095         /*
1096          * See if we need to reserve the PMU.
1097          * If no counters are currently in use, then we have to take a
1098          * mutex to ensure that we don't race with another task doing
1099          * reserve_pmc_hardware or release_pmc_hardware.
1100          */
1101         err = 0;
1102         if (!atomic_inc_not_zero(&num_counters)) {
1103                 mutex_lock(&pmc_reserve_mutex);
1104                 if (atomic_read(&num_counters) == 0 &&
1105                     reserve_pmc_hardware(perf_counter_interrupt))
1106                         err = -EBUSY;
1107                 else
1108                         atomic_inc(&num_counters);
1109                 mutex_unlock(&pmc_reserve_mutex);
1110         }
1111         counter->destroy = hw_perf_counter_destroy;
1112
1113         if (err)
1114                 return ERR_PTR(err);
1115         return &power_pmu;
1116 }
1117
1118 /*
1119  * A counter has overflowed; update its count and record
1120  * things if requested.  Note that interrupts are hard-disabled
1121  * here so there is no possibility of being interrupted.
1122  */
1123 static void record_and_restart(struct perf_counter *counter, unsigned long val,
1124                                struct pt_regs *regs, int nmi)
1125 {
1126         u64 period = counter->hw.sample_period;
1127         s64 prev, delta, left;
1128         int record = 0;
1129
1130         /* we don't have to worry about interrupts here */
1131         prev = atomic64_read(&counter->hw.prev_count);
1132         delta = (val - prev) & 0xfffffffful;
1133         atomic64_add(delta, &counter->count);
1134
1135         /*
1136          * See if the total period for this counter has expired,
1137          * and update for the next period.
1138          */
1139         val = 0;
1140         left = atomic64_read(&counter->hw.period_left) - delta;
1141         if (period) {
1142                 if (left <= 0) {
1143                         left += period;
1144                         if (left <= 0)
1145                                 left = period;
1146                         record = 1;
1147                 }
1148                 if (left < 0x80000000LL)
1149                         val = 0x80000000LL - left;
1150         }
1151
1152         /*
1153          * Finally record data if requested.
1154          */
1155         if (record) {
1156                 struct perf_sample_data data = {
1157                         .regs   = regs,
1158                         .addr   = 0,
1159                         .period = counter->hw.last_period,
1160                 };
1161
1162                 if (counter->attr.sample_type & PERF_SAMPLE_ADDR)
1163                         perf_get_data_addr(regs, &data.addr);
1164
1165                 if (perf_counter_overflow(counter, nmi, &data)) {
1166                         /*
1167                          * Interrupts are coming too fast - throttle them
1168                          * by setting the counter to 0, so it will be
1169                          * at least 2^30 cycles until the next interrupt
1170                          * (assuming each counter counts at most 2 counts
1171                          * per cycle).
1172                          */
1173                         val = 0;
1174                         left = ~0ULL >> 1;
1175                 }
1176         }
1177
1178         write_pmc(counter->hw.idx, val);
1179         atomic64_set(&counter->hw.prev_count, val);
1180         atomic64_set(&counter->hw.period_left, left);
1181         perf_counter_update_userpage(counter);
1182 }
1183
1184 /*
1185  * Called from generic code to get the misc flags (i.e. processor mode)
1186  * for an event.
1187  */
1188 unsigned long perf_misc_flags(struct pt_regs *regs)
1189 {
1190         u32 flags = perf_get_misc_flags(regs);
1191
1192         if (flags)
1193                 return flags;
1194         return user_mode(regs) ? PERF_EVENT_MISC_USER :
1195                 PERF_EVENT_MISC_KERNEL;
1196 }
1197
1198 /*
1199  * Called from generic code to get the instruction pointer
1200  * for an event.
1201  */
1202 unsigned long perf_instruction_pointer(struct pt_regs *regs)
1203 {
1204         unsigned long ip;
1205
1206         if (TRAP(regs) != 0xf00)
1207                 return regs->nip;       /* not a PMU interrupt */
1208
1209         ip = mfspr(SPRN_SIAR) + perf_ip_adjust(regs);
1210         return ip;
1211 }
1212
1213 /*
1214  * Performance monitor interrupt stuff
1215  */
1216 static void perf_counter_interrupt(struct pt_regs *regs)
1217 {
1218         int i;
1219         struct cpu_hw_counters *cpuhw = &__get_cpu_var(cpu_hw_counters);
1220         struct perf_counter *counter;
1221         unsigned long val;
1222         int found = 0;
1223         int nmi;
1224
1225         if (cpuhw->n_limited)
1226                 freeze_limited_counters(cpuhw, mfspr(SPRN_PMC5),
1227                                         mfspr(SPRN_PMC6));
1228
1229         perf_read_regs(regs);
1230
1231         nmi = perf_intr_is_nmi(regs);
1232         if (nmi)
1233                 nmi_enter();
1234         else
1235                 irq_enter();
1236
1237         for (i = 0; i < cpuhw->n_counters; ++i) {
1238                 counter = cpuhw->counter[i];
1239                 if (!counter->hw.idx || is_limited_pmc(counter->hw.idx))
1240                         continue;
1241                 val = read_pmc(counter->hw.idx);
1242                 if ((int)val < 0) {
1243                         /* counter has overflowed */
1244                         found = 1;
1245                         record_and_restart(counter, val, regs, nmi);
1246                 }
1247         }
1248
1249         /*
1250          * In case we didn't find and reset the counter that caused
1251          * the interrupt, scan all counters and reset any that are
1252          * negative, to avoid getting continual interrupts.
1253          * Any that we processed in the previous loop will not be negative.
1254          */
1255         if (!found) {
1256                 for (i = 0; i < ppmu->n_counter; ++i) {
1257                         if (is_limited_pmc(i + 1))
1258                                 continue;
1259                         val = read_pmc(i + 1);
1260                         if ((int)val < 0)
1261                                 write_pmc(i + 1, 0);
1262                 }
1263         }
1264
1265         /*
1266          * Reset MMCR0 to its normal value.  This will set PMXE and
1267          * clear FC (freeze counters) and PMAO (perf mon alert occurred)
1268          * and thus allow interrupts to occur again.
1269          * XXX might want to use MSR.PM to keep the counters frozen until
1270          * we get back out of this interrupt.
1271          */
1272         write_mmcr0(cpuhw, cpuhw->mmcr[0]);
1273
1274         if (nmi)
1275                 nmi_exit();
1276         else
1277                 irq_exit();
1278 }
1279
1280 void hw_perf_counter_setup(int cpu)
1281 {
1282         struct cpu_hw_counters *cpuhw = &per_cpu(cpu_hw_counters, cpu);
1283
1284         memset(cpuhw, 0, sizeof(*cpuhw));
1285         cpuhw->mmcr[0] = MMCR0_FC;
1286 }
1287
1288 int register_power_pmu(struct power_pmu *pmu)
1289 {
1290         if (ppmu)
1291                 return -EBUSY;          /* something's already registered */
1292
1293         ppmu = pmu;
1294         pr_info("%s performance monitor hardware support registered\n",
1295                 pmu->name);
1296
1297 #ifdef MSR_HV
1298         /*
1299          * Use FCHV to ignore kernel events if MSR.HV is set.
1300          */
1301         if (mfmsr() & MSR_HV)
1302                 freeze_counters_kernel = MMCR0_FCHV;
1303 #endif /* CONFIG_PPC64 */
1304
1305         return 0;
1306 }