kernel/perf_counter.c

   1 /*
   2  * Performance counter core code
   3  *
   4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
   6  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
   7  *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
   8  *
   9  *  For licensing details see kernel-base/COPYING
  10  */
  11
  12 #include <linux/fs.h>
  13 #include <linux/mm.h>
  14 #include <linux/cpu.h>
  15 #include <linux/smp.h>
  16 #include <linux/file.h>
  17 #include <linux/poll.h>
  18 #include <linux/sysfs.h>
  19 #include <linux/ptrace.h>
  20 #include <linux/percpu.h>
  21 #include <linux/vmstat.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/rculist.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/syscalls.h>
  26 #include <linux/anon_inodes.h>
  27 #include <linux/kernel_stat.h>
  28 #include <linux/perf_counter.h>
  29 #include <linux/dcache.h>
  30
  31 #include <asm/irq_regs.h>
  32
  33 /*
  34  * Each CPU has a list of per CPU counters:
  35  */
  36 DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
  37
  38 int perf_max_counters __read_mostly = 1;
  39 static int perf_reserved_percpu __read_mostly;
  40 static int perf_overcommit __read_mostly = 1;
  41
  42 static atomic_t nr_counters __read_mostly;
  43 static atomic_t nr_mmap_tracking __read_mostly;
  44 static atomic_t nr_munmap_tracking __read_mostly;
  45 static atomic_t nr_comm_tracking __read_mostly;
  46
  47 int sysctl_perf_counter_priv __read_mostly; /* do we need to be privileged */
  48 int sysctl_perf_counter_mlock __read_mostly = 128; /* 'free' kb per counter */
  49
  50 /*
  51  * Lock for (sysadmin-configurable) counter reservations:
  52  */
  53 static DEFINE_SPINLOCK(perf_resource_lock);
  54
  55 /*
  56  * Architecture provided APIs - weak aliases:
  57  */
  58 extern __weak const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
  59 {
  60         return NULL;
  61 }
  62
  63 u64 __weak hw_perf_save_disable(void)           { return 0; }
  64 void __weak hw_perf_restore(u64 ctrl)           { barrier(); }
  65 void __weak hw_perf_counter_setup(int cpu)      { barrier(); }
  66 int __weak hw_perf_group_sched_in(struct perf_counter *group_leader,
  67                struct perf_cpu_context *cpuctx,
  68                struct perf_counter_context *ctx, int cpu)
  69 {
  70         return 0;
  71 }
  72
  73 void __weak perf_counter_print_debug(void)      { }
  74
  75 static void
  76 list_add_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
  77 {
  78         struct perf_counter *group_leader = counter->group_leader;
  79
  80         /*
  81          * Depending on whether it is a standalone or sibling counter,
  82          * add it straight to the context's counter list, or to the group
  83          * leader's sibling list:
  84          */
  85         if (group_leader == counter)
  86                 list_add_tail(&counter->list_entry, &ctx->counter_list);
  87         else {
  88                 list_add_tail(&counter->list_entry, &group_leader->sibling_list);
  89                 group_leader->nr_siblings++;
  90         }
  91
  92         list_add_rcu(&counter->event_entry, &ctx->event_list);
  93 }
  94
  95 static void
  96 list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
  97 {
  98         struct perf_counter *sibling, *tmp;
  99
 100         list_del_init(&counter->list_entry);
 101         list_del_rcu(&counter->event_entry);
 102
 103         if (counter->group_leader != counter)
 104                 counter->group_leader->nr_siblings--;
 105
 106         /*
 107          * If this was a group counter with sibling counters then
 108          * upgrade the siblings to singleton counters by adding them
 109          * to the context list directly:
 110          */
 111         list_for_each_entry_safe(sibling, tmp,
 112                                  &counter->sibling_list, list_entry) {
 113
 114                 list_move_tail(&sibling->list_entry, &ctx->counter_list);
 115                 sibling->group_leader = sibling;
 116         }
 117 }
 118
 119 static void
 120 counter_sched_out(struct perf_counter *counter,
 121                   struct perf_cpu_context *cpuctx,
 122                   struct perf_counter_context *ctx)
 123 {
 124         if (counter->state != PERF_COUNTER_STATE_ACTIVE)
 125                 return;
 126
 127         counter->state = PERF_COUNTER_STATE_INACTIVE;
 128         counter->tstamp_stopped = ctx->time;
 129         counter->pmu->disable(counter);
 130         counter->oncpu = -1;
 131
 132         if (!is_software_counter(counter))
 133                 cpuctx->active_oncpu--;
 134         ctx->nr_active--;
 135         if (counter->hw_event.exclusive || !cpuctx->active_oncpu)
 136                 cpuctx->exclusive = 0;
 137 }
 138
 139 static void
 140 group_sched_out(struct perf_counter *group_counter,
 141                 struct perf_cpu_context *cpuctx,
 142                 struct perf_counter_context *ctx)
 143 {
 144         struct perf_counter *counter;
 145
 146         if (group_counter->state != PERF_COUNTER_STATE_ACTIVE)
 147                 return;
 148
 149         counter_sched_out(group_counter, cpuctx, ctx);
 150
 151         /*
 152          * Schedule out siblings (if any):
 153          */
 154         list_for_each_entry(counter, &group_counter->sibling_list, list_entry)
 155                 counter_sched_out(counter, cpuctx, ctx);
 156
 157         if (group_counter->hw_event.exclusive)
 158                 cpuctx->exclusive = 0;
 159 }
 160
 161 /*
 162  * Cross CPU call to remove a performance counter
 163  *
 164  * We disable the counter on the hardware level first. After that we
 165  * remove it from the context list.
 166  */
 167 static void __perf_counter_remove_from_context(void *info)
 168 {
 169         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 170         struct perf_counter *counter = info;
 171         struct perf_counter_context *ctx = counter->ctx;
 172         unsigned long flags;
 173         u64 perf_flags;
 174
 175         /*
 176          * If this is a task context, we need to check whether it is
 177          * the current task context of this cpu. If not it has been
 178          * scheduled out before the smp call arrived.
 179          */
 180         if (ctx->task && cpuctx->task_ctx != ctx)
 181                 return;
 182
 183         spin_lock_irqsave(&ctx->lock, flags);
 184
 185         counter_sched_out(counter, cpuctx, ctx);
 186
 187         counter->task = NULL;
 188         ctx->nr_counters--;
 189
 190         /*
 191          * Protect the list operation against NMI by disabling the
 192          * counters on a global level. NOP for non NMI based counters.
 193          */
 194         perf_flags = hw_perf_save_disable();
 195         list_del_counter(counter, ctx);
 196         hw_perf_restore(perf_flags);
 197
 198         if (!ctx->task) {
 199                 /*
 200                  * Allow more per task counters with respect to the
 201                  * reservation:
 202                  */
 203                 cpuctx->max_pertask =
 204                         min(perf_max_counters - ctx->nr_counters,
 205                             perf_max_counters - perf_reserved_percpu);
 206         }
 207
 208         spin_unlock_irqrestore(&ctx->lock, flags);
 209 }
 210
 211
 212 /*
 213  * Remove the counter from a task's (or a CPU's) list of counters.
 214  *
 215  * Must be called with counter->mutex and ctx->mutex held.
 216  *
 217  * CPU counters are removed with a smp call. For task counters we only
 218  * call when the task is on a CPU.
 219  */
 220 static void perf_counter_remove_from_context(struct perf_counter *counter)
 221 {
 222         struct perf_counter_context *ctx = counter->ctx;
 223         struct task_struct *task = ctx->task;
 224
 225         if (!task) {
 226                 /*
 227                  * Per cpu counters are removed via an smp call and
 228                  * the removal is always sucessful.
 229                  */
 230                 smp_call_function_single(counter->cpu,
 231                                          __perf_counter_remove_from_context,
 232                                          counter, 1);
 233                 return;
 234         }
 235
 236 retry:
 237         task_oncpu_function_call(task, __perf_counter_remove_from_context,
 238                                  counter);
 239
 240         spin_lock_irq(&ctx->lock);
 241         /*
 242          * If the context is active we need to retry the smp call.
 243          */
 244         if (ctx->nr_active && !list_empty(&counter->list_entry)) {
 245                 spin_unlock_irq(&ctx->lock);
 246                 goto retry;
 247         }
 248
 249         /*
 250          * The lock prevents that this context is scheduled in so we
 251          * can remove the counter safely, if the call above did not
 252          * succeed.
 253          */
 254         if (!list_empty(&counter->list_entry)) {
 255                 ctx->nr_counters--;
 256                 list_del_counter(counter, ctx);
 257                 counter->task = NULL;
 258         }
 259         spin_unlock_irq(&ctx->lock);
 260 }
 261
 262 static inline u64 perf_clock(void)
 263 {
 264         return cpu_clock(smp_processor_id());
 265 }
 266
 267 /*
 268  * Update the record of the current time in a context.
 269  */
 270 static void update_context_time(struct perf_counter_context *ctx)
 271 {
 272         u64 now = perf_clock();
 273
 274         ctx->time += now - ctx->timestamp;
 275         ctx->timestamp = now;
 276 }
 277
 278 /*
 279  * Update the total_time_enabled and total_time_running fields for a counter.
 280  */
 281 static void update_counter_times(struct perf_counter *counter)
 282 {
 283         struct perf_counter_context *ctx = counter->ctx;
 284         u64 run_end;
 285
 286         if (counter->state < PERF_COUNTER_STATE_INACTIVE)
 287                 return;
 288
 289         counter->total_time_enabled = ctx->time - counter->tstamp_enabled;
 290
 291         if (counter->state == PERF_COUNTER_STATE_INACTIVE)
 292                 run_end = counter->tstamp_stopped;
 293         else
 294                 run_end = ctx->time;
 295
 296         counter->total_time_running = run_end - counter->tstamp_running;
 297 }
 298
 299 /*
 300  * Update total_time_enabled and total_time_running for all counters in a group.
 301  */
 302 static void update_group_times(struct perf_counter *leader)
 303 {
 304         struct perf_counter *counter;
 305
 306         update_counter_times(leader);
 307         list_for_each_entry(counter, &leader->sibling_list, list_entry)
 308                 update_counter_times(counter);
 309 }
 310
 311 /*
 312  * Cross CPU call to disable a performance counter
 313  */
 314 static void __perf_counter_disable(void *info)
 315 {
 316         struct perf_counter *counter = info;
 317         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 318         struct perf_counter_context *ctx = counter->ctx;
 319         unsigned long flags;
 320
 321         /*
 322          * If this is a per-task counter, need to check whether this
 323          * counter's task is the current task on this cpu.
 324          */
 325         if (ctx->task && cpuctx->task_ctx != ctx)
 326                 return;
 327
 328         spin_lock_irqsave(&ctx->lock, flags);
 329
 330         /*
 331          * If the counter is on, turn it off.
 332          * If it is in error state, leave it in error state.
 333          */
 334         if (counter->state >= PERF_COUNTER_STATE_INACTIVE) {
 335                 update_context_time(ctx);
 336                 update_counter_times(counter);
 337                 if (counter == counter->group_leader)
 338                         group_sched_out(counter, cpuctx, ctx);
 339                 else
 340                         counter_sched_out(counter, cpuctx, ctx);
 341                 counter->state = PERF_COUNTER_STATE_OFF;
 342         }
 343
 344         spin_unlock_irqrestore(&ctx->lock, flags);
 345 }
 346
 347 /*
 348  * Disable a counter.
 349  */
 350 static void perf_counter_disable(struct perf_counter *counter)
 351 {
 352         struct perf_counter_context *ctx = counter->ctx;
 353         struct task_struct *task = ctx->task;
 354
 355         if (!task) {
 356                 /*
 357                  * Disable the counter on the cpu that it's on
 358                  */
 359                 smp_call_function_single(counter->cpu, __perf_counter_disable,
 360                                          counter, 1);
 361                 return;
 362         }
 363
 364  retry:
 365         task_oncpu_function_call(task, __perf_counter_disable, counter);
 366
 367         spin_lock_irq(&ctx->lock);
 368         /*
 369          * If the counter is still active, we need to retry the cross-call.
 370          */
 371         if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
 372                 spin_unlock_irq(&ctx->lock);
 373                 goto retry;
 374         }
 375
 376         /*
 377          * Since we have the lock this context can't be scheduled
 378          * in, so we can change the state safely.
 379          */
 380         if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
 381                 update_counter_times(counter);
 382                 counter->state = PERF_COUNTER_STATE_OFF;
 383         }
 384
 385         spin_unlock_irq(&ctx->lock);
 386 }
 387
 388 static int
 389 counter_sched_in(struct perf_counter *counter,
 390                  struct perf_cpu_context *cpuctx,
 391                  struct perf_counter_context *ctx,
 392                  int cpu)
 393 {
 394         if (counter->state <= PERF_COUNTER_STATE_OFF)
 395                 return 0;
 396
 397         counter->state = PERF_COUNTER_STATE_ACTIVE;
 398         counter->oncpu = cpu;   /* TODO: put 'cpu' into cpuctx->cpu */
 399         /*
 400          * The new state must be visible before we turn it on in the hardware:
 401          */
 402         smp_wmb();
 403
 404         if (counter->pmu->enable(counter)) {
 405                 counter->state = PERF_COUNTER_STATE_INACTIVE;
 406                 counter->oncpu = -1;
 407                 return -EAGAIN;
 408         }
 409
 410         counter->tstamp_running += ctx->time - counter->tstamp_stopped;
 411
 412         if (!is_software_counter(counter))
 413                 cpuctx->active_oncpu++;
 414         ctx->nr_active++;
 415
 416         if (counter->hw_event.exclusive)
 417                 cpuctx->exclusive = 1;
 418
 419         return 0;
 420 }
 421
 422 static int
 423 group_sched_in(struct perf_counter *group_counter,
 424                struct perf_cpu_context *cpuctx,
 425                struct perf_counter_context *ctx,
 426                int cpu)
 427 {
 428         struct perf_counter *counter, *partial_group;
 429         int ret;
 430
 431         if (group_counter->state == PERF_COUNTER_STATE_OFF)
 432                 return 0;
 433
 434         ret = hw_perf_group_sched_in(group_counter, cpuctx, ctx, cpu);
 435         if (ret)
 436                 return ret < 0 ? ret : 0;
 437
 438         group_counter->prev_state = group_counter->state;
 439         if (counter_sched_in(group_counter, cpuctx, ctx, cpu))
 440                 return -EAGAIN;
 441
 442         /*
 443          * Schedule in siblings as one group (if any):
 444          */
 445         list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
 446                 counter->prev_state = counter->state;
 447                 if (counter_sched_in(counter, cpuctx, ctx, cpu)) {
 448                         partial_group = counter;
 449                         goto group_error;
 450                 }
 451         }
 452
 453         return 0;
 454
 455 group_error:
 456         /*
 457          * Groups can be scheduled in as one unit only, so undo any
 458          * partial group before returning:
 459          */
 460         list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
 461                 if (counter == partial_group)
 462                         break;
 463                 counter_sched_out(counter, cpuctx, ctx);
 464         }
 465         counter_sched_out(group_counter, cpuctx, ctx);
 466
 467         return -EAGAIN;
 468 }
 469
 470 /*
 471  * Return 1 for a group consisting entirely of software counters,
 472  * 0 if the group contains any hardware counters.
 473  */
 474 static int is_software_only_group(struct perf_counter *leader)
 475 {
 476         struct perf_counter *counter;
 477
 478         if (!is_software_counter(leader))
 479                 return 0;
 480
 481         list_for_each_entry(counter, &leader->sibling_list, list_entry)
 482                 if (!is_software_counter(counter))
 483                         return 0;
 484
 485         return 1;
 486 }
 487
 488 /*
 489  * Work out whether we can put this counter group on the CPU now.
 490  */
 491 static int group_can_go_on(struct perf_counter *counter,
 492                            struct perf_cpu_context *cpuctx,
 493                            int can_add_hw)
 494 {
 495         /*
 496          * Groups consisting entirely of software counters can always go on.
 497          */
 498         if (is_software_only_group(counter))
 499                 return 1;
 500         /*
 501          * If an exclusive group is already on, no other hardware
 502          * counters can go on.
 503          */
 504         if (cpuctx->exclusive)
 505                 return 0;
 506         /*
 507          * If this group is exclusive and there are already
 508          * counters on the CPU, it can't go on.
 509          */
 510         if (counter->hw_event.exclusive && cpuctx->active_oncpu)
 511                 return 0;
 512         /*
 513          * Otherwise, try to add it if all previous groups were able
 514          * to go on.
 515          */
 516         return can_add_hw;
 517 }
 518
 519 static void add_counter_to_ctx(struct perf_counter *counter,
 520                                struct perf_counter_context *ctx)
 521 {
 522         list_add_counter(counter, ctx);
 523         ctx->nr_counters++;
 524         counter->prev_state = PERF_COUNTER_STATE_OFF;
 525         counter->tstamp_enabled = ctx->time;
 526         counter->tstamp_running = ctx->time;
 527         counter->tstamp_stopped = ctx->time;
 528 }
 529
 530 /*
 531  * Cross CPU call to install and enable a performance counter
 532  */
 533 static void __perf_install_in_context(void *info)
 534 {
 535         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 536         struct perf_counter *counter = info;
 537         struct perf_counter_context *ctx = counter->ctx;
 538         struct perf_counter *leader = counter->group_leader;
 539         int cpu = smp_processor_id();
 540         unsigned long flags;
 541         u64 perf_flags;
 542         int err;
 543
 544         /*
 545          * If this is a task context, we need to check whether it is
 546          * the current task context of this cpu. If not it has been
 547          * scheduled out before the smp call arrived.
 548          */
 549         if (ctx->task && cpuctx->task_ctx != ctx)
 550                 return;
 551
 552         spin_lock_irqsave(&ctx->lock, flags);
 553         update_context_time(ctx);
 554
 555         /*
 556          * Protect the list operation against NMI by disabling the
 557          * counters on a global level. NOP for non NMI based counters.
 558          */
 559         perf_flags = hw_perf_save_disable();
 560
 561         add_counter_to_ctx(counter, ctx);
 562
 563         /*
 564          * Don't put the counter on if it is disabled or if
 565          * it is in a group and the group isn't on.
 566          */
 567         if (counter->state != PERF_COUNTER_STATE_INACTIVE ||
 568             (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE))
 569                 goto unlock;
 570
 571         /*
 572          * An exclusive counter can't go on if there are already active
 573          * hardware counters, and no hardware counter can go on if there
 574          * is already an exclusive counter on.
 575          */
 576         if (!group_can_go_on(counter, cpuctx, 1))
 577                 err = -EEXIST;
 578         else
 579                 err = counter_sched_in(counter, cpuctx, ctx, cpu);
 580
 581         if (err) {
 582                 /*
 583                  * This counter couldn't go on.  If it is in a group
 584                  * then we have to pull the whole group off.
 585                  * If the counter group is pinned then put it in error state.
 586                  */
 587                 if (leader != counter)
 588                         group_sched_out(leader, cpuctx, ctx);
 589                 if (leader->hw_event.pinned) {
 590                         update_group_times(leader);
 591                         leader->state = PERF_COUNTER_STATE_ERROR;
 592                 }
 593         }
 594
 595         if (!err && !ctx->task && cpuctx->max_pertask)
 596                 cpuctx->max_pertask--;
 597
 598  unlock:
 599         hw_perf_restore(perf_flags);
 600
 601         spin_unlock_irqrestore(&ctx->lock, flags);
 602 }
 603
 604 /*
 605  * Attach a performance counter to a context
 606  *
 607  * First we add the counter to the list with the hardware enable bit
 608  * in counter->hw_config cleared.
 609  *
 610  * If the counter is attached to a task which is on a CPU we use a smp
 611  * call to enable it in the task context. The task might have been
 612  * scheduled away, but we check this in the smp call again.
 613  *
 614  * Must be called with ctx->mutex held.
 615  */
 616 static void
 617 perf_install_in_context(struct perf_counter_context *ctx,
 618                         struct perf_counter *counter,
 619                         int cpu)
 620 {
 621         struct task_struct *task = ctx->task;
 622
 623         if (!task) {
 624                 /*
 625                  * Per cpu counters are installed via an smp call and
 626                  * the install is always sucessful.
 627                  */
 628                 smp_call_function_single(cpu, __perf_install_in_context,
 629                                          counter, 1);
 630                 return;
 631         }
 632
 633         counter->task = task;
 634 retry:
 635         task_oncpu_function_call(task, __perf_install_in_context,
 636                                  counter);
 637
 638         spin_lock_irq(&ctx->lock);
 639         /*
 640          * we need to retry the smp call.
 641          */
 642         if (ctx->is_active && list_empty(&counter->list_entry)) {
 643                 spin_unlock_irq(&ctx->lock);
 644                 goto retry;
 645         }
 646
 647         /*
 648          * The lock prevents that this context is scheduled in so we
 649          * can add the counter safely, if it the call above did not
 650          * succeed.
 651          */
 652         if (list_empty(&counter->list_entry))
 653                 add_counter_to_ctx(counter, ctx);
 654         spin_unlock_irq(&ctx->lock);
 655 }
 656
 657 /*
 658  * Cross CPU call to enable a performance counter
 659  */
 660 static void __perf_counter_enable(void *info)
 661 {
 662         struct perf_counter *counter = info;
 663         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 664         struct perf_counter_context *ctx = counter->ctx;
 665         struct perf_counter *leader = counter->group_leader;
 666         unsigned long flags;
 667         int err;
 668
 669         /*
 670          * If this is a per-task counter, need to check whether this
 671          * counter's task is the current task on this cpu.
 672          */
 673         if (ctx->task && cpuctx->task_ctx != ctx)
 674                 return;
 675
 676         spin_lock_irqsave(&ctx->lock, flags);
 677         update_context_time(ctx);
 678
 679         counter->prev_state = counter->state;
 680         if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
 681                 goto unlock;
 682         counter->state = PERF_COUNTER_STATE_INACTIVE;
 683         counter->tstamp_enabled = ctx->time - counter->total_time_enabled;
 684
 685         /*
 686          * If the counter is in a group and isn't the group leader,
 687          * then don't put it on unless the group is on.
 688          */
 689         if (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE)
 690                 goto unlock;
 691
 692         if (!group_can_go_on(counter, cpuctx, 1))
 693                 err = -EEXIST;
 694         else if (counter == leader)
 695                 err = group_sched_in(counter, cpuctx, ctx,
 696                                      smp_processor_id());
 697         else
 698                 err = counter_sched_in(counter, cpuctx, ctx,
 699                                        smp_processor_id());
 700
 701         if (err) {
 702                 /*
 703                  * If this counter can't go on and it's part of a
 704                  * group, then the whole group has to come off.
 705                  */
 706                 if (leader != counter)
 707                         group_sched_out(leader, cpuctx, ctx);
 708                 if (leader->hw_event.pinned) {
 709                         update_group_times(leader);
 710                         leader->state = PERF_COUNTER_STATE_ERROR;
 711                 }
 712         }
 713
 714  unlock:
 715         spin_unlock_irqrestore(&ctx->lock, flags);
 716 }
 717
 718 /*
 719  * Enable a counter.
 720  */
 721 static void perf_counter_enable(struct perf_counter *counter)
 722 {
 723         struct perf_counter_context *ctx = counter->ctx;
 724         struct task_struct *task = ctx->task;
 725
 726         if (!task) {
 727                 /*
 728                  * Enable the counter on the cpu that it's on
 729                  */
 730                 smp_call_function_single(counter->cpu, __perf_counter_enable,
 731                                          counter, 1);
 732                 return;
 733         }
 734
 735         spin_lock_irq(&ctx->lock);
 736         if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
 737                 goto out;
 738
 739         /*
 740          * If the counter is in error state, clear that first.
 741          * That way, if we see the counter in error state below, we
 742          * know that it has gone back into error state, as distinct
 743          * from the task having been scheduled away before the
 744          * cross-call arrived.
 745          */
 746         if (counter->state == PERF_COUNTER_STATE_ERROR)
 747                 counter->state = PERF_COUNTER_STATE_OFF;
 748
 749  retry:
 750         spin_unlock_irq(&ctx->lock);
 751         task_oncpu_function_call(task, __perf_counter_enable, counter);
 752
 753         spin_lock_irq(&ctx->lock);
 754
 755         /*
 756          * If the context is active and the counter is still off,
 757          * we need to retry the cross-call.
 758          */
 759         if (ctx->is_active && counter->state == PERF_COUNTER_STATE_OFF)
 760                 goto retry;
 761
 762         /*
 763          * Since we have the lock this context can't be scheduled
 764          * in, so we can change the state safely.
 765          */
 766         if (counter->state == PERF_COUNTER_STATE_OFF) {
 767                 counter->state = PERF_COUNTER_STATE_INACTIVE;
 768                 counter->tstamp_enabled =
 769                         ctx->time - counter->total_time_enabled;
 770         }
 771  out:
 772         spin_unlock_irq(&ctx->lock);
 773 }
 774
 775 static int perf_counter_refresh(struct perf_counter *counter, int refresh)
 776 {
 777         /*
 778          * not supported on inherited counters
 779          */
 780         if (counter->hw_event.inherit)
 781                 return -EINVAL;
 782
 783         atomic_add(refresh, &counter->event_limit);
 784         perf_counter_enable(counter);
 785
 786         return 0;
 787 }
 788
 789 void __perf_counter_sched_out(struct perf_counter_context *ctx,
 790                               struct perf_cpu_context *cpuctx)
 791 {
 792         struct perf_counter *counter;
 793         u64 flags;
 794
 795         spin_lock(&ctx->lock);
 796         ctx->is_active = 0;
 797         if (likely(!ctx->nr_counters))
 798                 goto out;
 799         update_context_time(ctx);
 800
 801         flags = hw_perf_save_disable();
 802         if (ctx->nr_active) {
 803                 list_for_each_entry(counter, &ctx->counter_list, list_entry)
 804                         group_sched_out(counter, cpuctx, ctx);
 805         }
 806         hw_perf_restore(flags);
 807  out:
 808         spin_unlock(&ctx->lock);
 809 }
 810
 811 /*
 812  * Called from scheduler to remove the counters of the current task,
 813  * with interrupts disabled.
 814  *
 815  * We stop each counter and update the counter value in counter->count.
 816  *
 817  * This does not protect us against NMI, but disable()
 818  * sets the disabled bit in the control field of counter _before_
 819  * accessing the counter control register. If a NMI hits, then it will
 820  * not restart the counter.
 821  */
 822 void perf_counter_task_sched_out(struct task_struct *task, int cpu)
 823 {
 824         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
 825         struct perf_counter_context *ctx = &task->perf_counter_ctx;
 826         struct pt_regs *regs;
 827
 828         if (likely(!cpuctx->task_ctx))
 829                 return;
 830
 831         update_context_time(ctx);
 832
 833         regs = task_pt_regs(task);
 834         perf_swcounter_event(PERF_COUNT_CONTEXT_SWITCHES, 1, 1, regs, 0);
 835         __perf_counter_sched_out(ctx, cpuctx);
 836
 837         cpuctx->task_ctx = NULL;
 838 }
 839
 840 static void perf_counter_cpu_sched_out(struct perf_cpu_context *cpuctx)
 841 {
 842         __perf_counter_sched_out(&cpuctx->ctx, cpuctx);
 843 }
 844
 845 static void
 846 __perf_counter_sched_in(struct perf_counter_context *ctx,
 847                         struct perf_cpu_context *cpuctx, int cpu)
 848 {
 849         struct perf_counter *counter;
 850         u64 flags;
 851         int can_add_hw = 1;
 852
 853         spin_lock(&ctx->lock);
 854         ctx->is_active = 1;
 855         if (likely(!ctx->nr_counters))
 856                 goto out;
 857
 858         ctx->timestamp = perf_clock();
 859
 860         flags = hw_perf_save_disable();
 861
 862         /*
 863          * First go through the list and put on any pinned groups
 864          * in order to give them the best chance of going on.
 865          */
 866         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
 867                 if (counter->state <= PERF_COUNTER_STATE_OFF ||
 868                     !counter->hw_event.pinned)
 869                         continue;
 870                 if (counter->cpu != -1 && counter->cpu != cpu)
 871                         continue;
 872
 873                 if (group_can_go_on(counter, cpuctx, 1))
 874                         group_sched_in(counter, cpuctx, ctx, cpu);
 875
 876                 /*
 877                  * If this pinned group hasn't been scheduled,
 878                  * put it in error state.
 879                  */
 880                 if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
 881                         update_group_times(counter);
 882                         counter->state = PERF_COUNTER_STATE_ERROR;
 883                 }
 884         }
 885
 886         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
 887                 /*
 888                  * Ignore counters in OFF or ERROR state, and
 889                  * ignore pinned counters since we did them already.
 890                  */
 891                 if (counter->state <= PERF_COUNTER_STATE_OFF ||
 892                     counter->hw_event.pinned)
 893                         continue;
 894
 895                 /*
 896                  * Listen to the 'cpu' scheduling filter constraint
 897                  * of counters:
 898                  */
 899                 if (counter->cpu != -1 && counter->cpu != cpu)
 900                         continue;
 901
 902                 if (group_can_go_on(counter, cpuctx, can_add_hw)) {
 903                         if (group_sched_in(counter, cpuctx, ctx, cpu))
 904                                 can_add_hw = 0;
 905                 }
 906         }
 907         hw_perf_restore(flags);
 908  out:
 909         spin_unlock(&ctx->lock);
 910 }
 911
 912 /*
 913  * Called from scheduler to add the counters of the current task
 914  * with interrupts disabled.
 915  *
 916  * We restore the counter value and then enable it.
 917  *
 918  * This does not protect us against NMI, but enable()
 919  * sets the enabled bit in the control field of counter _before_
 920  * accessing the counter control register. If a NMI hits, then it will
 921  * keep the counter running.
 922  */
 923 void perf_counter_task_sched_in(struct task_struct *task, int cpu)
 924 {
 925         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
 926         struct perf_counter_context *ctx = &task->perf_counter_ctx;
 927
 928         __perf_counter_sched_in(ctx, cpuctx, cpu);
 929         cpuctx->task_ctx = ctx;
 930 }
 931
 932 static void perf_counter_cpu_sched_in(struct perf_cpu_context *cpuctx, int cpu)
 933 {
 934         struct perf_counter_context *ctx = &cpuctx->ctx;
 935
 936         __perf_counter_sched_in(ctx, cpuctx, cpu);
 937 }
 938
 939 int perf_counter_task_disable(void)
 940 {
 941         struct task_struct *curr = current;
 942         struct perf_counter_context *ctx = &curr->perf_counter_ctx;
 943         struct perf_counter *counter;
 944         unsigned long flags;
 945         u64 perf_flags;
 946         int cpu;
 947
 948         if (likely(!ctx->nr_counters))
 949                 return 0;
 950
 951         local_irq_save(flags);
 952         cpu = smp_processor_id();
 953
 954         perf_counter_task_sched_out(curr, cpu);
 955
 956         spin_lock(&ctx->lock);
 957
 958         /*
 959          * Disable all the counters:
 960          */
 961         perf_flags = hw_perf_save_disable();
 962
 963         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
 964                 if (counter->state != PERF_COUNTER_STATE_ERROR) {
 965                         update_group_times(counter);
 966                         counter->state = PERF_COUNTER_STATE_OFF;
 967                 }
 968         }
 969
 970         hw_perf_restore(perf_flags);
 971
 972         spin_unlock_irqrestore(&ctx->lock, flags);
 973
 974         return 0;
 975 }
 976
 977 int perf_counter_task_enable(void)
 978 {
 979         struct task_struct *curr = current;
 980         struct perf_counter_context *ctx = &curr->perf_counter_ctx;
 981         struct perf_counter *counter;
 982         unsigned long flags;
 983         u64 perf_flags;
 984         int cpu;
 985
 986         if (likely(!ctx->nr_counters))
 987                 return 0;
 988
 989         local_irq_save(flags);
 990         cpu = smp_processor_id();
 991
 992         perf_counter_task_sched_out(curr, cpu);
 993
 994         spin_lock(&ctx->lock);
 995
 996         /*
 997          * Disable all the counters:
 998          */
 999         perf_flags = hw_perf_save_disable();
1000
1001         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1002                 if (counter->state > PERF_COUNTER_STATE_OFF)
1003                         continue;
1004                 counter->state = PERF_COUNTER_STATE_INACTIVE;
1005                 counter->tstamp_enabled =
1006                         ctx->time - counter->total_time_enabled;
1007                 counter->hw_event.disabled = 0;
1008         }
1009         hw_perf_restore(perf_flags);
1010
1011         spin_unlock(&ctx->lock);
1012
1013         perf_counter_task_sched_in(curr, cpu);
1014
1015         local_irq_restore(flags);
1016
1017         return 0;
1018 }
1019
1020 /*
1021  * Round-robin a context's counters:
1022  */
1023 static void rotate_ctx(struct perf_counter_context *ctx)
1024 {
1025         struct perf_counter *counter;
1026         u64 perf_flags;
1027
1028         if (!ctx->nr_counters)
1029                 return;
1030
1031         spin_lock(&ctx->lock);
1032         /*
1033          * Rotate the first entry last (works just fine for group counters too):
1034          */
1035         perf_flags = hw_perf_save_disable();
1036         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1037                 list_move_tail(&counter->list_entry, &ctx->counter_list);
1038                 break;
1039         }
1040         hw_perf_restore(perf_flags);
1041
1042         spin_unlock(&ctx->lock);
1043 }
1044
1045 void perf_counter_task_tick(struct task_struct *curr, int cpu)
1046 {
1047         struct perf_cpu_context *cpuctx;
1048         struct perf_counter_context *ctx;
1049
1050         if (!atomic_read(&nr_counters))
1051                 return;
1052
1053         cpuctx = &per_cpu(perf_cpu_context, cpu);
1054         ctx = &curr->perf_counter_ctx;
1055
1056         perf_counter_cpu_sched_out(cpuctx);
1057         perf_counter_task_sched_out(curr, cpu);
1058
1059         rotate_ctx(&cpuctx->ctx);
1060         rotate_ctx(ctx);
1061
1062         perf_counter_cpu_sched_in(cpuctx, cpu);
1063         perf_counter_task_sched_in(curr, cpu);
1064 }
1065
1066 /*
1067  * Cross CPU call to read the hardware counter
1068  */
1069 static void __read(void *info)
1070 {
1071         struct perf_counter *counter = info;
1072         struct perf_counter_context *ctx = counter->ctx;
1073         unsigned long flags;
1074
1075         local_irq_save(flags);
1076         if (ctx->is_active)
1077                 update_context_time(ctx);
1078         counter->pmu->read(counter);
1079         update_counter_times(counter);
1080         local_irq_restore(flags);
1081 }
1082
1083 static u64 perf_counter_read(struct perf_counter *counter)
1084 {
1085         /*
1086          * If counter is enabled and currently active on a CPU, update the
1087          * value in the counter structure:
1088          */
1089         if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
1090                 smp_call_function_single(counter->oncpu,
1091                                          __read, counter, 1);
1092         } else if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
1093                 update_counter_times(counter);
1094         }
1095
1096         return atomic64_read(&counter->count);
1097 }
1098
1099 static void put_context(struct perf_counter_context *ctx)
1100 {
1101         if (ctx->task)
1102                 put_task_struct(ctx->task);
1103 }
1104
1105 static struct perf_counter_context *find_get_context(pid_t pid, int cpu)
1106 {
1107         struct perf_cpu_context *cpuctx;
1108         struct perf_counter_context *ctx;
1109         struct task_struct *task;
1110
1111         /*
1112          * If cpu is not a wildcard then this is a percpu counter:
1113          */
1114         if (cpu != -1) {
1115                 /* Must be root to operate on a CPU counter: */
1116                 if (sysctl_perf_counter_priv && !capable(CAP_SYS_ADMIN))
1117                         return ERR_PTR(-EACCES);
1118
1119                 if (cpu < 0 || cpu > num_possible_cpus())
1120                         return ERR_PTR(-EINVAL);
1121
1122                 /*
1123                  * We could be clever and allow to attach a counter to an
1124                  * offline CPU and activate it when the CPU comes up, but
1125                  * that's for later.
1126                  */
1127                 if (!cpu_isset(cpu, cpu_online_map))
1128                         return ERR_PTR(-ENODEV);
1129
1130                 cpuctx = &per_cpu(perf_cpu_context, cpu);
1131                 ctx = &cpuctx->ctx;
1132
1133                 return ctx;
1134         }
1135
1136         rcu_read_lock();
1137         if (!pid)
1138                 task = current;
1139         else
1140                 task = find_task_by_vpid(pid);
1141         if (task)
1142                 get_task_struct(task);
1143         rcu_read_unlock();
1144
1145         if (!task)
1146                 return ERR_PTR(-ESRCH);
1147
1148         ctx = &task->perf_counter_ctx;
1149         ctx->task = task;
1150
1151         /* Reuse ptrace permission checks for now. */
1152         if (!ptrace_may_access(task, PTRACE_MODE_READ)) {
1153                 put_context(ctx);
1154                 return ERR_PTR(-EACCES);
1155         }
1156
1157         return ctx;
1158 }
1159
1160 static void free_counter_rcu(struct rcu_head *head)
1161 {
1162         struct perf_counter *counter;
1163
1164         counter = container_of(head, struct perf_counter, rcu_head);
1165         kfree(counter);
1166 }
1167
1168 static void perf_pending_sync(struct perf_counter *counter);
1169
1170 static void free_counter(struct perf_counter *counter)
1171 {
1172         perf_pending_sync(counter);
1173
1174         atomic_dec(&nr_counters);
1175         if (counter->hw_event.mmap)
1176                 atomic_dec(&nr_mmap_tracking);
1177         if (counter->hw_event.munmap)
1178                 atomic_dec(&nr_munmap_tracking);
1179         if (counter->hw_event.comm)
1180                 atomic_dec(&nr_comm_tracking);
1181
1182         if (counter->destroy)
1183                 counter->destroy(counter);
1184
1185         call_rcu(&counter->rcu_head, free_counter_rcu);
1186 }
1187
1188 /*
1189  * Called when the last reference to the file is gone.
1190  */
1191 static int perf_release(struct inode *inode, struct file *file)
1192 {
1193         struct perf_counter *counter = file->private_data;
1194         struct perf_counter_context *ctx = counter->ctx;
1195
1196         file->private_data = NULL;
1197
1198         mutex_lock(&ctx->mutex);
1199         mutex_lock(&counter->mutex);
1200
1201         perf_counter_remove_from_context(counter);
1202
1203         mutex_unlock(&counter->mutex);
1204         mutex_unlock(&ctx->mutex);
1205
1206         free_counter(counter);
1207         put_context(ctx);
1208
1209         return 0;
1210 }
1211
1212 /*
1213  * Read the performance counter - simple non blocking version for now
1214  */
1215 static ssize_t
1216 perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count)
1217 {
1218         u64 values[3];
1219         int n;
1220
1221         /*
1222          * Return end-of-file for a read on a counter that is in
1223          * error state (i.e. because it was pinned but it couldn't be
1224          * scheduled on to the CPU at some point).
1225          */
1226         if (counter->state == PERF_COUNTER_STATE_ERROR)
1227                 return 0;
1228
1229         mutex_lock(&counter->mutex);
1230         values[0] = perf_counter_read(counter);
1231         n = 1;
1232         if (counter->hw_event.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1233                 values[n++] = counter->total_time_enabled +
1234                         atomic64_read(&counter->child_total_time_enabled);
1235         if (counter->hw_event.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1236                 values[n++] = counter->total_time_running +
1237                         atomic64_read(&counter->child_total_time_running);
1238         mutex_unlock(&counter->mutex);
1239
1240         if (count < n * sizeof(u64))
1241                 return -EINVAL;
1242         count = n * sizeof(u64);
1243
1244         if (copy_to_user(buf, values, count))
1245                 return -EFAULT;
1246
1247         return count;
1248 }
1249
1250 static ssize_t
1251 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
1252 {
1253         struct perf_counter *counter = file->private_data;
1254
1255         return perf_read_hw(counter, buf, count);
1256 }
1257
1258 static unsigned int perf_poll(struct file *file, poll_table *wait)
1259 {
1260         struct perf_counter *counter = file->private_data;
1261         struct perf_mmap_data *data;
1262         unsigned int events = POLL_HUP;
1263
1264         rcu_read_lock();
1265         data = rcu_dereference(counter->data);
1266         if (data)
1267                 events = atomic_xchg(&data->poll, 0);
1268         rcu_read_unlock();
1269
1270         poll_wait(file, &counter->waitq, wait);
1271
1272         return events;
1273 }
1274
1275 static void perf_counter_reset(struct perf_counter *counter)
1276 {
1277         (void)perf_counter_read(counter);
1278         atomic_set(&counter->count, 0);
1279         perf_counter_update_userpage(counter);
1280 }
1281
1282 static void perf_counter_for_each_sibling(struct perf_counter *counter,
1283                                           void (*func)(struct perf_counter *))
1284 {
1285         struct perf_counter_context *ctx = counter->ctx;
1286         struct perf_counter *sibling;
1287
1288         spin_lock_irq(&ctx->lock);
1289         counter = counter->group_leader;
1290
1291         func(counter);
1292         list_for_each_entry(sibling, &counter->sibling_list, list_entry)
1293                 func(sibling);
1294         spin_unlock_irq(&ctx->lock);
1295 }
1296
1297 static void perf_counter_for_each_child(struct perf_counter *counter,
1298                                         void (*func)(struct perf_counter *))
1299 {
1300         struct perf_counter *child;
1301
1302         mutex_lock(&counter->mutex);
1303         func(counter);
1304         list_for_each_entry(child, &counter->child_list, child_list)
1305                 func(child);
1306         mutex_unlock(&counter->mutex);
1307 }
1308
1309 static void perf_counter_for_each(struct perf_counter *counter,
1310                                   void (*func)(struct perf_counter *))
1311 {
1312         struct perf_counter *child;
1313
1314         mutex_lock(&counter->mutex);
1315         perf_counter_for_each_sibling(counter, func);
1316         list_for_each_entry(child, &counter->child_list, child_list)
1317                 perf_counter_for_each_sibling(child, func);
1318         mutex_unlock(&counter->mutex);
1319 }
1320
1321 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1322 {
1323         struct perf_counter *counter = file->private_data;
1324         void (*func)(struct perf_counter *);
1325         u32 flags = arg;
1326
1327         switch (cmd) {
1328         case PERF_COUNTER_IOC_ENABLE:
1329                 func = perf_counter_enable;
1330                 break;
1331         case PERF_COUNTER_IOC_DISABLE:
1332                 func = perf_counter_disable;
1333                 break;
1334         case PERF_COUNTER_IOC_RESET:
1335                 func = perf_counter_reset;
1336                 break;
1337
1338         case PERF_COUNTER_IOC_REFRESH:
1339                 return perf_counter_refresh(counter, arg);
1340         default:
1341                 return -ENOTTY;
1342         }
1343
1344         if (flags & PERF_IOC_FLAG_GROUP)
1345                 perf_counter_for_each(counter, func);
1346         else
1347                 perf_counter_for_each_child(counter, func);
1348
1349         return 0;
1350 }
1351
1352 /*
1353  * Callers need to ensure there can be no nesting of this function, otherwise
1354  * the seqlock logic goes bad. We can not serialize this because the arch
1355  * code calls this from NMI context.
1356  */
1357 void perf_counter_update_userpage(struct perf_counter *counter)
1358 {
1359         struct perf_mmap_data *data;
1360         struct perf_counter_mmap_page *userpg;
1361
1362         rcu_read_lock();
1363         data = rcu_dereference(counter->data);
1364         if (!data)
1365                 goto unlock;
1366
1367         userpg = data->user_page;
1368
1369         /*
1370          * Disable preemption so as to not let the corresponding user-space
1371          * spin too long if we get preempted.
1372          */
1373         preempt_disable();
1374         ++userpg->lock;
1375         barrier();
1376         userpg->index = counter->hw.idx;
1377         userpg->offset = atomic64_read(&counter->count);
1378         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
1379                 userpg->offset -= atomic64_read(&counter->hw.prev_count);
1380
1381         barrier();
1382         ++userpg->lock;
1383         preempt_enable();
1384 unlock:
1385         rcu_read_unlock();
1386 }
1387
1388 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1389 {
1390         struct perf_counter *counter = vma->vm_file->private_data;
1391         struct perf_mmap_data *data;
1392         int ret = VM_FAULT_SIGBUS;
1393
1394         rcu_read_lock();
1395         data = rcu_dereference(counter->data);
1396         if (!data)
1397                 goto unlock;
1398
1399         if (vmf->pgoff == 0) {
1400                 vmf->page = virt_to_page(data->user_page);
1401         } else {
1402                 int nr = vmf->pgoff - 1;
1403
1404                 if ((unsigned)nr > data->nr_pages)
1405                         goto unlock;
1406
1407                 vmf->page = virt_to_page(data->data_pages[nr]);
1408         }
1409         get_page(vmf->page);
1410         ret = 0;
1411 unlock:
1412         rcu_read_unlock();
1413
1414         return ret;
1415 }
1416
1417 static int perf_mmap_data_alloc(struct perf_counter *counter, int nr_pages)
1418 {
1419         struct perf_mmap_data *data;
1420         unsigned long size;
1421         int i;
1422
1423         WARN_ON(atomic_read(&counter->mmap_count));
1424
1425         size = sizeof(struct perf_mmap_data);
1426         size += nr_pages * sizeof(void *);
1427
1428         data = kzalloc(size, GFP_KERNEL);
1429         if (!data)
1430                 goto fail;
1431
1432         data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
1433         if (!data->user_page)
1434                 goto fail_user_page;
1435
1436         for (i = 0; i < nr_pages; i++) {
1437                 data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
1438                 if (!data->data_pages[i])
1439                         goto fail_data_pages;
1440         }
1441
1442         data->nr_pages = nr_pages;
1443         atomic_set(&data->lock, -1);
1444
1445         rcu_assign_pointer(counter->data, data);
1446
1447         return 0;
1448
1449 fail_data_pages:
1450         for (i--; i >= 0; i--)
1451                 free_page((unsigned long)data->data_pages[i]);
1452
1453         free_page((unsigned long)data->user_page);
1454
1455 fail_user_page:
1456         kfree(data);
1457
1458 fail:
1459         return -ENOMEM;
1460 }
1461
1462 static void __perf_mmap_data_free(struct rcu_head *rcu_head)
1463 {
1464         struct perf_mmap_data *data = container_of(rcu_head,
1465                         struct perf_mmap_data, rcu_head);
1466         int i;
1467
1468         free_page((unsigned long)data->user_page);
1469         for (i = 0; i < data->nr_pages; i++)
1470                 free_page((unsigned long)data->data_pages[i]);
1471         kfree(data);
1472 }
1473
1474 static void perf_mmap_data_free(struct perf_counter *counter)
1475 {
1476         struct perf_mmap_data *data = counter->data;
1477
1478         WARN_ON(atomic_read(&counter->mmap_count));
1479
1480         rcu_assign_pointer(counter->data, NULL);
1481         call_rcu(&data->rcu_head, __perf_mmap_data_free);
1482 }
1483
1484 static void perf_mmap_open(struct vm_area_struct *vma)
1485 {
1486         struct perf_counter *counter = vma->vm_file->private_data;
1487
1488         atomic_inc(&counter->mmap_count);
1489 }
1490
1491 static void perf_mmap_close(struct vm_area_struct *vma)
1492 {
1493         struct perf_counter *counter = vma->vm_file->private_data;
1494
1495         if (atomic_dec_and_mutex_lock(&counter->mmap_count,
1496                                       &counter->mmap_mutex)) {
1497                 vma->vm_mm->locked_vm -= counter->data->nr_locked;
1498                 perf_mmap_data_free(counter);
1499                 mutex_unlock(&counter->mmap_mutex);
1500         }
1501 }
1502
1503 static struct vm_operations_struct perf_mmap_vmops = {
1504         .open  = perf_mmap_open,
1505         .close = perf_mmap_close,
1506         .fault = perf_mmap_fault,
1507 };
1508
1509 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
1510 {
1511         struct perf_counter *counter = file->private_data;
1512         unsigned long vma_size;
1513         unsigned long nr_pages;
1514         unsigned long locked, lock_limit;
1515         int ret = 0;
1516         long extra;
1517
1518         if (!(vma->vm_flags & VM_SHARED) || (vma->vm_flags & VM_WRITE))
1519                 return -EINVAL;
1520
1521         vma_size = vma->vm_end - vma->vm_start;
1522         nr_pages = (vma_size / PAGE_SIZE) - 1;
1523
1524         /*
1525          * If we have data pages ensure they're a power-of-two number, so we
1526          * can do bitmasks instead of modulo.
1527          */
1528         if (nr_pages != 0 && !is_power_of_2(nr_pages))
1529                 return -EINVAL;
1530
1531         if (vma_size != PAGE_SIZE * (1 + nr_pages))
1532                 return -EINVAL;
1533
1534         if (vma->vm_pgoff != 0)
1535                 return -EINVAL;
1536
1537         mutex_lock(&counter->mmap_mutex);
1538         if (atomic_inc_not_zero(&counter->mmap_count)) {
1539                 if (nr_pages != counter->data->nr_pages)
1540                         ret = -EINVAL;
1541                 goto unlock;
1542         }
1543
1544         extra = nr_pages /* + 1 only account the data pages */;
1545         extra -= sysctl_perf_counter_mlock >> (PAGE_SHIFT - 10);
1546         if (extra < 0)
1547                 extra = 0;
1548
1549         locked = vma->vm_mm->locked_vm + extra;
1550
1551         lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
1552         lock_limit >>= PAGE_SHIFT;
1553
1554         if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) {
1555                 ret = -EPERM;
1556                 goto unlock;
1557         }
1558
1559         WARN_ON(counter->data);
1560         ret = perf_mmap_data_alloc(counter, nr_pages);
1561         if (ret)
1562                 goto unlock;
1563
1564         atomic_set(&counter->mmap_count, 1);
1565         vma->vm_mm->locked_vm += extra;
1566         counter->data->nr_locked = extra;
1567 unlock:
1568         mutex_unlock(&counter->mmap_mutex);
1569
1570         vma->vm_flags &= ~VM_MAYWRITE;
1571         vma->vm_flags |= VM_RESERVED;
1572         vma->vm_ops = &perf_mmap_vmops;
1573
1574         return ret;
1575 }
1576
1577 static int perf_fasync(int fd, struct file *filp, int on)
1578 {
1579         struct perf_counter *counter = filp->private_data;
1580         struct inode *inode = filp->f_path.dentry->d_inode;
1581         int retval;
1582
1583         mutex_lock(&inode->i_mutex);
1584         retval = fasync_helper(fd, filp, on, &counter->fasync);
1585         mutex_unlock(&inode->i_mutex);
1586
1587         if (retval < 0)
1588                 return retval;
1589
1590         return 0;
1591 }
1592
1593 static const struct file_operations perf_fops = {
1594         .release                = perf_release,
1595         .read                   = perf_read,
1596         .poll                   = perf_poll,
1597         .unlocked_ioctl         = perf_ioctl,
1598         .compat_ioctl           = perf_ioctl,
1599         .mmap                   = perf_mmap,
1600         .fasync                 = perf_fasync,
1601 };
1602
1603 /*
1604  * Perf counter wakeup
1605  *
1606  * If there's data, ensure we set the poll() state and publish everything
1607  * to user-space before waking everybody up.
1608  */
1609
1610 void perf_counter_wakeup(struct perf_counter *counter)
1611 {
1612         wake_up_all(&counter->waitq);
1613
1614         if (counter->pending_kill) {
1615                 kill_fasync(&counter->fasync, SIGIO, counter->pending_kill);
1616                 counter->pending_kill = 0;
1617         }
1618 }
1619
1620 /*
1621  * Pending wakeups
1622  *
1623  * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
1624  *
1625  * The NMI bit means we cannot possibly take locks. Therefore, maintain a
1626  * single linked list and use cmpxchg() to add entries lockless.
1627  */
1628
1629 static void perf_pending_counter(struct perf_pending_entry *entry)
1630 {
1631         struct perf_counter *counter = container_of(entry,
1632                         struct perf_counter, pending);
1633
1634         if (counter->pending_disable) {
1635                 counter->pending_disable = 0;
1636                 perf_counter_disable(counter);
1637         }
1638
1639         if (counter->pending_wakeup) {
1640                 counter->pending_wakeup = 0;
1641                 perf_counter_wakeup(counter);
1642         }
1643 }
1644
1645 #define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
1646
1647 static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
1648         PENDING_TAIL,
1649 };
1650
1651 static void perf_pending_queue(struct perf_pending_entry *entry,
1652                                void (*func)(struct perf_pending_entry *))
1653 {
1654         struct perf_pending_entry **head;
1655
1656         if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
1657                 return;
1658
1659         entry->func = func;
1660
1661         head = &get_cpu_var(perf_pending_head);
1662
1663         do {
1664                 entry->next = *head;
1665         } while (cmpxchg(head, entry->next, entry) != entry->next);
1666
1667         set_perf_counter_pending();
1668
1669         put_cpu_var(perf_pending_head);
1670 }
1671
1672 static int __perf_pending_run(void)
1673 {
1674         struct perf_pending_entry *list;
1675         int nr = 0;
1676
1677         list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
1678         while (list != PENDING_TAIL) {
1679                 void (*func)(struct perf_pending_entry *);
1680                 struct perf_pending_entry *entry = list;
1681
1682                 list = list->next;
1683
1684                 func = entry->func;
1685                 entry->next = NULL;
1686                 /*
1687                  * Ensure we observe the unqueue before we issue the wakeup,
1688                  * so that we won't be waiting forever.
1689                  * -- see perf_not_pending().
1690                  */
1691                 smp_wmb();
1692
1693                 func(entry);
1694                 nr++;
1695         }
1696
1697         return nr;
1698 }
1699
1700 static inline int perf_not_pending(struct perf_counter *counter)
1701 {
1702         /*
1703          * If we flush on whatever cpu we run, there is a chance we don't
1704          * need to wait.
1705          */
1706         get_cpu();
1707         __perf_pending_run();
1708         put_cpu();
1709
1710         /*
1711          * Ensure we see the proper queue state before going to sleep
1712          * so that we do not miss the wakeup. -- see perf_pending_handle()
1713          */
1714         smp_rmb();
1715         return counter->pending.next == NULL;
1716 }
1717
1718 static void perf_pending_sync(struct perf_counter *counter)
1719 {
1720         wait_event(counter->waitq, perf_not_pending(counter));
1721 }
1722
1723 void perf_counter_do_pending(void)
1724 {
1725         __perf_pending_run();
1726 }
1727
1728 /*
1729  * Callchain support -- arch specific
1730  */
1731
1732 __weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
1733 {
1734         return NULL;
1735 }
1736
1737 /*
1738  * Output
1739  */
1740
1741 struct perf_output_handle {
1742         struct perf_counter     *counter;
1743         struct perf_mmap_data   *data;
1744         unsigned int            offset;
1745         unsigned int            head;
1746         int                     nmi;
1747         int                     overflow;
1748         int                     locked;
1749         unsigned long           flags;
1750 };
1751
1752 static void perf_output_wakeup(struct perf_output_handle *handle)
1753 {
1754         atomic_set(&handle->data->poll, POLL_IN);
1755
1756         if (handle->nmi) {
1757                 handle->counter->pending_wakeup = 1;
1758                 perf_pending_queue(&handle->counter->pending,
1759                                    perf_pending_counter);
1760         } else
1761                 perf_counter_wakeup(handle->counter);
1762 }
1763
1764 /*
1765  * Curious locking construct.
1766  *
1767  * We need to ensure a later event doesn't publish a head when a former
1768  * event isn't done writing. However since we need to deal with NMIs we
1769  * cannot fully serialize things.
1770  *
1771  * What we do is serialize between CPUs so we only have to deal with NMI
1772  * nesting on a single CPU.
1773  *
1774  * We only publish the head (and generate a wakeup) when the outer-most
1775  * event completes.
1776  */
1777 static void perf_output_lock(struct perf_output_handle *handle)
1778 {
1779         struct perf_mmap_data *data = handle->data;
1780         int cpu;
1781
1782         handle->locked = 0;
1783
1784         local_irq_save(handle->flags);
1785         cpu = smp_processor_id();
1786
1787         if (in_nmi() && atomic_read(&data->lock) == cpu)
1788                 return;
1789
1790         while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
1791                 cpu_relax();
1792
1793         handle->locked = 1;
1794 }
1795
1796 static void perf_output_unlock(struct perf_output_handle *handle)
1797 {
1798         struct perf_mmap_data *data = handle->data;
1799         int head, cpu;
1800
1801         data->done_head = data->head;
1802
1803         if (!handle->locked)
1804                 goto out;
1805
1806 again:
1807         /*
1808          * The xchg implies a full barrier that ensures all writes are done
1809          * before we publish the new head, matched by a rmb() in userspace when
1810          * reading this position.
1811          */
1812         while ((head = atomic_xchg(&data->done_head, 0)))
1813                 data->user_page->data_head = head;
1814
1815         /*
1816          * NMI can happen here, which means we can miss a done_head update.
1817          */
1818
1819         cpu = atomic_xchg(&data->lock, -1);
1820         WARN_ON_ONCE(cpu != smp_processor_id());
1821
1822         /*
1823          * Therefore we have to validate we did not indeed do so.
1824          */
1825         if (unlikely(atomic_read(&data->done_head))) {
1826                 /*
1827                  * Since we had it locked, we can lock it again.
1828                  */
1829                 while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
1830                         cpu_relax();
1831
1832                 goto again;
1833         }
1834
1835         if (atomic_xchg(&data->wakeup, 0))
1836                 perf_output_wakeup(handle);
1837 out:
1838         local_irq_restore(handle->flags);
1839 }
1840
1841 static int perf_output_begin(struct perf_output_handle *handle,
1842                              struct perf_counter *counter, unsigned int size,
1843                              int nmi, int overflow)
1844 {
1845         struct perf_mmap_data *data;
1846         unsigned int offset, head;
1847
1848         /*
1849          * For inherited counters we send all the output towards the parent.
1850          */
1851         if (counter->parent)
1852                 counter = counter->parent;
1853
1854         rcu_read_lock();
1855         data = rcu_dereference(counter->data);
1856         if (!data)
1857                 goto out;
1858
1859         handle->data     = data;
1860         handle->counter  = counter;
1861         handle->nmi      = nmi;
1862         handle->overflow = overflow;
1863
1864         if (!data->nr_pages)
1865                 goto fail;
1866
1867         perf_output_lock(handle);
1868
1869         do {
1870                 offset = head = atomic_read(&data->head);
1871                 head += size;
1872         } while (atomic_cmpxchg(&data->head, offset, head) != offset);
1873
1874         handle->offset  = offset;
1875         handle->head    = head;
1876
1877         if ((offset >> PAGE_SHIFT) != (head >> PAGE_SHIFT))
1878                 atomic_set(&data->wakeup, 1);
1879
1880         return 0;
1881
1882 fail:
1883         perf_output_wakeup(handle);
1884 out:
1885         rcu_read_unlock();
1886
1887         return -ENOSPC;
1888 }
1889
1890 static void perf_output_copy(struct perf_output_handle *handle,
1891                              void *buf, unsigned int len)
1892 {
1893         unsigned int pages_mask;
1894         unsigned int offset;
1895         unsigned int size;
1896         void **pages;
1897
1898         offset          = handle->offset;
1899         pages_mask      = handle->data->nr_pages - 1;
1900         pages           = handle->data->data_pages;
1901
1902         do {
1903                 unsigned int page_offset;
1904                 int nr;
1905
1906                 nr          = (offset >> PAGE_SHIFT) & pages_mask;
1907                 page_offset = offset & (PAGE_SIZE - 1);
1908                 size        = min_t(unsigned int, PAGE_SIZE - page_offset, len);
1909
1910                 memcpy(pages[nr] + page_offset, buf, size);
1911
1912                 len         -= size;
1913                 buf         += size;
1914                 offset      += size;
1915         } while (len);
1916
1917         handle->offset = offset;
1918
1919         WARN_ON_ONCE(handle->offset > handle->head);
1920 }
1921
1922 #define perf_output_put(handle, x) \
1923         perf_output_copy((handle), &(x), sizeof(x))
1924
1925 static void perf_output_end(struct perf_output_handle *handle)
1926 {
1927         struct perf_counter *counter = handle->counter;
1928         struct perf_mmap_data *data = handle->data;
1929
1930         int wakeup_events = counter->hw_event.wakeup_events;
1931
1932         if (handle->overflow && wakeup_events) {
1933                 int events = atomic_inc_return(&data->events);
1934                 if (events >= wakeup_events) {
1935                         atomic_sub(wakeup_events, &data->events);
1936                         atomic_set(&data->wakeup, 1);
1937                 }
1938         }
1939
1940         perf_output_unlock(handle);
1941         rcu_read_unlock();
1942 }
1943
1944 static void perf_counter_output(struct perf_counter *counter,
1945                                 int nmi, struct pt_regs *regs, u64 addr)
1946 {
1947         int ret;
1948         u64 record_type = counter->hw_event.record_type;
1949         struct perf_output_handle handle;
1950         struct perf_event_header header;
1951         u64 ip;
1952         struct {
1953                 u32 pid, tid;
1954         } tid_entry;
1955         struct {
1956                 u64 event;
1957                 u64 counter;
1958         } group_entry;
1959         struct perf_callchain_entry *callchain = NULL;
1960         int callchain_size = 0;
1961         u64 time;
1962         struct {
1963                 u32 cpu, reserved;
1964         } cpu_entry;
1965
1966         header.type = 0;
1967         header.size = sizeof(header);
1968
1969         header.misc = PERF_EVENT_MISC_OVERFLOW;
1970         header.misc |= user_mode(regs) ?
1971                 PERF_EVENT_MISC_USER : PERF_EVENT_MISC_KERNEL;
1972
1973         if (record_type & PERF_RECORD_IP) {
1974                 ip = instruction_pointer(regs);
1975                 header.type |= PERF_RECORD_IP;
1976                 header.size += sizeof(ip);
1977         }
1978
1979         if (record_type & PERF_RECORD_TID) {
1980                 /* namespace issues */
1981                 tid_entry.pid = current->group_leader->pid;
1982                 tid_entry.tid = current->pid;
1983
1984                 header.type |= PERF_RECORD_TID;
1985                 header.size += sizeof(tid_entry);
1986         }
1987
1988         if (record_type & PERF_RECORD_TIME) {
1989                 /*
1990                  * Maybe do better on x86 and provide cpu_clock_nmi()
1991                  */
1992                 time = sched_clock();
1993
1994                 header.type |= PERF_RECORD_TIME;
1995                 header.size += sizeof(u64);
1996         }
1997
1998         if (record_type & PERF_RECORD_ADDR) {
1999                 header.type |= PERF_RECORD_ADDR;
2000                 header.size += sizeof(u64);
2001         }
2002
2003         if (record_type & PERF_RECORD_CONFIG) {
2004                 header.type |= PERF_RECORD_CONFIG;
2005                 header.size += sizeof(u64);
2006         }
2007
2008         if (record_type & PERF_RECORD_CPU) {
2009                 header.type |= PERF_RECORD_CPU;
2010                 header.size += sizeof(cpu_entry);
2011
2012                 cpu_entry.cpu = raw_smp_processor_id();
2013         }
2014
2015         if (record_type & PERF_RECORD_GROUP) {
2016                 header.type |= PERF_RECORD_GROUP;
2017                 header.size += sizeof(u64) +
2018                         counter->nr_siblings * sizeof(group_entry);
2019         }
2020
2021         if (record_type & PERF_RECORD_CALLCHAIN) {
2022                 callchain = perf_callchain(regs);
2023
2024                 if (callchain) {
2025                         callchain_size = (1 + callchain->nr) * sizeof(u64);
2026
2027                         header.type |= PERF_RECORD_CALLCHAIN;
2028                         header.size += callchain_size;
2029                 }
2030         }
2031
2032         ret = perf_output_begin(&handle, counter, header.size, nmi, 1);
2033         if (ret)
2034                 return;
2035
2036         perf_output_put(&handle, header);
2037
2038         if (record_type & PERF_RECORD_IP)
2039                 perf_output_put(&handle, ip);
2040
2041         if (record_type & PERF_RECORD_TID)
2042                 perf_output_put(&handle, tid_entry);
2043
2044         if (record_type & PERF_RECORD_TIME)
2045                 perf_output_put(&handle, time);
2046
2047         if (record_type & PERF_RECORD_ADDR)
2048                 perf_output_put(&handle, addr);
2049
2050         if (record_type & PERF_RECORD_CONFIG)
2051                 perf_output_put(&handle, counter->hw_event.config);
2052
2053         if (record_type & PERF_RECORD_CPU)
2054                 perf_output_put(&handle, cpu_entry);
2055
2056         /*
2057          * XXX PERF_RECORD_GROUP vs inherited counters seems difficult.
2058          */
2059         if (record_type & PERF_RECORD_GROUP) {
2060                 struct perf_counter *leader, *sub;
2061                 u64 nr = counter->nr_siblings;
2062
2063                 perf_output_put(&handle, nr);
2064
2065                 leader = counter->group_leader;
2066                 list_for_each_entry(sub, &leader->sibling_list, list_entry) {
2067                         if (sub != counter)
2068                                 sub->pmu->read(sub);
2069
2070                         group_entry.event = sub->hw_event.config;
2071                         group_entry.counter = atomic64_read(&sub->count);
2072
2073                         perf_output_put(&handle, group_entry);
2074                 }
2075         }
2076
2077         if (callchain)
2078                 perf_output_copy(&handle, callchain, callchain_size);
2079
2080         perf_output_end(&handle);
2081 }
2082
2083 /*
2084  * comm tracking
2085  */
2086
2087 struct perf_comm_event {
2088         struct task_struct      *task;
2089         char                    *comm;
2090         int                     comm_size;
2091
2092         struct {
2093                 struct perf_event_header        header;
2094
2095                 u32                             pid;
2096                 u32                             tid;
2097         } event;
2098 };
2099
2100 static void perf_counter_comm_output(struct perf_counter *counter,
2101                                      struct perf_comm_event *comm_event)
2102 {
2103         struct perf_output_handle handle;
2104         int size = comm_event->event.header.size;
2105         int ret = perf_output_begin(&handle, counter, size, 0, 0);
2106
2107         if (ret)
2108                 return;
2109
2110         perf_output_put(&handle, comm_event->event);
2111         perf_output_copy(&handle, comm_event->comm,
2112                                    comm_event->comm_size);
2113         perf_output_end(&handle);
2114 }
2115
2116 static int perf_counter_comm_match(struct perf_counter *counter,
2117                                    struct perf_comm_event *comm_event)
2118 {
2119         if (counter->hw_event.comm &&
2120             comm_event->event.header.type == PERF_EVENT_COMM)
2121                 return 1;
2122
2123         return 0;
2124 }
2125
2126 static void perf_counter_comm_ctx(struct perf_counter_context *ctx,
2127                                   struct perf_comm_event *comm_event)
2128 {
2129         struct perf_counter *counter;
2130
2131         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2132                 return;
2133
2134         rcu_read_lock();
2135         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2136                 if (perf_counter_comm_match(counter, comm_event))
2137                         perf_counter_comm_output(counter, comm_event);
2138         }
2139         rcu_read_unlock();
2140 }
2141
2142 static void perf_counter_comm_event(struct perf_comm_event *comm_event)
2143 {
2144         struct perf_cpu_context *cpuctx;
2145         unsigned int size;
2146         char *comm = comm_event->task->comm;
2147
2148         size = ALIGN(strlen(comm)+1, sizeof(u64));
2149
2150         comm_event->comm = comm;
2151         comm_event->comm_size = size;
2152
2153         comm_event->event.header.size = sizeof(comm_event->event) + size;
2154
2155         cpuctx = &get_cpu_var(perf_cpu_context);
2156         perf_counter_comm_ctx(&cpuctx->ctx, comm_event);
2157         put_cpu_var(perf_cpu_context);
2158
2159         perf_counter_comm_ctx(&current->perf_counter_ctx, comm_event);
2160 }
2161
2162 void perf_counter_comm(struct task_struct *task)
2163 {
2164         struct perf_comm_event comm_event;
2165
2166         if (!atomic_read(&nr_comm_tracking))
2167                 return;
2168
2169         comm_event = (struct perf_comm_event){
2170                 .task   = task,
2171                 .event  = {
2172                         .header = { .type = PERF_EVENT_COMM, },
2173                         .pid    = task->group_leader->pid,
2174                         .tid    = task->pid,
2175                 },
2176         };
2177
2178         perf_counter_comm_event(&comm_event);
2179 }
2180
2181 /*
2182  * mmap tracking
2183  */
2184
2185 struct perf_mmap_event {
2186         struct file     *file;
2187         char            *file_name;
2188         int             file_size;
2189
2190         struct {
2191                 struct perf_event_header        header;
2192
2193                 u32                             pid;
2194                 u32                             tid;
2195                 u64                             start;
2196                 u64                             len;
2197                 u64                             pgoff;
2198         } event;
2199 };
2200
2201 static void perf_counter_mmap_output(struct perf_counter *counter,
2202                                      struct perf_mmap_event *mmap_event)
2203 {
2204         struct perf_output_handle handle;
2205         int size = mmap_event->event.header.size;
2206         int ret = perf_output_begin(&handle, counter, size, 0, 0);
2207
2208         if (ret)
2209                 return;
2210
2211         perf_output_put(&handle, mmap_event->event);
2212         perf_output_copy(&handle, mmap_event->file_name,
2213                                    mmap_event->file_size);
2214         perf_output_end(&handle);
2215 }
2216
2217 static int perf_counter_mmap_match(struct perf_counter *counter,
2218                                    struct perf_mmap_event *mmap_event)
2219 {
2220         if (counter->hw_event.mmap &&
2221             mmap_event->event.header.type == PERF_EVENT_MMAP)
2222                 return 1;
2223
2224         if (counter->hw_event.munmap &&
2225             mmap_event->event.header.type == PERF_EVENT_MUNMAP)
2226                 return 1;
2227
2228         return 0;
2229 }
2230
2231 static void perf_counter_mmap_ctx(struct perf_counter_context *ctx,
2232                                   struct perf_mmap_event *mmap_event)
2233 {
2234         struct perf_counter *counter;
2235
2236         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2237                 return;
2238
2239         rcu_read_lock();
2240         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2241                 if (perf_counter_mmap_match(counter, mmap_event))
2242                         perf_counter_mmap_output(counter, mmap_event);
2243         }
2244         rcu_read_unlock();
2245 }
2246
2247 static void perf_counter_mmap_event(struct perf_mmap_event *mmap_event)
2248 {
2249         struct perf_cpu_context *cpuctx;
2250         struct file *file = mmap_event->file;
2251         unsigned int size;
2252         char tmp[16];
2253         char *buf = NULL;
2254         char *name;
2255
2256         if (file) {
2257                 buf = kzalloc(PATH_MAX, GFP_KERNEL);
2258                 if (!buf) {
2259                         name = strncpy(tmp, "//enomem", sizeof(tmp));
2260                         goto got_name;
2261                 }
2262                 name = d_path(&file->f_path, buf, PATH_MAX);
2263                 if (IS_ERR(name)) {
2264                         name = strncpy(tmp, "//toolong", sizeof(tmp));
2265                         goto got_name;
2266                 }
2267         } else {
2268                 name = strncpy(tmp, "//anon", sizeof(tmp));
2269                 goto got_name;
2270         }
2271
2272 got_name:
2273         size = ALIGN(strlen(name)+1, sizeof(u64));
2274
2275         mmap_event->file_name = name;
2276         mmap_event->file_size = size;
2277
2278         mmap_event->event.header.size = sizeof(mmap_event->event) + size;
2279
2280         cpuctx = &get_cpu_var(perf_cpu_context);
2281         perf_counter_mmap_ctx(&cpuctx->ctx, mmap_event);
2282         put_cpu_var(perf_cpu_context);
2283
2284         perf_counter_mmap_ctx(&current->perf_counter_ctx, mmap_event);
2285
2286         kfree(buf);
2287 }
2288
2289 void perf_counter_mmap(unsigned long addr, unsigned long len,
2290                        unsigned long pgoff, struct file *file)
2291 {
2292         struct perf_mmap_event mmap_event;
2293
2294         if (!atomic_read(&nr_mmap_tracking))
2295                 return;
2296
2297         mmap_event = (struct perf_mmap_event){
2298                 .file   = file,
2299                 .event  = {
2300                         .header = { .type = PERF_EVENT_MMAP, },
2301                         .pid    = current->group_leader->pid,
2302                         .tid    = current->pid,
2303                         .start  = addr,
2304                         .len    = len,
2305                         .pgoff  = pgoff,
2306                 },
2307         };
2308
2309         perf_counter_mmap_event(&mmap_event);
2310 }
2311
2312 void perf_counter_munmap(unsigned long addr, unsigned long len,
2313                          unsigned long pgoff, struct file *file)
2314 {
2315         struct perf_mmap_event mmap_event;
2316
2317         if (!atomic_read(&nr_munmap_tracking))
2318                 return;
2319
2320         mmap_event = (struct perf_mmap_event){
2321                 .file   = file,
2322                 .event  = {
2323                         .header = { .type = PERF_EVENT_MUNMAP, },
2324                         .pid    = current->group_leader->pid,
2325                         .tid    = current->pid,
2326                         .start  = addr,
2327                         .len    = len,
2328                         .pgoff  = pgoff,
2329                 },
2330         };
2331
2332         perf_counter_mmap_event(&mmap_event);
2333 }
2334
2335 /*
2336  * Generic counter overflow handling.
2337  */
2338
2339 int perf_counter_overflow(struct perf_counter *counter,
2340                           int nmi, struct pt_regs *regs, u64 addr)
2341 {
2342         int events = atomic_read(&counter->event_limit);
2343         int ret = 0;
2344
2345         /*
2346          * XXX event_limit might not quite work as expected on inherited
2347          * counters
2348          */
2349
2350         counter->pending_kill = POLL_IN;
2351         if (events && atomic_dec_and_test(&counter->event_limit)) {
2352                 ret = 1;
2353                 counter->pending_kill = POLL_HUP;
2354                 if (nmi) {
2355                         counter->pending_disable = 1;
2356                         perf_pending_queue(&counter->pending,
2357                                            perf_pending_counter);
2358                 } else
2359                         perf_counter_disable(counter);
2360         }
2361
2362         perf_counter_output(counter, nmi, regs, addr);
2363         return ret;
2364 }
2365
2366 /*
2367  * Generic software counter infrastructure
2368  */
2369
2370 static void perf_swcounter_update(struct perf_counter *counter)
2371 {
2372         struct hw_perf_counter *hwc = &counter->hw;
2373         u64 prev, now;
2374         s64 delta;
2375
2376 again:
2377         prev = atomic64_read(&hwc->prev_count);
2378         now = atomic64_read(&hwc->count);
2379         if (atomic64_cmpxchg(&hwc->prev_count, prev, now) != prev)
2380                 goto again;
2381
2382         delta = now - prev;
2383
2384         atomic64_add(delta, &counter->count);
2385         atomic64_sub(delta, &hwc->period_left);
2386 }
2387
2388 static void perf_swcounter_set_period(struct perf_counter *counter)
2389 {
2390         struct hw_perf_counter *hwc = &counter->hw;
2391         s64 left = atomic64_read(&hwc->period_left);
2392         s64 period = hwc->irq_period;
2393
2394         if (unlikely(left <= -period)) {
2395                 left = period;
2396                 atomic64_set(&hwc->period_left, left);
2397         }
2398
2399         if (unlikely(left <= 0)) {
2400                 left += period;
2401                 atomic64_add(period, &hwc->period_left);
2402         }
2403
2404         atomic64_set(&hwc->prev_count, -left);
2405         atomic64_set(&hwc->count, -left);
2406 }
2407
2408 static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer)
2409 {
2410         enum hrtimer_restart ret = HRTIMER_RESTART;
2411         struct perf_counter *counter;
2412         struct pt_regs *regs;
2413
2414         counter = container_of(hrtimer, struct perf_counter, hw.hrtimer);
2415         counter->pmu->read(counter);
2416
2417         regs = get_irq_regs();
2418         /*
2419          * In case we exclude kernel IPs or are somehow not in interrupt
2420          * context, provide the next best thing, the user IP.
2421          */
2422         if ((counter->hw_event.exclude_kernel || !regs) &&
2423                         !counter->hw_event.exclude_user)
2424                 regs = task_pt_regs(current);
2425
2426         if (regs) {
2427                 if (perf_counter_overflow(counter, 0, regs, 0))
2428                         ret = HRTIMER_NORESTART;
2429         }
2430
2431         hrtimer_forward_now(hrtimer, ns_to_ktime(counter->hw.irq_period));
2432
2433         return ret;
2434 }
2435
2436 static void perf_swcounter_overflow(struct perf_counter *counter,
2437                                     int nmi, struct pt_regs *regs, u64 addr)
2438 {
2439         perf_swcounter_update(counter);
2440         perf_swcounter_set_period(counter);
2441         if (perf_counter_overflow(counter, nmi, regs, addr))
2442                 /* soft-disable the counter */
2443                 ;
2444
2445 }
2446
2447 static int perf_swcounter_match(struct perf_counter *counter,
2448                                 enum perf_event_types type,
2449                                 u32 event, struct pt_regs *regs)
2450 {
2451         if (counter->state != PERF_COUNTER_STATE_ACTIVE)
2452                 return 0;
2453
2454         if (perf_event_raw(&counter->hw_event))
2455                 return 0;
2456
2457         if (perf_event_type(&counter->hw_event) != type)
2458                 return 0;
2459
2460         if (perf_event_id(&counter->hw_event) != event)
2461                 return 0;
2462
2463         if (counter->hw_event.exclude_user && user_mode(regs))
2464                 return 0;
2465
2466         if (counter->hw_event.exclude_kernel && !user_mode(regs))
2467                 return 0;
2468
2469         return 1;
2470 }
2471
2472 static void perf_swcounter_add(struct perf_counter *counter, u64 nr,
2473                                int nmi, struct pt_regs *regs, u64 addr)
2474 {
2475         int neg = atomic64_add_negative(nr, &counter->hw.count);
2476         if (counter->hw.irq_period && !neg)
2477                 perf_swcounter_overflow(counter, nmi, regs, addr);
2478 }
2479
2480 static void perf_swcounter_ctx_event(struct perf_counter_context *ctx,
2481                                      enum perf_event_types type, u32 event,
2482                                      u64 nr, int nmi, struct pt_regs *regs,
2483                                      u64 addr)
2484 {
2485         struct perf_counter *counter;
2486
2487         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2488                 return;
2489
2490         rcu_read_lock();
2491         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2492                 if (perf_swcounter_match(counter, type, event, regs))
2493                         perf_swcounter_add(counter, nr, nmi, regs, addr);
2494         }
2495         rcu_read_unlock();
2496 }
2497
2498 static int *perf_swcounter_recursion_context(struct perf_cpu_context *cpuctx)
2499 {
2500         if (in_nmi())
2501                 return &cpuctx->recursion[3];
2502
2503         if (in_irq())
2504                 return &cpuctx->recursion[2];
2505
2506         if (in_softirq())
2507                 return &cpuctx->recursion[1];
2508
2509         return &cpuctx->recursion[0];
2510 }
2511
2512 static void __perf_swcounter_event(enum perf_event_types type, u32 event,
2513                                    u64 nr, int nmi, struct pt_regs *regs,
2514                                    u64 addr)
2515 {
2516         struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
2517         int *recursion = perf_swcounter_recursion_context(cpuctx);
2518
2519         if (*recursion)
2520                 goto out;
2521
2522         (*recursion)++;
2523         barrier();
2524
2525         perf_swcounter_ctx_event(&cpuctx->ctx, type, event,
2526                                  nr, nmi, regs, addr);
2527         if (cpuctx->task_ctx) {
2528                 perf_swcounter_ctx_event(cpuctx->task_ctx, type, event,
2529                                          nr, nmi, regs, addr);
2530         }
2531
2532         barrier();
2533         (*recursion)--;
2534
2535 out:
2536         put_cpu_var(perf_cpu_context);
2537 }
2538
2539 void
2540 perf_swcounter_event(u32 event, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
2541 {
2542         __perf_swcounter_event(PERF_TYPE_SOFTWARE, event, nr, nmi, regs, addr);
2543 }
2544
2545 static void perf_swcounter_read(struct perf_counter *counter)
2546 {
2547         perf_swcounter_update(counter);
2548 }
2549
2550 static int perf_swcounter_enable(struct perf_counter *counter)
2551 {
2552         perf_swcounter_set_period(counter);
2553         return 0;
2554 }
2555
2556 static void perf_swcounter_disable(struct perf_counter *counter)
2557 {
2558         perf_swcounter_update(counter);
2559 }
2560
2561 static const struct pmu perf_ops_generic = {
2562         .enable         = perf_swcounter_enable,
2563         .disable        = perf_swcounter_disable,
2564         .read           = perf_swcounter_read,
2565 };
2566
2567 /*
2568  * Software counter: cpu wall time clock
2569  */
2570
2571 static void cpu_clock_perf_counter_update(struct perf_counter *counter)
2572 {
2573         int cpu = raw_smp_processor_id();
2574         s64 prev;
2575         u64 now;
2576
2577         now = cpu_clock(cpu);
2578         prev = atomic64_read(&counter->hw.prev_count);
2579         atomic64_set(&counter->hw.prev_count, now);
2580         atomic64_add(now - prev, &counter->count);
2581 }
2582
2583 static int cpu_clock_perf_counter_enable(struct perf_counter *counter)
2584 {
2585         struct hw_perf_counter *hwc = &counter->hw;
2586         int cpu = raw_smp_processor_id();
2587
2588         atomic64_set(&hwc->prev_count, cpu_clock(cpu));
2589         hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2590         hwc->hrtimer.function = perf_swcounter_hrtimer;
2591         if (hwc->irq_period) {
2592                 __hrtimer_start_range_ns(&hwc->hrtimer,
2593                                 ns_to_ktime(hwc->irq_period), 0,
2594                                 HRTIMER_MODE_REL, 0);
2595         }
2596
2597         return 0;
2598 }
2599
2600 static void cpu_clock_perf_counter_disable(struct perf_counter *counter)
2601 {
2602         hrtimer_cancel(&counter->hw.hrtimer);
2603         cpu_clock_perf_counter_update(counter);
2604 }
2605
2606 static void cpu_clock_perf_counter_read(struct perf_counter *counter)
2607 {
2608         cpu_clock_perf_counter_update(counter);
2609 }
2610
2611 static const struct pmu perf_ops_cpu_clock = {
2612         .enable         = cpu_clock_perf_counter_enable,
2613         .disable        = cpu_clock_perf_counter_disable,
2614         .read           = cpu_clock_perf_counter_read,
2615 };
2616
2617 /*
2618  * Software counter: task time clock
2619  */
2620
2621 static void task_clock_perf_counter_update(struct perf_counter *counter, u64 now)
2622 {
2623         u64 prev;
2624         s64 delta;
2625
2626         prev = atomic64_xchg(&counter->hw.prev_count, now);
2627         delta = now - prev;
2628         atomic64_add(delta, &counter->count);
2629 }
2630
2631 static int task_clock_perf_counter_enable(struct perf_counter *counter)
2632 {
2633         struct hw_perf_counter *hwc = &counter->hw;
2634         u64 now;
2635
2636         now = counter->ctx->time;
2637
2638         atomic64_set(&hwc->prev_count, now);
2639         hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2640         hwc->hrtimer.function = perf_swcounter_hrtimer;
2641         if (hwc->irq_period) {
2642                 __hrtimer_start_range_ns(&hwc->hrtimer,
2643                                 ns_to_ktime(hwc->irq_period), 0,
2644                                 HRTIMER_MODE_REL, 0);
2645         }
2646
2647         return 0;
2648 }
2649
2650 static void task_clock_perf_counter_disable(struct perf_counter *counter)
2651 {
2652         hrtimer_cancel(&counter->hw.hrtimer);
2653         task_clock_perf_counter_update(counter, counter->ctx->time);
2654
2655 }
2656
2657 static void task_clock_perf_counter_read(struct perf_counter *counter)
2658 {
2659         u64 time;
2660
2661         if (!in_nmi()) {
2662                 update_context_time(counter->ctx);
2663                 time = counter->ctx->time;
2664         } else {
2665                 u64 now = perf_clock();
2666                 u64 delta = now - counter->ctx->timestamp;
2667                 time = counter->ctx->time + delta;
2668         }
2669
2670         task_clock_perf_counter_update(counter, time);
2671 }
2672
2673 static const struct pmu perf_ops_task_clock = {
2674         .enable         = task_clock_perf_counter_enable,
2675         .disable        = task_clock_perf_counter_disable,
2676         .read           = task_clock_perf_counter_read,
2677 };
2678
2679 /*
2680  * Software counter: cpu migrations
2681  */
2682
2683 static inline u64 get_cpu_migrations(struct perf_counter *counter)
2684 {
2685         struct task_struct *curr = counter->ctx->task;
2686
2687         if (curr)
2688                 return curr->se.nr_migrations;
2689         return cpu_nr_migrations(smp_processor_id());
2690 }
2691
2692 static void cpu_migrations_perf_counter_update(struct perf_counter *counter)
2693 {
2694         u64 prev, now;
2695         s64 delta;
2696
2697         prev = atomic64_read(&counter->hw.prev_count);
2698         now = get_cpu_migrations(counter);
2699
2700         atomic64_set(&counter->hw.prev_count, now);
2701
2702         delta = now - prev;
2703
2704         atomic64_add(delta, &counter->count);
2705 }
2706
2707 static void cpu_migrations_perf_counter_read(struct perf_counter *counter)
2708 {
2709         cpu_migrations_perf_counter_update(counter);
2710 }
2711
2712 static int cpu_migrations_perf_counter_enable(struct perf_counter *counter)
2713 {
2714         if (counter->prev_state <= PERF_COUNTER_STATE_OFF)
2715                 atomic64_set(&counter->hw.prev_count,
2716                              get_cpu_migrations(counter));
2717         return 0;
2718 }
2719
2720 static void cpu_migrations_perf_counter_disable(struct perf_counter *counter)
2721 {
2722         cpu_migrations_perf_counter_update(counter);
2723 }
2724
2725 static const struct pmu perf_ops_cpu_migrations = {
2726         .enable         = cpu_migrations_perf_counter_enable,
2727         .disable        = cpu_migrations_perf_counter_disable,
2728         .read           = cpu_migrations_perf_counter_read,
2729 };
2730
2731 #ifdef CONFIG_EVENT_PROFILE
2732 void perf_tpcounter_event(int event_id)
2733 {
2734         struct pt_regs *regs = get_irq_regs();
2735
2736         if (!regs)
2737                 regs = task_pt_regs(current);
2738
2739         __perf_swcounter_event(PERF_TYPE_TRACEPOINT, event_id, 1, 1, regs, 0);
2740 }
2741 EXPORT_SYMBOL_GPL(perf_tpcounter_event);
2742
2743 extern int ftrace_profile_enable(int);
2744 extern void ftrace_profile_disable(int);
2745
2746 static void tp_perf_counter_destroy(struct perf_counter *counter)
2747 {
2748         ftrace_profile_disable(perf_event_id(&counter->hw_event));
2749 }
2750
2751 static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
2752 {
2753         int event_id = perf_event_id(&counter->hw_event);
2754         int ret;
2755
2756         ret = ftrace_profile_enable(event_id);
2757         if (ret)
2758                 return NULL;
2759
2760         counter->destroy = tp_perf_counter_destroy;
2761         counter->hw.irq_period = counter->hw_event.irq_period;
2762
2763         return &perf_ops_generic;
2764 }
2765 #else
2766 static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
2767 {
2768         return NULL;
2769 }
2770 #endif
2771
2772 static const struct pmu *sw_perf_counter_init(struct perf_counter *counter)
2773 {
2774         struct perf_counter_hw_event *hw_event = &counter->hw_event;
2775         const struct pmu *pmu = NULL;
2776         struct hw_perf_counter *hwc = &counter->hw;
2777
2778         /*
2779          * Software counters (currently) can't in general distinguish
2780          * between user, kernel and hypervisor events.
2781          * However, context switches and cpu migrations are considered
2782          * to be kernel events, and page faults are never hypervisor
2783          * events.
2784          */
2785         switch (perf_event_id(&counter->hw_event)) {
2786         case PERF_COUNT_CPU_CLOCK:
2787                 pmu = &perf_ops_cpu_clock;
2788
2789                 if (hw_event->irq_period && hw_event->irq_period < 10000)
2790                         hw_event->irq_period = 10000;
2791                 break;
2792         case PERF_COUNT_TASK_CLOCK:
2793                 /*
2794                  * If the user instantiates this as a per-cpu counter,
2795                  * use the cpu_clock counter instead.
2796                  */
2797                 if (counter->ctx->task)
2798                         pmu = &perf_ops_task_clock;
2799                 else
2800                         pmu = &perf_ops_cpu_clock;
2801
2802                 if (hw_event->irq_period && hw_event->irq_period < 10000)
2803                         hw_event->irq_period = 10000;
2804                 break;
2805         case PERF_COUNT_PAGE_FAULTS:
2806         case PERF_COUNT_PAGE_FAULTS_MIN:
2807         case PERF_COUNT_PAGE_FAULTS_MAJ:
2808         case PERF_COUNT_CONTEXT_SWITCHES:
2809                 pmu = &perf_ops_generic;
2810                 break;
2811         case PERF_COUNT_CPU_MIGRATIONS:
2812                 if (!counter->hw_event.exclude_kernel)
2813                         pmu = &perf_ops_cpu_migrations;
2814                 break;
2815         }
2816
2817         if (pmu)
2818                 hwc->irq_period = hw_event->irq_period;
2819
2820         return pmu;
2821 }
2822
2823 /*
2824  * Allocate and initialize a counter structure
2825  */
2826 static struct perf_counter *
2827 perf_counter_alloc(struct perf_counter_hw_event *hw_event,
2828                    int cpu,
2829                    struct perf_counter_context *ctx,
2830                    struct perf_counter *group_leader,
2831                    gfp_t gfpflags)
2832 {
2833         const struct pmu *pmu;
2834         struct perf_counter *counter;
2835         long err;
2836
2837         counter = kzalloc(sizeof(*counter), gfpflags);
2838         if (!counter)
2839                 return ERR_PTR(-ENOMEM);
2840
2841         /*
2842          * Single counters are their own group leaders, with an
2843          * empty sibling list:
2844          */
2845         if (!group_leader)
2846                 group_leader = counter;
2847
2848         mutex_init(&counter->mutex);
2849         INIT_LIST_HEAD(&counter->list_entry);
2850         INIT_LIST_HEAD(&counter->event_entry);
2851         INIT_LIST_HEAD(&counter->sibling_list);
2852         init_waitqueue_head(&counter->waitq);
2853
2854         mutex_init(&counter->mmap_mutex);
2855
2856         INIT_LIST_HEAD(&counter->child_list);
2857
2858         counter->cpu                    = cpu;
2859         counter->hw_event               = *hw_event;
2860         counter->group_leader           = group_leader;
2861         counter->pmu                    = NULL;
2862         counter->ctx                    = ctx;
2863
2864         counter->state = PERF_COUNTER_STATE_INACTIVE;
2865         if (hw_event->disabled)
2866                 counter->state = PERF_COUNTER_STATE_OFF;
2867
2868         pmu = NULL;
2869
2870         /*
2871          * we currently do not support PERF_RECORD_GROUP on inherited counters
2872          */
2873         if (hw_event->inherit && (hw_event->record_type & PERF_RECORD_GROUP))
2874                 goto done;
2875
2876         if (perf_event_raw(hw_event)) {
2877                 pmu = hw_perf_counter_init(counter);
2878                 goto done;
2879         }
2880
2881         switch (perf_event_type(hw_event)) {
2882         case PERF_TYPE_HARDWARE:
2883                 pmu = hw_perf_counter_init(counter);
2884                 break;
2885
2886         case PERF_TYPE_SOFTWARE:
2887                 pmu = sw_perf_counter_init(counter);
2888                 break;
2889
2890         case PERF_TYPE_TRACEPOINT:
2891                 pmu = tp_perf_counter_init(counter);
2892                 break;
2893         }
2894 done:
2895         err = 0;
2896         if (!pmu)
2897                 err = -EINVAL;
2898         else if (IS_ERR(pmu))
2899                 err = PTR_ERR(pmu);
2900
2901         if (err) {
2902                 kfree(counter);
2903                 return ERR_PTR(err);
2904         }
2905
2906         counter->pmu = pmu;
2907
2908         atomic_inc(&nr_counters);
2909         if (counter->hw_event.mmap)
2910                 atomic_inc(&nr_mmap_tracking);
2911         if (counter->hw_event.munmap)
2912                 atomic_inc(&nr_munmap_tracking);
2913         if (counter->hw_event.comm)
2914                 atomic_inc(&nr_comm_tracking);
2915
2916         return counter;
2917 }
2918
2919 /**
2920  * sys_perf_counter_open - open a performance counter, associate it to a task/cpu
2921  *
2922  * @hw_event_uptr:      event type attributes for monitoring/sampling
2923  * @pid:                target pid
2924  * @cpu:                target cpu
2925  * @group_fd:           group leader counter fd
2926  */
2927 SYSCALL_DEFINE5(perf_counter_open,
2928                 const struct perf_counter_hw_event __user *, hw_event_uptr,
2929                 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
2930 {
2931         struct perf_counter *counter, *group_leader;
2932         struct perf_counter_hw_event hw_event;
2933         struct perf_counter_context *ctx;
2934         struct file *counter_file = NULL;
2935         struct file *group_file = NULL;
2936         int fput_needed = 0;
2937         int fput_needed2 = 0;
2938         int ret;
2939
2940         /* for future expandability... */
2941         if (flags)
2942                 return -EINVAL;
2943
2944         if (copy_from_user(&hw_event, hw_event_uptr, sizeof(hw_event)) != 0)
2945                 return -EFAULT;
2946
2947         /*
2948          * Get the target context (task or percpu):
2949          */
2950         ctx = find_get_context(pid, cpu);
2951         if (IS_ERR(ctx))
2952                 return PTR_ERR(ctx);
2953
2954         /*
2955          * Look up the group leader (we will attach this counter to it):
2956          */
2957         group_leader = NULL;
2958         if (group_fd != -1) {
2959                 ret = -EINVAL;
2960                 group_file = fget_light(group_fd, &fput_needed);
2961                 if (!group_file)
2962                         goto err_put_context;
2963                 if (group_file->f_op != &perf_fops)
2964                         goto err_put_context;
2965
2966                 group_leader = group_file->private_data;
2967                 /*
2968                  * Do not allow a recursive hierarchy (this new sibling
2969                  * becoming part of another group-sibling):
2970                  */
2971                 if (group_leader->group_leader != group_leader)
2972                         goto err_put_context;
2973                 /*
2974                  * Do not allow to attach to a group in a different
2975                  * task or CPU context:
2976                  */
2977                 if (group_leader->ctx != ctx)
2978                         goto err_put_context;
2979                 /*
2980                  * Only a group leader can be exclusive or pinned
2981                  */
2982                 if (hw_event.exclusive || hw_event.pinned)
2983                         goto err_put_context;
2984         }
2985
2986         counter = perf_counter_alloc(&hw_event, cpu, ctx, group_leader,
2987                                      GFP_KERNEL);
2988         ret = PTR_ERR(counter);
2989         if (IS_ERR(counter))
2990                 goto err_put_context;
2991
2992         ret = anon_inode_getfd("[perf_counter]", &perf_fops, counter, 0);
2993         if (ret < 0)
2994                 goto err_free_put_context;
2995
2996         counter_file = fget_light(ret, &fput_needed2);
2997         if (!counter_file)
2998                 goto err_free_put_context;
2999
3000         counter->filp = counter_file;
3001         mutex_lock(&ctx->mutex);
3002         perf_install_in_context(ctx, counter, cpu);
3003         mutex_unlock(&ctx->mutex);
3004
3005         fput_light(counter_file, fput_needed2);
3006
3007 out_fput:
3008         fput_light(group_file, fput_needed);
3009
3010         return ret;
3011
3012 err_free_put_context:
3013         kfree(counter);
3014
3015 err_put_context:
3016         put_context(ctx);
3017
3018         goto out_fput;
3019 }
3020
3021 /*
3022  * Initialize the perf_counter context in a task_struct:
3023  */
3024 static void
3025 __perf_counter_init_context(struct perf_counter_context *ctx,
3026                             struct task_struct *task)
3027 {
3028         memset(ctx, 0, sizeof(*ctx));
3029         spin_lock_init(&ctx->lock);
3030         mutex_init(&ctx->mutex);
3031         INIT_LIST_HEAD(&ctx->counter_list);
3032         INIT_LIST_HEAD(&ctx->event_list);
3033         ctx->task = task;
3034 }
3035
3036 /*
3037  * inherit a counter from parent task to child task:
3038  */
3039 static struct perf_counter *
3040 inherit_counter(struct perf_counter *parent_counter,
3041               struct task_struct *parent,
3042               struct perf_counter_context *parent_ctx,
3043               struct task_struct *child,
3044               struct perf_counter *group_leader,
3045               struct perf_counter_context *child_ctx)
3046 {
3047         struct perf_counter *child_counter;
3048
3049         /*
3050          * Instead of creating recursive hierarchies of counters,
3051          * we link inherited counters back to the original parent,
3052          * which has a filp for sure, which we use as the reference
3053          * count:
3054          */
3055         if (parent_counter->parent)
3056                 parent_counter = parent_counter->parent;
3057
3058         child_counter = perf_counter_alloc(&parent_counter->hw_event,
3059                                            parent_counter->cpu, child_ctx,
3060                                            group_leader, GFP_KERNEL);
3061         if (IS_ERR(child_counter))
3062                 return child_counter;
3063
3064         /*
3065          * Link it up in the child's context:
3066          */
3067         child_counter->task = child;
3068         add_counter_to_ctx(child_counter, child_ctx);
3069
3070         child_counter->parent = parent_counter;
3071         /*
3072          * inherit into child's child as well:
3073          */
3074         child_counter->hw_event.inherit = 1;
3075
3076         /*
3077          * Get a reference to the parent filp - we will fput it
3078          * when the child counter exits. This is safe to do because
3079          * we are in the parent and we know that the filp still
3080          * exists and has a nonzero count:
3081          */
3082         atomic_long_inc(&parent_counter->filp->f_count);
3083
3084         /*
3085          * Link this into the parent counter's child list
3086          */
3087         mutex_lock(&parent_counter->mutex);
3088         list_add_tail(&child_counter->child_list, &parent_counter->child_list);
3089
3090         /*
3091          * Make the child state follow the state of the parent counter,
3092          * not its hw_event.disabled bit.  We hold the parent's mutex,
3093          * so we won't race with perf_counter_{en,dis}able_family.
3094          */
3095         if (parent_counter->state >= PERF_COUNTER_STATE_INACTIVE)
3096                 child_counter->state = PERF_COUNTER_STATE_INACTIVE;
3097         else
3098                 child_counter->state = PERF_COUNTER_STATE_OFF;
3099
3100         mutex_unlock(&parent_counter->mutex);
3101
3102         return child_counter;
3103 }
3104
3105 static int inherit_group(struct perf_counter *parent_counter,
3106               struct task_struct *parent,
3107               struct perf_counter_context *parent_ctx,
3108               struct task_struct *child,
3109               struct perf_counter_context *child_ctx)
3110 {
3111         struct perf_counter *leader;
3112         struct perf_counter *sub;
3113         struct perf_counter *child_ctr;
3114
3115         leader = inherit_counter(parent_counter, parent, parent_ctx,
3116                                  child, NULL, child_ctx);
3117         if (IS_ERR(leader))
3118                 return PTR_ERR(leader);
3119         list_for_each_entry(sub, &parent_counter->sibling_list, list_entry) {
3120                 child_ctr = inherit_counter(sub, parent, parent_ctx,
3121                                             child, leader, child_ctx);
3122                 if (IS_ERR(child_ctr))
3123                         return PTR_ERR(child_ctr);
3124         }
3125         return 0;
3126 }
3127
3128 static void sync_child_counter(struct perf_counter *child_counter,
3129                                struct perf_counter *parent_counter)
3130 {
3131         u64 parent_val, child_val;
3132
3133         parent_val = atomic64_read(&parent_counter->count);
3134         child_val = atomic64_read(&child_counter->count);
3135
3136         /*
3137          * Add back the child's count to the parent's count:
3138          */
3139         atomic64_add(child_val, &parent_counter->count);
3140         atomic64_add(child_counter->total_time_enabled,
3141                      &parent_counter->child_total_time_enabled);
3142         atomic64_add(child_counter->total_time_running,
3143                      &parent_counter->child_total_time_running);
3144
3145         /*
3146          * Remove this counter from the parent's list
3147          */
3148         mutex_lock(&parent_counter->mutex);
3149         list_del_init(&child_counter->child_list);
3150         mutex_unlock(&parent_counter->mutex);
3151
3152         /*
3153          * Release the parent counter, if this was the last
3154          * reference to it.
3155          */
3156         fput(parent_counter->filp);
3157 }
3158
3159 static void
3160 __perf_counter_exit_task(struct task_struct *child,
3161                          struct perf_counter *child_counter,
3162                          struct perf_counter_context *child_ctx)
3163 {
3164         struct perf_counter *parent_counter;
3165         struct perf_counter *sub, *tmp;
3166
3167         /*
3168          * If we do not self-reap then we have to wait for the
3169          * child task to unschedule (it will happen for sure),
3170          * so that its counter is at its final count. (This
3171          * condition triggers rarely - child tasks usually get
3172          * off their CPU before the parent has a chance to
3173          * get this far into the reaping action)
3174          */
3175         if (child != current) {
3176                 wait_task_inactive(child, 0);
3177                 list_del_init(&child_counter->list_entry);
3178                 update_counter_times(child_counter);
3179         } else {
3180                 struct perf_cpu_context *cpuctx;
3181                 unsigned long flags;
3182                 u64 perf_flags;
3183
3184                 /*
3185                  * Disable and unlink this counter.
3186                  *
3187                  * Be careful about zapping the list - IRQ/NMI context
3188                  * could still be processing it:
3189                  */
3190                 local_irq_save(flags);
3191                 perf_flags = hw_perf_save_disable();
3192
3193                 cpuctx = &__get_cpu_var(perf_cpu_context);
3194
3195                 group_sched_out(child_counter, cpuctx, child_ctx);
3196                 update_counter_times(child_counter);
3197
3198                 list_del_init(&child_counter->list_entry);
3199
3200                 child_ctx->nr_counters--;
3201
3202                 hw_perf_restore(perf_flags);
3203                 local_irq_restore(flags);
3204         }
3205
3206         parent_counter = child_counter->parent;
3207         /*
3208          * It can happen that parent exits first, and has counters
3209          * that are still around due to the child reference. These
3210          * counters need to be zapped - but otherwise linger.
3211          */
3212         if (parent_counter) {
3213                 sync_child_counter(child_counter, parent_counter);
3214                 list_for_each_entry_safe(sub, tmp, &child_counter->sibling_list,
3215                                          list_entry) {
3216                         if (sub->parent) {
3217                                 sync_child_counter(sub, sub->parent);
3218                                 free_counter(sub);
3219                         }
3220                 }
3221                 free_counter(child_counter);
3222         }
3223 }
3224
3225 /*
3226  * When a child task exits, feed back counter values to parent counters.
3227  *
3228  * Note: we may be running in child context, but the PID is not hashed
3229  * anymore so new counters will not be added.
3230  */
3231 void perf_counter_exit_task(struct task_struct *child)
3232 {
3233         struct perf_counter *child_counter, *tmp;
3234         struct perf_counter_context *child_ctx;
3235
3236         child_ctx = &child->perf_counter_ctx;
3237
3238         if (likely(!child_ctx->nr_counters))
3239                 return;
3240
3241         list_for_each_entry_safe(child_counter, tmp, &child_ctx->counter_list,
3242                                  list_entry)
3243                 __perf_counter_exit_task(child, child_counter, child_ctx);
3244 }
3245
3246 /*
3247  * Initialize the perf_counter context in task_struct
3248  */
3249 void perf_counter_init_task(struct task_struct *child)
3250 {
3251         struct perf_counter_context *child_ctx, *parent_ctx;
3252         struct perf_counter *counter;
3253         struct task_struct *parent = current;
3254
3255         child_ctx  =  &child->perf_counter_ctx;
3256         parent_ctx = &parent->perf_counter_ctx;
3257
3258         __perf_counter_init_context(child_ctx, child);
3259
3260         /*
3261          * This is executed from the parent task context, so inherit
3262          * counters that have been marked for cloning:
3263          */
3264
3265         if (likely(!parent_ctx->nr_counters))
3266                 return;
3267
3268         /*
3269          * Lock the parent list. No need to lock the child - not PID
3270          * hashed yet and not running, so nobody can access it.
3271          */
3272         mutex_lock(&parent_ctx->mutex);
3273
3274         /*
3275          * We dont have to disable NMIs - we are only looking at
3276          * the list, not manipulating it:
3277          */
3278         list_for_each_entry(counter, &parent_ctx->counter_list, list_entry) {
3279                 if (!counter->hw_event.inherit)
3280                         continue;
3281
3282                 if (inherit_group(counter, parent,
3283                                   parent_ctx, child, child_ctx))
3284                         break;
3285         }
3286
3287         mutex_unlock(&parent_ctx->mutex);
3288 }
3289
3290 static void __cpuinit perf_counter_init_cpu(int cpu)
3291 {
3292         struct perf_cpu_context *cpuctx;
3293
3294         cpuctx = &per_cpu(perf_cpu_context, cpu);
3295         __perf_counter_init_context(&cpuctx->ctx, NULL);
3296
3297         spin_lock(&perf_resource_lock);
3298         cpuctx->max_pertask = perf_max_counters - perf_reserved_percpu;
3299         spin_unlock(&perf_resource_lock);
3300
3301         hw_perf_counter_setup(cpu);
3302 }
3303
3304 #ifdef CONFIG_HOTPLUG_CPU
3305 static void __perf_counter_exit_cpu(void *info)
3306 {
3307         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
3308         struct perf_counter_context *ctx = &cpuctx->ctx;
3309         struct perf_counter *counter, *tmp;
3310
3311         list_for_each_entry_safe(counter, tmp, &ctx->counter_list, list_entry)
3312                 __perf_counter_remove_from_context(counter);
3313 }
3314 static void perf_counter_exit_cpu(int cpu)
3315 {
3316         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
3317         struct perf_counter_context *ctx = &cpuctx->ctx;
3318
3319         mutex_lock(&ctx->mutex);
3320         smp_call_function_single(cpu, __perf_counter_exit_cpu, NULL, 1);
3321         mutex_unlock(&ctx->mutex);
3322 }
3323 #else
3324 static inline void perf_counter_exit_cpu(int cpu) { }
3325 #endif
3326
3327 static int __cpuinit
3328 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
3329 {
3330         unsigned int cpu = (long)hcpu;
3331
3332         switch (action) {
3333
3334         case CPU_UP_PREPARE:
3335         case CPU_UP_PREPARE_FROZEN:
3336                 perf_counter_init_cpu(cpu);
3337                 break;
3338
3339         case CPU_DOWN_PREPARE:
3340         case CPU_DOWN_PREPARE_FROZEN:
3341                 perf_counter_exit_cpu(cpu);
3342                 break;
3343
3344         default:
3345                 break;
3346         }
3347
3348         return NOTIFY_OK;
3349 }
3350
3351 static struct notifier_block __cpuinitdata perf_cpu_nb = {
3352         .notifier_call          = perf_cpu_notify,
3353 };
3354
3355 void __init perf_counter_init(void)
3356 {
3357         perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
3358                         (void *)(long)smp_processor_id());
3359         register_cpu_notifier(&perf_cpu_nb);
3360 }
3361
3362 static ssize_t perf_show_reserve_percpu(struct sysdev_class *class, char *buf)
3363 {
3364         return sprintf(buf, "%d\n", perf_reserved_percpu);
3365 }
3366
3367 static ssize_t
3368 perf_set_reserve_percpu(struct sysdev_class *class,
3369                         const char *buf,
3370                         size_t count)
3371 {
3372         struct perf_cpu_context *cpuctx;
3373         unsigned long val;
3374         int err, cpu, mpt;
3375
3376         err = strict_strtoul(buf, 10, &val);
3377         if (err)
3378                 return err;
3379         if (val > perf_max_counters)
3380                 return -EINVAL;
3381
3382         spin_lock(&perf_resource_lock);
3383         perf_reserved_percpu = val;
3384         for_each_online_cpu(cpu) {
3385                 cpuctx = &per_cpu(perf_cpu_context, cpu);
3386                 spin_lock_irq(&cpuctx->ctx.lock);
3387                 mpt = min(perf_max_counters - cpuctx->ctx.nr_counters,
3388                           perf_max_counters - perf_reserved_percpu);
3389                 cpuctx->max_pertask = mpt;
3390                 spin_unlock_irq(&cpuctx->ctx.lock);
3391         }
3392         spin_unlock(&perf_resource_lock);
3393
3394         return count;
3395 }
3396
3397 static ssize_t perf_show_overcommit(struct sysdev_class *class, char *buf)
3398 {
3399         return sprintf(buf, "%d\n", perf_overcommit);
3400 }
3401
3402 static ssize_t
3403 perf_set_overcommit(struct sysdev_class *class, const char *buf, size_t count)
3404 {
3405         unsigned long val;
3406         int err;
3407
3408         err = strict_strtoul(buf, 10, &val);
3409         if (err)
3410                 return err;
3411         if (val > 1)
3412                 return -EINVAL;
3413
3414         spin_lock(&perf_resource_lock);
3415         perf_overcommit = val;
3416         spin_unlock(&perf_resource_lock);
3417
3418         return count;
3419 }
3420
3421 static SYSDEV_CLASS_ATTR(
3422                                 reserve_percpu,
3423                                 0644,
3424                                 perf_show_reserve_percpu,
3425                                 perf_set_reserve_percpu
3426                         );
3427
3428 static SYSDEV_CLASS_ATTR(
3429                                 overcommit,
3430                                 0644,
3431                                 perf_show_overcommit,
3432                                 perf_set_overcommit
3433                         );
3434
3435 static struct attribute *perfclass_attrs[] = {
3436         &attr_reserve_percpu.attr,
3437         &attr_overcommit.attr,
3438         NULL
3439 };
3440
3441 static struct attribute_group perfclass_attr_group = {
3442         .attrs                  = perfclass_attrs,
3443         .name                   = "perf_counters",
3444 };
3445
3446 static int __init perf_counter_sysfs_init(void)
3447 {
3448         return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
3449                                   &perfclass_attr_group);
3450 }
3451 device_initcall(perf_counter_sysfs_init);