kernel/perf_counter.c

   1 /*
   2  * Performance counter core code
   3  *
   4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
   6  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
   7  *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
   8  *
   9  *  For licensing details see kernel-base/COPYING
  10  */
  11
  12 #include <linux/fs.h>
  13 #include <linux/mm.h>
  14 #include <linux/cpu.h>
  15 #include <linux/smp.h>
  16 #include <linux/file.h>
  17 #include <linux/poll.h>
  18 #include <linux/sysfs.h>
  19 #include <linux/ptrace.h>
  20 #include <linux/percpu.h>
  21 #include <linux/vmstat.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/rculist.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/syscalls.h>
  26 #include <linux/anon_inodes.h>
  27 #include <linux/kernel_stat.h>
  28 #include <linux/perf_counter.h>
  29 #include <linux/dcache.h>
  30
  31 #include <asm/irq_regs.h>
  32
  33 /*
  34  * Each CPU has a list of per CPU counters:
  35  */
  36 DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
  37
  38 int perf_max_counters __read_mostly = 1;
  39 static int perf_reserved_percpu __read_mostly;
  40 static int perf_overcommit __read_mostly = 1;
  41
  42 static atomic_t nr_mmap_tracking __read_mostly;
  43 static atomic_t nr_munmap_tracking __read_mostly;
  44 static atomic_t nr_comm_tracking __read_mostly;
  45
  46 int sysctl_perf_counter_priv __read_mostly; /* do we need to be privileged */
  47 int sysctl_perf_counter_mlock __read_mostly = 128; /* 'free' kb per counter */
  48
  49 /*
  50  * Lock for (sysadmin-configurable) counter reservations:
  51  */
  52 static DEFINE_SPINLOCK(perf_resource_lock);
  53
  54 /*
  55  * Architecture provided APIs - weak aliases:
  56  */
  57 extern __weak const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
  58 {
  59         return NULL;
  60 }
  61
  62 u64 __weak hw_perf_save_disable(void)           { return 0; }
  63 void __weak hw_perf_restore(u64 ctrl)           { barrier(); }
  64 void __weak hw_perf_counter_setup(int cpu)      { barrier(); }
  65 int __weak hw_perf_group_sched_in(struct perf_counter *group_leader,
  66                struct perf_cpu_context *cpuctx,
  67                struct perf_counter_context *ctx, int cpu)
  68 {
  69         return 0;
  70 }
  71
  72 void __weak perf_counter_print_debug(void)      { }
  73
  74 static void
  75 list_add_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
  76 {
  77         struct perf_counter *group_leader = counter->group_leader;
  78
  79         /*
  80          * Depending on whether it is a standalone or sibling counter,
  81          * add it straight to the context's counter list, or to the group
  82          * leader's sibling list:
  83          */
  84         if (counter->group_leader == counter)
  85                 list_add_tail(&counter->list_entry, &ctx->counter_list);
  86         else {
  87                 list_add_tail(&counter->list_entry, &group_leader->sibling_list);
  88                 group_leader->nr_siblings++;
  89         }
  90
  91         list_add_rcu(&counter->event_entry, &ctx->event_list);
  92 }
  93
  94 static void
  95 list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
  96 {
  97         struct perf_counter *sibling, *tmp;
  98
  99         list_del_init(&counter->list_entry);
 100         list_del_rcu(&counter->event_entry);
 101
 102         if (counter->group_leader != counter)
 103                 counter->group_leader->nr_siblings--;
 104
 105         /*
 106          * If this was a group counter with sibling counters then
 107          * upgrade the siblings to singleton counters by adding them
 108          * to the context list directly:
 109          */
 110         list_for_each_entry_safe(sibling, tmp,
 111                                  &counter->sibling_list, list_entry) {
 112
 113                 list_move_tail(&sibling->list_entry, &ctx->counter_list);
 114                 sibling->group_leader = sibling;
 115         }
 116 }
 117
 118 static void
 119 counter_sched_out(struct perf_counter *counter,
 120                   struct perf_cpu_context *cpuctx,
 121                   struct perf_counter_context *ctx)
 122 {
 123         if (counter->state != PERF_COUNTER_STATE_ACTIVE)
 124                 return;
 125
 126         counter->state = PERF_COUNTER_STATE_INACTIVE;
 127         counter->tstamp_stopped = ctx->time;
 128         counter->pmu->disable(counter);
 129         counter->oncpu = -1;
 130
 131         if (!is_software_counter(counter))
 132                 cpuctx->active_oncpu--;
 133         ctx->nr_active--;
 134         if (counter->hw_event.exclusive || !cpuctx->active_oncpu)
 135                 cpuctx->exclusive = 0;
 136 }
 137
 138 static void
 139 group_sched_out(struct perf_counter *group_counter,
 140                 struct perf_cpu_context *cpuctx,
 141                 struct perf_counter_context *ctx)
 142 {
 143         struct perf_counter *counter;
 144
 145         if (group_counter->state != PERF_COUNTER_STATE_ACTIVE)
 146                 return;
 147
 148         counter_sched_out(group_counter, cpuctx, ctx);
 149
 150         /*
 151          * Schedule out siblings (if any):
 152          */
 153         list_for_each_entry(counter, &group_counter->sibling_list, list_entry)
 154                 counter_sched_out(counter, cpuctx, ctx);
 155
 156         if (group_counter->hw_event.exclusive)
 157                 cpuctx->exclusive = 0;
 158 }
 159
 160 /*
 161  * Cross CPU call to remove a performance counter
 162  *
 163  * We disable the counter on the hardware level first. After that we
 164  * remove it from the context list.
 165  */
 166 static void __perf_counter_remove_from_context(void *info)
 167 {
 168         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 169         struct perf_counter *counter = info;
 170         struct perf_counter_context *ctx = counter->ctx;
 171         unsigned long flags;
 172         u64 perf_flags;
 173
 174         /*
 175          * If this is a task context, we need to check whether it is
 176          * the current task context of this cpu. If not it has been
 177          * scheduled out before the smp call arrived.
 178          */
 179         if (ctx->task && cpuctx->task_ctx != ctx)
 180                 return;
 181
 182         spin_lock_irqsave(&ctx->lock, flags);
 183
 184         counter_sched_out(counter, cpuctx, ctx);
 185
 186         counter->task = NULL;
 187         ctx->nr_counters--;
 188
 189         /*
 190          * Protect the list operation against NMI by disabling the
 191          * counters on a global level. NOP for non NMI based counters.
 192          */
 193         perf_flags = hw_perf_save_disable();
 194         list_del_counter(counter, ctx);
 195         hw_perf_restore(perf_flags);
 196
 197         if (!ctx->task) {
 198                 /*
 199                  * Allow more per task counters with respect to the
 200                  * reservation:
 201                  */
 202                 cpuctx->max_pertask =
 203                         min(perf_max_counters - ctx->nr_counters,
 204                             perf_max_counters - perf_reserved_percpu);
 205         }
 206
 207         spin_unlock_irqrestore(&ctx->lock, flags);
 208 }
 209
 210
 211 /*
 212  * Remove the counter from a task's (or a CPU's) list of counters.
 213  *
 214  * Must be called with counter->mutex and ctx->mutex held.
 215  *
 216  * CPU counters are removed with a smp call. For task counters we only
 217  * call when the task is on a CPU.
 218  */
 219 static void perf_counter_remove_from_context(struct perf_counter *counter)
 220 {
 221         struct perf_counter_context *ctx = counter->ctx;
 222         struct task_struct *task = ctx->task;
 223
 224         if (!task) {
 225                 /*
 226                  * Per cpu counters are removed via an smp call and
 227                  * the removal is always sucessful.
 228                  */
 229                 smp_call_function_single(counter->cpu,
 230                                          __perf_counter_remove_from_context,
 231                                          counter, 1);
 232                 return;
 233         }
 234
 235 retry:
 236         task_oncpu_function_call(task, __perf_counter_remove_from_context,
 237                                  counter);
 238
 239         spin_lock_irq(&ctx->lock);
 240         /*
 241          * If the context is active we need to retry the smp call.
 242          */
 243         if (ctx->nr_active && !list_empty(&counter->list_entry)) {
 244                 spin_unlock_irq(&ctx->lock);
 245                 goto retry;
 246         }
 247
 248         /*
 249          * The lock prevents that this context is scheduled in so we
 250          * can remove the counter safely, if the call above did not
 251          * succeed.
 252          */
 253         if (!list_empty(&counter->list_entry)) {
 254                 ctx->nr_counters--;
 255                 list_del_counter(counter, ctx);
 256                 counter->task = NULL;
 257         }
 258         spin_unlock_irq(&ctx->lock);
 259 }
 260
 261 static inline u64 perf_clock(void)
 262 {
 263         return cpu_clock(smp_processor_id());
 264 }
 265
 266 /*
 267  * Update the record of the current time in a context.
 268  */
 269 static void update_context_time(struct perf_counter_context *ctx)
 270 {
 271         u64 now = perf_clock();
 272
 273         ctx->time += now - ctx->timestamp;
 274         ctx->timestamp = now;
 275 }
 276
 277 /*
 278  * Update the total_time_enabled and total_time_running fields for a counter.
 279  */
 280 static void update_counter_times(struct perf_counter *counter)
 281 {
 282         struct perf_counter_context *ctx = counter->ctx;
 283         u64 run_end;
 284
 285         if (counter->state < PERF_COUNTER_STATE_INACTIVE)
 286                 return;
 287
 288         counter->total_time_enabled = ctx->time - counter->tstamp_enabled;
 289
 290         if (counter->state == PERF_COUNTER_STATE_INACTIVE)
 291                 run_end = counter->tstamp_stopped;
 292         else
 293                 run_end = ctx->time;
 294
 295         counter->total_time_running = run_end - counter->tstamp_running;
 296 }
 297
 298 /*
 299  * Update total_time_enabled and total_time_running for all counters in a group.
 300  */
 301 static void update_group_times(struct perf_counter *leader)
 302 {
 303         struct perf_counter *counter;
 304
 305         update_counter_times(leader);
 306         list_for_each_entry(counter, &leader->sibling_list, list_entry)
 307                 update_counter_times(counter);
 308 }
 309
 310 /*
 311  * Cross CPU call to disable a performance counter
 312  */
 313 static void __perf_counter_disable(void *info)
 314 {
 315         struct perf_counter *counter = info;
 316         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 317         struct perf_counter_context *ctx = counter->ctx;
 318         unsigned long flags;
 319
 320         /*
 321          * If this is a per-task counter, need to check whether this
 322          * counter's task is the current task on this cpu.
 323          */
 324         if (ctx->task && cpuctx->task_ctx != ctx)
 325                 return;
 326
 327         spin_lock_irqsave(&ctx->lock, flags);
 328
 329         /*
 330          * If the counter is on, turn it off.
 331          * If it is in error state, leave it in error state.
 332          */
 333         if (counter->state >= PERF_COUNTER_STATE_INACTIVE) {
 334                 update_context_time(ctx);
 335                 update_counter_times(counter);
 336                 if (counter == counter->group_leader)
 337                         group_sched_out(counter, cpuctx, ctx);
 338                 else
 339                         counter_sched_out(counter, cpuctx, ctx);
 340                 counter->state = PERF_COUNTER_STATE_OFF;
 341         }
 342
 343         spin_unlock_irqrestore(&ctx->lock, flags);
 344 }
 345
 346 /*
 347  * Disable a counter.
 348  */
 349 static void perf_counter_disable(struct perf_counter *counter)
 350 {
 351         struct perf_counter_context *ctx = counter->ctx;
 352         struct task_struct *task = ctx->task;
 353
 354         if (!task) {
 355                 /*
 356                  * Disable the counter on the cpu that it's on
 357                  */
 358                 smp_call_function_single(counter->cpu, __perf_counter_disable,
 359                                          counter, 1);
 360                 return;
 361         }
 362
 363  retry:
 364         task_oncpu_function_call(task, __perf_counter_disable, counter);
 365
 366         spin_lock_irq(&ctx->lock);
 367         /*
 368          * If the counter is still active, we need to retry the cross-call.
 369          */
 370         if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
 371                 spin_unlock_irq(&ctx->lock);
 372                 goto retry;
 373         }
 374
 375         /*
 376          * Since we have the lock this context can't be scheduled
 377          * in, so we can change the state safely.
 378          */
 379         if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
 380                 update_counter_times(counter);
 381                 counter->state = PERF_COUNTER_STATE_OFF;
 382         }
 383
 384         spin_unlock_irq(&ctx->lock);
 385 }
 386
 387 /*
 388  * Disable a counter and all its children.
 389  */
 390 static void perf_counter_disable_family(struct perf_counter *counter)
 391 {
 392         struct perf_counter *child;
 393
 394         perf_counter_disable(counter);
 395
 396         /*
 397          * Lock the mutex to protect the list of children
 398          */
 399         mutex_lock(&counter->mutex);
 400         list_for_each_entry(child, &counter->child_list, child_list)
 401                 perf_counter_disable(child);
 402         mutex_unlock(&counter->mutex);
 403 }
 404
 405 static int
 406 counter_sched_in(struct perf_counter *counter,
 407                  struct perf_cpu_context *cpuctx,
 408                  struct perf_counter_context *ctx,
 409                  int cpu)
 410 {
 411         if (counter->state <= PERF_COUNTER_STATE_OFF)
 412                 return 0;
 413
 414         counter->state = PERF_COUNTER_STATE_ACTIVE;
 415         counter->oncpu = cpu;   /* TODO: put 'cpu' into cpuctx->cpu */
 416         /*
 417          * The new state must be visible before we turn it on in the hardware:
 418          */
 419         smp_wmb();
 420
 421         if (counter->pmu->enable(counter)) {
 422                 counter->state = PERF_COUNTER_STATE_INACTIVE;
 423                 counter->oncpu = -1;
 424                 return -EAGAIN;
 425         }
 426
 427         counter->tstamp_running += ctx->time - counter->tstamp_stopped;
 428
 429         if (!is_software_counter(counter))
 430                 cpuctx->active_oncpu++;
 431         ctx->nr_active++;
 432
 433         if (counter->hw_event.exclusive)
 434                 cpuctx->exclusive = 1;
 435
 436         return 0;
 437 }
 438
 439 /*
 440  * Return 1 for a group consisting entirely of software counters,
 441  * 0 if the group contains any hardware counters.
 442  */
 443 static int is_software_only_group(struct perf_counter *leader)
 444 {
 445         struct perf_counter *counter;
 446
 447         if (!is_software_counter(leader))
 448                 return 0;
 449
 450         list_for_each_entry(counter, &leader->sibling_list, list_entry)
 451                 if (!is_software_counter(counter))
 452                         return 0;
 453
 454         return 1;
 455 }
 456
 457 /*
 458  * Work out whether we can put this counter group on the CPU now.
 459  */
 460 static int group_can_go_on(struct perf_counter *counter,
 461                            struct perf_cpu_context *cpuctx,
 462                            int can_add_hw)
 463 {
 464         /*
 465          * Groups consisting entirely of software counters can always go on.
 466          */
 467         if (is_software_only_group(counter))
 468                 return 1;
 469         /*
 470          * If an exclusive group is already on, no other hardware
 471          * counters can go on.
 472          */
 473         if (cpuctx->exclusive)
 474                 return 0;
 475         /*
 476          * If this group is exclusive and there are already
 477          * counters on the CPU, it can't go on.
 478          */
 479         if (counter->hw_event.exclusive && cpuctx->active_oncpu)
 480                 return 0;
 481         /*
 482          * Otherwise, try to add it if all previous groups were able
 483          * to go on.
 484          */
 485         return can_add_hw;
 486 }
 487
 488 static void add_counter_to_ctx(struct perf_counter *counter,
 489                                struct perf_counter_context *ctx)
 490 {
 491         list_add_counter(counter, ctx);
 492         ctx->nr_counters++;
 493         counter->prev_state = PERF_COUNTER_STATE_OFF;
 494         counter->tstamp_enabled = ctx->time;
 495         counter->tstamp_running = ctx->time;
 496         counter->tstamp_stopped = ctx->time;
 497 }
 498
 499 /*
 500  * Cross CPU call to install and enable a performance counter
 501  */
 502 static void __perf_install_in_context(void *info)
 503 {
 504         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 505         struct perf_counter *counter = info;
 506         struct perf_counter_context *ctx = counter->ctx;
 507         struct perf_counter *leader = counter->group_leader;
 508         int cpu = smp_processor_id();
 509         unsigned long flags;
 510         u64 perf_flags;
 511         int err;
 512
 513         /*
 514          * If this is a task context, we need to check whether it is
 515          * the current task context of this cpu. If not it has been
 516          * scheduled out before the smp call arrived.
 517          */
 518         if (ctx->task && cpuctx->task_ctx != ctx)
 519                 return;
 520
 521         spin_lock_irqsave(&ctx->lock, flags);
 522         update_context_time(ctx);
 523
 524         /*
 525          * Protect the list operation against NMI by disabling the
 526          * counters on a global level. NOP for non NMI based counters.
 527          */
 528         perf_flags = hw_perf_save_disable();
 529
 530         add_counter_to_ctx(counter, ctx);
 531
 532         /*
 533          * Don't put the counter on if it is disabled or if
 534          * it is in a group and the group isn't on.
 535          */
 536         if (counter->state != PERF_COUNTER_STATE_INACTIVE ||
 537             (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE))
 538                 goto unlock;
 539
 540         /*
 541          * An exclusive counter can't go on if there are already active
 542          * hardware counters, and no hardware counter can go on if there
 543          * is already an exclusive counter on.
 544          */
 545         if (!group_can_go_on(counter, cpuctx, 1))
 546                 err = -EEXIST;
 547         else
 548                 err = counter_sched_in(counter, cpuctx, ctx, cpu);
 549
 550         if (err) {
 551                 /*
 552                  * This counter couldn't go on.  If it is in a group
 553                  * then we have to pull the whole group off.
 554                  * If the counter group is pinned then put it in error state.
 555                  */
 556                 if (leader != counter)
 557                         group_sched_out(leader, cpuctx, ctx);
 558                 if (leader->hw_event.pinned) {
 559                         update_group_times(leader);
 560                         leader->state = PERF_COUNTER_STATE_ERROR;
 561                 }
 562         }
 563
 564         if (!err && !ctx->task && cpuctx->max_pertask)
 565                 cpuctx->max_pertask--;
 566
 567  unlock:
 568         hw_perf_restore(perf_flags);
 569
 570         spin_unlock_irqrestore(&ctx->lock, flags);
 571 }
 572
 573 /*
 574  * Attach a performance counter to a context
 575  *
 576  * First we add the counter to the list with the hardware enable bit
 577  * in counter->hw_config cleared.
 578  *
 579  * If the counter is attached to a task which is on a CPU we use a smp
 580  * call to enable it in the task context. The task might have been
 581  * scheduled away, but we check this in the smp call again.
 582  *
 583  * Must be called with ctx->mutex held.
 584  */
 585 static void
 586 perf_install_in_context(struct perf_counter_context *ctx,
 587                         struct perf_counter *counter,
 588                         int cpu)
 589 {
 590         struct task_struct *task = ctx->task;
 591
 592         if (!task) {
 593                 /*
 594                  * Per cpu counters are installed via an smp call and
 595                  * the install is always sucessful.
 596                  */
 597                 smp_call_function_single(cpu, __perf_install_in_context,
 598                                          counter, 1);
 599                 return;
 600         }
 601
 602         counter->task = task;
 603 retry:
 604         task_oncpu_function_call(task, __perf_install_in_context,
 605                                  counter);
 606
 607         spin_lock_irq(&ctx->lock);
 608         /*
 609          * we need to retry the smp call.
 610          */
 611         if (ctx->is_active && list_empty(&counter->list_entry)) {
 612                 spin_unlock_irq(&ctx->lock);
 613                 goto retry;
 614         }
 615
 616         /*
 617          * The lock prevents that this context is scheduled in so we
 618          * can add the counter safely, if it the call above did not
 619          * succeed.
 620          */
 621         if (list_empty(&counter->list_entry))
 622                 add_counter_to_ctx(counter, ctx);
 623         spin_unlock_irq(&ctx->lock);
 624 }
 625
 626 /*
 627  * Cross CPU call to enable a performance counter
 628  */
 629 static void __perf_counter_enable(void *info)
 630 {
 631         struct perf_counter *counter = info;
 632         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 633         struct perf_counter_context *ctx = counter->ctx;
 634         struct perf_counter *leader = counter->group_leader;
 635         unsigned long flags;
 636         int err;
 637
 638         /*
 639          * If this is a per-task counter, need to check whether this
 640          * counter's task is the current task on this cpu.
 641          */
 642         if (ctx->task && cpuctx->task_ctx != ctx)
 643                 return;
 644
 645         spin_lock_irqsave(&ctx->lock, flags);
 646         update_context_time(ctx);
 647
 648         counter->prev_state = counter->state;
 649         if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
 650                 goto unlock;
 651         counter->state = PERF_COUNTER_STATE_INACTIVE;
 652         counter->tstamp_enabled = ctx->time - counter->total_time_enabled;
 653
 654         /*
 655          * If the counter is in a group and isn't the group leader,
 656          * then don't put it on unless the group is on.
 657          */
 658         if (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE)
 659                 goto unlock;
 660
 661         if (!group_can_go_on(counter, cpuctx, 1))
 662                 err = -EEXIST;
 663         else
 664                 err = counter_sched_in(counter, cpuctx, ctx,
 665                                        smp_processor_id());
 666
 667         if (err) {
 668                 /*
 669                  * If this counter can't go on and it's part of a
 670                  * group, then the whole group has to come off.
 671                  */
 672                 if (leader != counter)
 673                         group_sched_out(leader, cpuctx, ctx);
 674                 if (leader->hw_event.pinned) {
 675                         update_group_times(leader);
 676                         leader->state = PERF_COUNTER_STATE_ERROR;
 677                 }
 678         }
 679
 680  unlock:
 681         spin_unlock_irqrestore(&ctx->lock, flags);
 682 }
 683
 684 /*
 685  * Enable a counter.
 686  */
 687 static void perf_counter_enable(struct perf_counter *counter)
 688 {
 689         struct perf_counter_context *ctx = counter->ctx;
 690         struct task_struct *task = ctx->task;
 691
 692         if (!task) {
 693                 /*
 694                  * Enable the counter on the cpu that it's on
 695                  */
 696                 smp_call_function_single(counter->cpu, __perf_counter_enable,
 697                                          counter, 1);
 698                 return;
 699         }
 700
 701         spin_lock_irq(&ctx->lock);
 702         if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
 703                 goto out;
 704
 705         /*
 706          * If the counter is in error state, clear that first.
 707          * That way, if we see the counter in error state below, we
 708          * know that it has gone back into error state, as distinct
 709          * from the task having been scheduled away before the
 710          * cross-call arrived.
 711          */
 712         if (counter->state == PERF_COUNTER_STATE_ERROR)
 713                 counter->state = PERF_COUNTER_STATE_OFF;
 714
 715  retry:
 716         spin_unlock_irq(&ctx->lock);
 717         task_oncpu_function_call(task, __perf_counter_enable, counter);
 718
 719         spin_lock_irq(&ctx->lock);
 720
 721         /*
 722          * If the context is active and the counter is still off,
 723          * we need to retry the cross-call.
 724          */
 725         if (ctx->is_active && counter->state == PERF_COUNTER_STATE_OFF)
 726                 goto retry;
 727
 728         /*
 729          * Since we have the lock this context can't be scheduled
 730          * in, so we can change the state safely.
 731          */
 732         if (counter->state == PERF_COUNTER_STATE_OFF) {
 733                 counter->state = PERF_COUNTER_STATE_INACTIVE;
 734                 counter->tstamp_enabled =
 735                         ctx->time - counter->total_time_enabled;
 736         }
 737  out:
 738         spin_unlock_irq(&ctx->lock);
 739 }
 740
 741 static int perf_counter_refresh(struct perf_counter *counter, int refresh)
 742 {
 743         /*
 744          * not supported on inherited counters
 745          */
 746         if (counter->hw_event.inherit)
 747                 return -EINVAL;
 748
 749         atomic_add(refresh, &counter->event_limit);
 750         perf_counter_enable(counter);
 751
 752         return 0;
 753 }
 754
 755 /*
 756  * Enable a counter and all its children.
 757  */
 758 static void perf_counter_enable_family(struct perf_counter *counter)
 759 {
 760         struct perf_counter *child;
 761
 762         perf_counter_enable(counter);
 763
 764         /*
 765          * Lock the mutex to protect the list of children
 766          */
 767         mutex_lock(&counter->mutex);
 768         list_for_each_entry(child, &counter->child_list, child_list)
 769                 perf_counter_enable(child);
 770         mutex_unlock(&counter->mutex);
 771 }
 772
 773 void __perf_counter_sched_out(struct perf_counter_context *ctx,
 774                               struct perf_cpu_context *cpuctx)
 775 {
 776         struct perf_counter *counter;
 777         u64 flags;
 778
 779         spin_lock(&ctx->lock);
 780         ctx->is_active = 0;
 781         if (likely(!ctx->nr_counters))
 782                 goto out;
 783         update_context_time(ctx);
 784
 785         flags = hw_perf_save_disable();
 786         if (ctx->nr_active) {
 787                 list_for_each_entry(counter, &ctx->counter_list, list_entry)
 788                         group_sched_out(counter, cpuctx, ctx);
 789         }
 790         hw_perf_restore(flags);
 791  out:
 792         spin_unlock(&ctx->lock);
 793 }
 794
 795 /*
 796  * Called from scheduler to remove the counters of the current task,
 797  * with interrupts disabled.
 798  *
 799  * We stop each counter and update the counter value in counter->count.
 800  *
 801  * This does not protect us against NMI, but disable()
 802  * sets the disabled bit in the control field of counter _before_
 803  * accessing the counter control register. If a NMI hits, then it will
 804  * not restart the counter.
 805  */
 806 void perf_counter_task_sched_out(struct task_struct *task, int cpu)
 807 {
 808         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
 809         struct perf_counter_context *ctx = &task->perf_counter_ctx;
 810         struct pt_regs *regs;
 811
 812         if (likely(!cpuctx->task_ctx))
 813                 return;
 814
 815         update_context_time(ctx);
 816
 817         regs = task_pt_regs(task);
 818         perf_swcounter_event(PERF_COUNT_CONTEXT_SWITCHES, 1, 1, regs, 0);
 819         __perf_counter_sched_out(ctx, cpuctx);
 820
 821         cpuctx->task_ctx = NULL;
 822 }
 823
 824 static void perf_counter_cpu_sched_out(struct perf_cpu_context *cpuctx)
 825 {
 826         __perf_counter_sched_out(&cpuctx->ctx, cpuctx);
 827 }
 828
 829 static int
 830 group_sched_in(struct perf_counter *group_counter,
 831                struct perf_cpu_context *cpuctx,
 832                struct perf_counter_context *ctx,
 833                int cpu)
 834 {
 835         struct perf_counter *counter, *partial_group;
 836         int ret;
 837
 838         if (group_counter->state == PERF_COUNTER_STATE_OFF)
 839                 return 0;
 840
 841         ret = hw_perf_group_sched_in(group_counter, cpuctx, ctx, cpu);
 842         if (ret)
 843                 return ret < 0 ? ret : 0;
 844
 845         group_counter->prev_state = group_counter->state;
 846         if (counter_sched_in(group_counter, cpuctx, ctx, cpu))
 847                 return -EAGAIN;
 848
 849         /*
 850          * Schedule in siblings as one group (if any):
 851          */
 852         list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
 853                 counter->prev_state = counter->state;
 854                 if (counter_sched_in(counter, cpuctx, ctx, cpu)) {
 855                         partial_group = counter;
 856                         goto group_error;
 857                 }
 858         }
 859
 860         return 0;
 861
 862 group_error:
 863         /*
 864          * Groups can be scheduled in as one unit only, so undo any
 865          * partial group before returning:
 866          */
 867         list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
 868                 if (counter == partial_group)
 869                         break;
 870                 counter_sched_out(counter, cpuctx, ctx);
 871         }
 872         counter_sched_out(group_counter, cpuctx, ctx);
 873
 874         return -EAGAIN;
 875 }
 876
 877 static void
 878 __perf_counter_sched_in(struct perf_counter_context *ctx,
 879                         struct perf_cpu_context *cpuctx, int cpu)
 880 {
 881         struct perf_counter *counter;
 882         u64 flags;
 883         int can_add_hw = 1;
 884
 885         spin_lock(&ctx->lock);
 886         ctx->is_active = 1;
 887         if (likely(!ctx->nr_counters))
 888                 goto out;
 889
 890         ctx->timestamp = perf_clock();
 891
 892         flags = hw_perf_save_disable();
 893
 894         /*
 895          * First go through the list and put on any pinned groups
 896          * in order to give them the best chance of going on.
 897          */
 898         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
 899                 if (counter->state <= PERF_COUNTER_STATE_OFF ||
 900                     !counter->hw_event.pinned)
 901                         continue;
 902                 if (counter->cpu != -1 && counter->cpu != cpu)
 903                         continue;
 904
 905                 if (group_can_go_on(counter, cpuctx, 1))
 906                         group_sched_in(counter, cpuctx, ctx, cpu);
 907
 908                 /*
 909                  * If this pinned group hasn't been scheduled,
 910                  * put it in error state.
 911                  */
 912                 if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
 913                         update_group_times(counter);
 914                         counter->state = PERF_COUNTER_STATE_ERROR;
 915                 }
 916         }
 917
 918         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
 919                 /*
 920                  * Ignore counters in OFF or ERROR state, and
 921                  * ignore pinned counters since we did them already.
 922                  */
 923                 if (counter->state <= PERF_COUNTER_STATE_OFF ||
 924                     counter->hw_event.pinned)
 925                         continue;
 926
 927                 /*
 928                  * Listen to the 'cpu' scheduling filter constraint
 929                  * of counters:
 930                  */
 931                 if (counter->cpu != -1 && counter->cpu != cpu)
 932                         continue;
 933
 934                 if (group_can_go_on(counter, cpuctx, can_add_hw)) {
 935                         if (group_sched_in(counter, cpuctx, ctx, cpu))
 936                                 can_add_hw = 0;
 937                 }
 938         }
 939         hw_perf_restore(flags);
 940  out:
 941         spin_unlock(&ctx->lock);
 942 }
 943
 944 /*
 945  * Called from scheduler to add the counters of the current task
 946  * with interrupts disabled.
 947  *
 948  * We restore the counter value and then enable it.
 949  *
 950  * This does not protect us against NMI, but enable()
 951  * sets the enabled bit in the control field of counter _before_
 952  * accessing the counter control register. If a NMI hits, then it will
 953  * keep the counter running.
 954  */
 955 void perf_counter_task_sched_in(struct task_struct *task, int cpu)
 956 {
 957         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
 958         struct perf_counter_context *ctx = &task->perf_counter_ctx;
 959
 960         __perf_counter_sched_in(ctx, cpuctx, cpu);
 961         cpuctx->task_ctx = ctx;
 962 }
 963
 964 static void perf_counter_cpu_sched_in(struct perf_cpu_context *cpuctx, int cpu)
 965 {
 966         struct perf_counter_context *ctx = &cpuctx->ctx;
 967
 968         __perf_counter_sched_in(ctx, cpuctx, cpu);
 969 }
 970
 971 int perf_counter_task_disable(void)
 972 {
 973         struct task_struct *curr = current;
 974         struct perf_counter_context *ctx = &curr->perf_counter_ctx;
 975         struct perf_counter *counter;
 976         unsigned long flags;
 977         u64 perf_flags;
 978         int cpu;
 979
 980         if (likely(!ctx->nr_counters))
 981                 return 0;
 982
 983         local_irq_save(flags);
 984         cpu = smp_processor_id();
 985
 986         perf_counter_task_sched_out(curr, cpu);
 987
 988         spin_lock(&ctx->lock);
 989
 990         /*
 991          * Disable all the counters:
 992          */
 993         perf_flags = hw_perf_save_disable();
 994
 995         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
 996                 if (counter->state != PERF_COUNTER_STATE_ERROR) {
 997                         update_group_times(counter);
 998                         counter->state = PERF_COUNTER_STATE_OFF;
 999                 }
1000         }
1001
1002         hw_perf_restore(perf_flags);
1003
1004         spin_unlock_irqrestore(&ctx->lock, flags);
1005
1006         return 0;
1007 }
1008
1009 int perf_counter_task_enable(void)
1010 {
1011         struct task_struct *curr = current;
1012         struct perf_counter_context *ctx = &curr->perf_counter_ctx;
1013         struct perf_counter *counter;
1014         unsigned long flags;
1015         u64 perf_flags;
1016         int cpu;
1017
1018         if (likely(!ctx->nr_counters))
1019                 return 0;
1020
1021         local_irq_save(flags);
1022         cpu = smp_processor_id();
1023
1024         perf_counter_task_sched_out(curr, cpu);
1025
1026         spin_lock(&ctx->lock);
1027
1028         /*
1029          * Disable all the counters:
1030          */
1031         perf_flags = hw_perf_save_disable();
1032
1033         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1034                 if (counter->state > PERF_COUNTER_STATE_OFF)
1035                         continue;
1036                 counter->state = PERF_COUNTER_STATE_INACTIVE;
1037                 counter->tstamp_enabled =
1038                         ctx->time - counter->total_time_enabled;
1039                 counter->hw_event.disabled = 0;
1040         }
1041         hw_perf_restore(perf_flags);
1042
1043         spin_unlock(&ctx->lock);
1044
1045         perf_counter_task_sched_in(curr, cpu);
1046
1047         local_irq_restore(flags);
1048
1049         return 0;
1050 }
1051
1052 /*
1053  * Round-robin a context's counters:
1054  */
1055 static void rotate_ctx(struct perf_counter_context *ctx)
1056 {
1057         struct perf_counter *counter;
1058         u64 perf_flags;
1059
1060         if (!ctx->nr_counters)
1061                 return;
1062
1063         spin_lock(&ctx->lock);
1064         /*
1065          * Rotate the first entry last (works just fine for group counters too):
1066          */
1067         perf_flags = hw_perf_save_disable();
1068         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1069                 list_move_tail(&counter->list_entry, &ctx->counter_list);
1070                 break;
1071         }
1072         hw_perf_restore(perf_flags);
1073
1074         spin_unlock(&ctx->lock);
1075 }
1076
1077 void perf_counter_task_tick(struct task_struct *curr, int cpu)
1078 {
1079         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
1080         struct perf_counter_context *ctx = &curr->perf_counter_ctx;
1081
1082         perf_counter_cpu_sched_out(cpuctx);
1083         perf_counter_task_sched_out(curr, cpu);
1084
1085         rotate_ctx(&cpuctx->ctx);
1086         rotate_ctx(ctx);
1087
1088         perf_counter_cpu_sched_in(cpuctx, cpu);
1089         perf_counter_task_sched_in(curr, cpu);
1090 }
1091
1092 /*
1093  * Cross CPU call to read the hardware counter
1094  */
1095 static void __read(void *info)
1096 {
1097         struct perf_counter *counter = info;
1098         struct perf_counter_context *ctx = counter->ctx;
1099         unsigned long flags;
1100
1101         local_irq_save(flags);
1102         if (ctx->is_active)
1103                 update_context_time(ctx);
1104         counter->pmu->read(counter);
1105         update_counter_times(counter);
1106         local_irq_restore(flags);
1107 }
1108
1109 static u64 perf_counter_read(struct perf_counter *counter)
1110 {
1111         /*
1112          * If counter is enabled and currently active on a CPU, update the
1113          * value in the counter structure:
1114          */
1115         if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
1116                 smp_call_function_single(counter->oncpu,
1117                                          __read, counter, 1);
1118         } else if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
1119                 update_counter_times(counter);
1120         }
1121
1122         return atomic64_read(&counter->count);
1123 }
1124
1125 static void put_context(struct perf_counter_context *ctx)
1126 {
1127         if (ctx->task)
1128                 put_task_struct(ctx->task);
1129 }
1130
1131 static struct perf_counter_context *find_get_context(pid_t pid, int cpu)
1132 {
1133         struct perf_cpu_context *cpuctx;
1134         struct perf_counter_context *ctx;
1135         struct task_struct *task;
1136
1137         /*
1138          * If cpu is not a wildcard then this is a percpu counter:
1139          */
1140         if (cpu != -1) {
1141                 /* Must be root to operate on a CPU counter: */
1142                 if (sysctl_perf_counter_priv && !capable(CAP_SYS_ADMIN))
1143                         return ERR_PTR(-EACCES);
1144
1145                 if (cpu < 0 || cpu > num_possible_cpus())
1146                         return ERR_PTR(-EINVAL);
1147
1148                 /*
1149                  * We could be clever and allow to attach a counter to an
1150                  * offline CPU and activate it when the CPU comes up, but
1151                  * that's for later.
1152                  */
1153                 if (!cpu_isset(cpu, cpu_online_map))
1154                         return ERR_PTR(-ENODEV);
1155
1156                 cpuctx = &per_cpu(perf_cpu_context, cpu);
1157                 ctx = &cpuctx->ctx;
1158
1159                 return ctx;
1160         }
1161
1162         rcu_read_lock();
1163         if (!pid)
1164                 task = current;
1165         else
1166                 task = find_task_by_vpid(pid);
1167         if (task)
1168                 get_task_struct(task);
1169         rcu_read_unlock();
1170
1171         if (!task)
1172                 return ERR_PTR(-ESRCH);
1173
1174         ctx = &task->perf_counter_ctx;
1175         ctx->task = task;
1176
1177         /* Reuse ptrace permission checks for now. */
1178         if (!ptrace_may_access(task, PTRACE_MODE_READ)) {
1179                 put_context(ctx);
1180                 return ERR_PTR(-EACCES);
1181         }
1182
1183         return ctx;
1184 }
1185
1186 static void free_counter_rcu(struct rcu_head *head)
1187 {
1188         struct perf_counter *counter;
1189
1190         counter = container_of(head, struct perf_counter, rcu_head);
1191         kfree(counter);
1192 }
1193
1194 static void perf_pending_sync(struct perf_counter *counter);
1195
1196 static void free_counter(struct perf_counter *counter)
1197 {
1198         perf_pending_sync(counter);
1199
1200         if (counter->hw_event.mmap)
1201                 atomic_dec(&nr_mmap_tracking);
1202         if (counter->hw_event.munmap)
1203                 atomic_dec(&nr_munmap_tracking);
1204         if (counter->hw_event.comm)
1205                 atomic_dec(&nr_comm_tracking);
1206
1207         if (counter->destroy)
1208                 counter->destroy(counter);
1209
1210         call_rcu(&counter->rcu_head, free_counter_rcu);
1211 }
1212
1213 /*
1214  * Called when the last reference to the file is gone.
1215  */
1216 static int perf_release(struct inode *inode, struct file *file)
1217 {
1218         struct perf_counter *counter = file->private_data;
1219         struct perf_counter_context *ctx = counter->ctx;
1220
1221         file->private_data = NULL;
1222
1223         mutex_lock(&ctx->mutex);
1224         mutex_lock(&counter->mutex);
1225
1226         perf_counter_remove_from_context(counter);
1227
1228         mutex_unlock(&counter->mutex);
1229         mutex_unlock(&ctx->mutex);
1230
1231         free_counter(counter);
1232         put_context(ctx);
1233
1234         return 0;
1235 }
1236
1237 /*
1238  * Read the performance counter - simple non blocking version for now
1239  */
1240 static ssize_t
1241 perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count)
1242 {
1243         u64 values[3];
1244         int n;
1245
1246         /*
1247          * Return end-of-file for a read on a counter that is in
1248          * error state (i.e. because it was pinned but it couldn't be
1249          * scheduled on to the CPU at some point).
1250          */
1251         if (counter->state == PERF_COUNTER_STATE_ERROR)
1252                 return 0;
1253
1254         mutex_lock(&counter->mutex);
1255         values[0] = perf_counter_read(counter);
1256         n = 1;
1257         if (counter->hw_event.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1258                 values[n++] = counter->total_time_enabled +
1259                         atomic64_read(&counter->child_total_time_enabled);
1260         if (counter->hw_event.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1261                 values[n++] = counter->total_time_running +
1262                         atomic64_read(&counter->child_total_time_running);
1263         mutex_unlock(&counter->mutex);
1264
1265         if (count < n * sizeof(u64))
1266                 return -EINVAL;
1267         count = n * sizeof(u64);
1268
1269         if (copy_to_user(buf, values, count))
1270                 return -EFAULT;
1271
1272         return count;
1273 }
1274
1275 static ssize_t
1276 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
1277 {
1278         struct perf_counter *counter = file->private_data;
1279
1280         return perf_read_hw(counter, buf, count);
1281 }
1282
1283 static unsigned int perf_poll(struct file *file, poll_table *wait)
1284 {
1285         struct perf_counter *counter = file->private_data;
1286         struct perf_mmap_data *data;
1287         unsigned int events = POLL_HUP;
1288
1289         rcu_read_lock();
1290         data = rcu_dereference(counter->data);
1291         if (data)
1292                 events = atomic_xchg(&data->poll, 0);
1293         rcu_read_unlock();
1294
1295         poll_wait(file, &counter->waitq, wait);
1296
1297         return events;
1298 }
1299
1300 static void perf_counter_reset(struct perf_counter *counter)
1301 {
1302         atomic_set(&counter->count, 0);
1303 }
1304
1305 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1306 {
1307         struct perf_counter *counter = file->private_data;
1308         int err = 0;
1309
1310         switch (cmd) {
1311         case PERF_COUNTER_IOC_ENABLE:
1312                 perf_counter_enable_family(counter);
1313                 break;
1314         case PERF_COUNTER_IOC_DISABLE:
1315                 perf_counter_disable_family(counter);
1316                 break;
1317         case PERF_COUNTER_IOC_REFRESH:
1318                 err = perf_counter_refresh(counter, arg);
1319                 break;
1320         case PERF_COUNTER_IOC_RESET:
1321                 perf_counter_reset(counter);
1322                 break;
1323         default:
1324                 err = -ENOTTY;
1325         }
1326         return err;
1327 }
1328
1329 /*
1330  * Callers need to ensure there can be no nesting of this function, otherwise
1331  * the seqlock logic goes bad. We can not serialize this because the arch
1332  * code calls this from NMI context.
1333  */
1334 void perf_counter_update_userpage(struct perf_counter *counter)
1335 {
1336         struct perf_mmap_data *data;
1337         struct perf_counter_mmap_page *userpg;
1338
1339         rcu_read_lock();
1340         data = rcu_dereference(counter->data);
1341         if (!data)
1342                 goto unlock;
1343
1344         userpg = data->user_page;
1345
1346         /*
1347          * Disable preemption so as to not let the corresponding user-space
1348          * spin too long if we get preempted.
1349          */
1350         preempt_disable();
1351         ++userpg->lock;
1352         barrier();
1353         userpg->index = counter->hw.idx;
1354         userpg->offset = atomic64_read(&counter->count);
1355         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
1356                 userpg->offset -= atomic64_read(&counter->hw.prev_count);
1357
1358         barrier();
1359         ++userpg->lock;
1360         preempt_enable();
1361 unlock:
1362         rcu_read_unlock();
1363 }
1364
1365 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1366 {
1367         struct perf_counter *counter = vma->vm_file->private_data;
1368         struct perf_mmap_data *data;
1369         int ret = VM_FAULT_SIGBUS;
1370
1371         rcu_read_lock();
1372         data = rcu_dereference(counter->data);
1373         if (!data)
1374                 goto unlock;
1375
1376         if (vmf->pgoff == 0) {
1377                 vmf->page = virt_to_page(data->user_page);
1378         } else {
1379                 int nr = vmf->pgoff - 1;
1380
1381                 if ((unsigned)nr > data->nr_pages)
1382                         goto unlock;
1383
1384                 vmf->page = virt_to_page(data->data_pages[nr]);
1385         }
1386         get_page(vmf->page);
1387         ret = 0;
1388 unlock:
1389         rcu_read_unlock();
1390
1391         return ret;
1392 }
1393
1394 static int perf_mmap_data_alloc(struct perf_counter *counter, int nr_pages)
1395 {
1396         struct perf_mmap_data *data;
1397         unsigned long size;
1398         int i;
1399
1400         WARN_ON(atomic_read(&counter->mmap_count));
1401
1402         size = sizeof(struct perf_mmap_data);
1403         size += nr_pages * sizeof(void *);
1404
1405         data = kzalloc(size, GFP_KERNEL);
1406         if (!data)
1407                 goto fail;
1408
1409         data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
1410         if (!data->user_page)
1411                 goto fail_user_page;
1412
1413         for (i = 0; i < nr_pages; i++) {
1414                 data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
1415                 if (!data->data_pages[i])
1416                         goto fail_data_pages;
1417         }
1418
1419         data->nr_pages = nr_pages;
1420         atomic_set(&data->lock, -1);
1421
1422         rcu_assign_pointer(counter->data, data);
1423
1424         return 0;
1425
1426 fail_data_pages:
1427         for (i--; i >= 0; i--)
1428                 free_page((unsigned long)data->data_pages[i]);
1429
1430         free_page((unsigned long)data->user_page);
1431
1432 fail_user_page:
1433         kfree(data);
1434
1435 fail:
1436         return -ENOMEM;
1437 }
1438
1439 static void __perf_mmap_data_free(struct rcu_head *rcu_head)
1440 {
1441         struct perf_mmap_data *data = container_of(rcu_head,
1442                         struct perf_mmap_data, rcu_head);
1443         int i;
1444
1445         free_page((unsigned long)data->user_page);
1446         for (i = 0; i < data->nr_pages; i++)
1447                 free_page((unsigned long)data->data_pages[i]);
1448         kfree(data);
1449 }
1450
1451 static void perf_mmap_data_free(struct perf_counter *counter)
1452 {
1453         struct perf_mmap_data *data = counter->data;
1454
1455         WARN_ON(atomic_read(&counter->mmap_count));
1456
1457         rcu_assign_pointer(counter->data, NULL);
1458         call_rcu(&data->rcu_head, __perf_mmap_data_free);
1459 }
1460
1461 static void perf_mmap_open(struct vm_area_struct *vma)
1462 {
1463         struct perf_counter *counter = vma->vm_file->private_data;
1464
1465         atomic_inc(&counter->mmap_count);
1466 }
1467
1468 static void perf_mmap_close(struct vm_area_struct *vma)
1469 {
1470         struct perf_counter *counter = vma->vm_file->private_data;
1471
1472         if (atomic_dec_and_mutex_lock(&counter->mmap_count,
1473                                       &counter->mmap_mutex)) {
1474                 vma->vm_mm->locked_vm -= counter->data->nr_locked;
1475                 perf_mmap_data_free(counter);
1476                 mutex_unlock(&counter->mmap_mutex);
1477         }
1478 }
1479
1480 static struct vm_operations_struct perf_mmap_vmops = {
1481         .open  = perf_mmap_open,
1482         .close = perf_mmap_close,
1483         .fault = perf_mmap_fault,
1484 };
1485
1486 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
1487 {
1488         struct perf_counter *counter = file->private_data;
1489         unsigned long vma_size;
1490         unsigned long nr_pages;
1491         unsigned long locked, lock_limit;
1492         int ret = 0;
1493         long extra;
1494
1495         if (!(vma->vm_flags & VM_SHARED) || (vma->vm_flags & VM_WRITE))
1496                 return -EINVAL;
1497
1498         vma_size = vma->vm_end - vma->vm_start;
1499         nr_pages = (vma_size / PAGE_SIZE) - 1;
1500
1501         /*
1502          * If we have data pages ensure they're a power-of-two number, so we
1503          * can do bitmasks instead of modulo.
1504          */
1505         if (nr_pages != 0 && !is_power_of_2(nr_pages))
1506                 return -EINVAL;
1507
1508         if (vma_size != PAGE_SIZE * (1 + nr_pages))
1509                 return -EINVAL;
1510
1511         if (vma->vm_pgoff != 0)
1512                 return -EINVAL;
1513
1514         mutex_lock(&counter->mmap_mutex);
1515         if (atomic_inc_not_zero(&counter->mmap_count)) {
1516                 if (nr_pages != counter->data->nr_pages)
1517                         ret = -EINVAL;
1518                 goto unlock;
1519         }
1520
1521         extra = nr_pages /* + 1 only account the data pages */;
1522         extra -= sysctl_perf_counter_mlock >> (PAGE_SHIFT - 10);
1523         if (extra < 0)
1524                 extra = 0;
1525
1526         locked = vma->vm_mm->locked_vm + extra;
1527
1528         lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
1529         lock_limit >>= PAGE_SHIFT;
1530
1531         if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) {
1532                 ret = -EPERM;
1533                 goto unlock;
1534         }
1535
1536         WARN_ON(counter->data);
1537         ret = perf_mmap_data_alloc(counter, nr_pages);
1538         if (ret)
1539                 goto unlock;
1540
1541         atomic_set(&counter->mmap_count, 1);
1542         vma->vm_mm->locked_vm += extra;
1543         counter->data->nr_locked = extra;
1544 unlock:
1545         mutex_unlock(&counter->mmap_mutex);
1546
1547         vma->vm_flags &= ~VM_MAYWRITE;
1548         vma->vm_flags |= VM_RESERVED;
1549         vma->vm_ops = &perf_mmap_vmops;
1550
1551         return ret;
1552 }
1553
1554 static int perf_fasync(int fd, struct file *filp, int on)
1555 {
1556         struct perf_counter *counter = filp->private_data;
1557         struct inode *inode = filp->f_path.dentry->d_inode;
1558         int retval;
1559
1560         mutex_lock(&inode->i_mutex);
1561         retval = fasync_helper(fd, filp, on, &counter->fasync);
1562         mutex_unlock(&inode->i_mutex);
1563
1564         if (retval < 0)
1565                 return retval;
1566
1567         return 0;
1568 }
1569
1570 static const struct file_operations perf_fops = {
1571         .release                = perf_release,
1572         .read                   = perf_read,
1573         .poll                   = perf_poll,
1574         .unlocked_ioctl         = perf_ioctl,
1575         .compat_ioctl           = perf_ioctl,
1576         .mmap                   = perf_mmap,
1577         .fasync                 = perf_fasync,
1578 };
1579
1580 /*
1581  * Perf counter wakeup
1582  *
1583  * If there's data, ensure we set the poll() state and publish everything
1584  * to user-space before waking everybody up.
1585  */
1586
1587 void perf_counter_wakeup(struct perf_counter *counter)
1588 {
1589         wake_up_all(&counter->waitq);
1590
1591         if (counter->pending_kill) {
1592                 kill_fasync(&counter->fasync, SIGIO, counter->pending_kill);
1593                 counter->pending_kill = 0;
1594         }
1595 }
1596
1597 /*
1598  * Pending wakeups
1599  *
1600  * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
1601  *
1602  * The NMI bit means we cannot possibly take locks. Therefore, maintain a
1603  * single linked list and use cmpxchg() to add entries lockless.
1604  */
1605
1606 static void perf_pending_counter(struct perf_pending_entry *entry)
1607 {
1608         struct perf_counter *counter = container_of(entry,
1609                         struct perf_counter, pending);
1610
1611         if (counter->pending_disable) {
1612                 counter->pending_disable = 0;
1613                 perf_counter_disable(counter);
1614         }
1615
1616         if (counter->pending_wakeup) {
1617                 counter->pending_wakeup = 0;
1618                 perf_counter_wakeup(counter);
1619         }
1620 }
1621
1622 #define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
1623
1624 static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
1625         PENDING_TAIL,
1626 };
1627
1628 static void perf_pending_queue(struct perf_pending_entry *entry,
1629                                void (*func)(struct perf_pending_entry *))
1630 {
1631         struct perf_pending_entry **head;
1632
1633         if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
1634                 return;
1635
1636         entry->func = func;
1637
1638         head = &get_cpu_var(perf_pending_head);
1639
1640         do {
1641                 entry->next = *head;
1642         } while (cmpxchg(head, entry->next, entry) != entry->next);
1643
1644         set_perf_counter_pending();
1645
1646         put_cpu_var(perf_pending_head);
1647 }
1648
1649 static int __perf_pending_run(void)
1650 {
1651         struct perf_pending_entry *list;
1652         int nr = 0;
1653
1654         list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
1655         while (list != PENDING_TAIL) {
1656                 void (*func)(struct perf_pending_entry *);
1657                 struct perf_pending_entry *entry = list;
1658
1659                 list = list->next;
1660
1661                 func = entry->func;
1662                 entry->next = NULL;
1663                 /*
1664                  * Ensure we observe the unqueue before we issue the wakeup,
1665                  * so that we won't be waiting forever.
1666                  * -- see perf_not_pending().
1667                  */
1668                 smp_wmb();
1669
1670                 func(entry);
1671                 nr++;
1672         }
1673
1674         return nr;
1675 }
1676
1677 static inline int perf_not_pending(struct perf_counter *counter)
1678 {
1679         /*
1680          * If we flush on whatever cpu we run, there is a chance we don't
1681          * need to wait.
1682          */
1683         get_cpu();
1684         __perf_pending_run();
1685         put_cpu();
1686
1687         /*
1688          * Ensure we see the proper queue state before going to sleep
1689          * so that we do not miss the wakeup. -- see perf_pending_handle()
1690          */
1691         smp_rmb();
1692         return counter->pending.next == NULL;
1693 }
1694
1695 static void perf_pending_sync(struct perf_counter *counter)
1696 {
1697         wait_event(counter->waitq, perf_not_pending(counter));
1698 }
1699
1700 void perf_counter_do_pending(void)
1701 {
1702         __perf_pending_run();
1703 }
1704
1705 /*
1706  * Callchain support -- arch specific
1707  */
1708
1709 __weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
1710 {
1711         return NULL;
1712 }
1713
1714 /*
1715  * Output
1716  */
1717
1718 struct perf_output_handle {
1719         struct perf_counter     *counter;
1720         struct perf_mmap_data   *data;
1721         unsigned int            offset;
1722         unsigned int            head;
1723         int                     nmi;
1724         int                     overflow;
1725         int                     locked;
1726         unsigned long           flags;
1727 };
1728
1729 static void perf_output_wakeup(struct perf_output_handle *handle)
1730 {
1731         atomic_set(&handle->data->poll, POLL_IN);
1732
1733         if (handle->nmi) {
1734                 handle->counter->pending_wakeup = 1;
1735                 perf_pending_queue(&handle->counter->pending,
1736                                    perf_pending_counter);
1737         } else
1738                 perf_counter_wakeup(handle->counter);
1739 }
1740
1741 /*
1742  * Curious locking construct.
1743  *
1744  * We need to ensure a later event doesn't publish a head when a former
1745  * event isn't done writing. However since we need to deal with NMIs we
1746  * cannot fully serialize things.
1747  *
1748  * What we do is serialize between CPUs so we only have to deal with NMI
1749  * nesting on a single CPU.
1750  *
1751  * We only publish the head (and generate a wakeup) when the outer-most
1752  * event completes.
1753  */
1754 static void perf_output_lock(struct perf_output_handle *handle)
1755 {
1756         struct perf_mmap_data *data = handle->data;
1757         int cpu;
1758
1759         handle->locked = 0;
1760
1761         local_irq_save(handle->flags);
1762         cpu = smp_processor_id();
1763
1764         if (in_nmi() && atomic_read(&data->lock) == cpu)
1765                 return;
1766
1767         while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
1768                 cpu_relax();
1769
1770         handle->locked = 1;
1771 }
1772
1773 static void perf_output_unlock(struct perf_output_handle *handle)
1774 {
1775         struct perf_mmap_data *data = handle->data;
1776         int head, cpu;
1777
1778         data->done_head = data->head;
1779
1780         if (!handle->locked)
1781                 goto out;
1782
1783 again:
1784         /*
1785          * The xchg implies a full barrier that ensures all writes are done
1786          * before we publish the new head, matched by a rmb() in userspace when
1787          * reading this position.
1788          */
1789         while ((head = atomic_xchg(&data->done_head, 0)))
1790                 data->user_page->data_head = head;
1791
1792         /*
1793          * NMI can happen here, which means we can miss a done_head update.
1794          */
1795
1796         cpu = atomic_xchg(&data->lock, -1);
1797         WARN_ON_ONCE(cpu != smp_processor_id());
1798
1799         /*
1800          * Therefore we have to validate we did not indeed do so.
1801          */
1802         if (unlikely(atomic_read(&data->done_head))) {
1803                 /*
1804                  * Since we had it locked, we can lock it again.
1805                  */
1806                 while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
1807                         cpu_relax();
1808
1809                 goto again;
1810         }
1811
1812         if (atomic_xchg(&data->wakeup, 0))
1813                 perf_output_wakeup(handle);
1814 out:
1815         local_irq_restore(handle->flags);
1816 }
1817
1818 static int perf_output_begin(struct perf_output_handle *handle,
1819                              struct perf_counter *counter, unsigned int size,
1820                              int nmi, int overflow)
1821 {
1822         struct perf_mmap_data *data;
1823         unsigned int offset, head;
1824
1825         /*
1826          * For inherited counters we send all the output towards the parent.
1827          */
1828         if (counter->parent)
1829                 counter = counter->parent;
1830
1831         rcu_read_lock();
1832         data = rcu_dereference(counter->data);
1833         if (!data)
1834                 goto out;
1835
1836         handle->data     = data;
1837         handle->counter  = counter;
1838         handle->nmi      = nmi;
1839         handle->overflow = overflow;
1840
1841         if (!data->nr_pages)
1842                 goto fail;
1843
1844         perf_output_lock(handle);
1845
1846         do {
1847                 offset = head = atomic_read(&data->head);
1848                 head += size;
1849         } while (atomic_cmpxchg(&data->head, offset, head) != offset);
1850
1851         handle->offset  = offset;
1852         handle->head    = head;
1853
1854         if ((offset >> PAGE_SHIFT) != (head >> PAGE_SHIFT))
1855                 atomic_set(&data->wakeup, 1);
1856
1857         return 0;
1858
1859 fail:
1860         perf_output_wakeup(handle);
1861 out:
1862         rcu_read_unlock();
1863
1864         return -ENOSPC;
1865 }
1866
1867 static void perf_output_copy(struct perf_output_handle *handle,
1868                              void *buf, unsigned int len)
1869 {
1870         unsigned int pages_mask;
1871         unsigned int offset;
1872         unsigned int size;
1873         void **pages;
1874
1875         offset          = handle->offset;
1876         pages_mask      = handle->data->nr_pages - 1;
1877         pages           = handle->data->data_pages;
1878
1879         do {
1880                 unsigned int page_offset;
1881                 int nr;
1882
1883                 nr          = (offset >> PAGE_SHIFT) & pages_mask;
1884                 page_offset = offset & (PAGE_SIZE - 1);
1885                 size        = min_t(unsigned int, PAGE_SIZE - page_offset, len);
1886
1887                 memcpy(pages[nr] + page_offset, buf, size);
1888
1889                 len         -= size;
1890                 buf         += size;
1891                 offset      += size;
1892         } while (len);
1893
1894         handle->offset = offset;
1895
1896         WARN_ON_ONCE(handle->offset > handle->head);
1897 }
1898
1899 #define perf_output_put(handle, x) \
1900         perf_output_copy((handle), &(x), sizeof(x))
1901
1902 static void perf_output_end(struct perf_output_handle *handle)
1903 {
1904         struct perf_counter *counter = handle->counter;
1905         struct perf_mmap_data *data = handle->data;
1906
1907         int wakeup_events = counter->hw_event.wakeup_events;
1908
1909         if (handle->overflow && wakeup_events) {
1910                 int events = atomic_inc_return(&data->events);
1911                 if (events >= wakeup_events) {
1912                         atomic_sub(wakeup_events, &data->events);
1913                         atomic_set(&data->wakeup, 1);
1914                 }
1915         }
1916
1917         perf_output_unlock(handle);
1918         rcu_read_unlock();
1919 }
1920
1921 static void perf_counter_output(struct perf_counter *counter,
1922                                 int nmi, struct pt_regs *regs, u64 addr)
1923 {
1924         int ret;
1925         u64 record_type = counter->hw_event.record_type;
1926         struct perf_output_handle handle;
1927         struct perf_event_header header;
1928         u64 ip;
1929         struct {
1930                 u32 pid, tid;
1931         } tid_entry;
1932         struct {
1933                 u64 event;
1934                 u64 counter;
1935         } group_entry;
1936         struct perf_callchain_entry *callchain = NULL;
1937         int callchain_size = 0;
1938         u64 time;
1939
1940         header.type = 0;
1941         header.size = sizeof(header);
1942
1943         header.misc = PERF_EVENT_MISC_OVERFLOW;
1944         header.misc |= user_mode(regs) ?
1945                 PERF_EVENT_MISC_USER : PERF_EVENT_MISC_KERNEL;
1946
1947         if (record_type & PERF_RECORD_IP) {
1948                 ip = instruction_pointer(regs);
1949                 header.type |= PERF_RECORD_IP;
1950                 header.size += sizeof(ip);
1951         }
1952
1953         if (record_type & PERF_RECORD_TID) {
1954                 /* namespace issues */
1955                 tid_entry.pid = current->group_leader->pid;
1956                 tid_entry.tid = current->pid;
1957
1958                 header.type |= PERF_RECORD_TID;
1959                 header.size += sizeof(tid_entry);
1960         }
1961
1962         if (record_type & PERF_RECORD_TIME) {
1963                 /*
1964                  * Maybe do better on x86 and provide cpu_clock_nmi()
1965                  */
1966                 time = sched_clock();
1967
1968                 header.type |= PERF_RECORD_TIME;
1969                 header.size += sizeof(u64);
1970         }
1971
1972         if (record_type & PERF_RECORD_ADDR) {
1973                 header.type |= PERF_RECORD_ADDR;
1974                 header.size += sizeof(u64);
1975         }
1976
1977         if (record_type & PERF_RECORD_GROUP) {
1978                 header.type |= PERF_RECORD_GROUP;
1979                 header.size += sizeof(u64) +
1980                         counter->nr_siblings * sizeof(group_entry);
1981         }
1982
1983         if (record_type & PERF_RECORD_CALLCHAIN) {
1984                 callchain = perf_callchain(regs);
1985
1986                 if (callchain) {
1987                         callchain_size = (1 + callchain->nr) * sizeof(u64);
1988
1989                         header.type |= PERF_RECORD_CALLCHAIN;
1990                         header.size += callchain_size;
1991                 }
1992         }
1993
1994         ret = perf_output_begin(&handle, counter, header.size, nmi, 1);
1995         if (ret)
1996                 return;
1997
1998         perf_output_put(&handle, header);
1999
2000         if (record_type & PERF_RECORD_IP)
2001                 perf_output_put(&handle, ip);
2002
2003         if (record_type & PERF_RECORD_TID)
2004                 perf_output_put(&handle, tid_entry);
2005
2006         if (record_type & PERF_RECORD_TIME)
2007                 perf_output_put(&handle, time);
2008
2009         if (record_type & PERF_RECORD_ADDR)
2010                 perf_output_put(&handle, addr);
2011
2012         /*
2013          * XXX PERF_RECORD_GROUP vs inherited counters seems difficult.
2014          */
2015         if (record_type & PERF_RECORD_GROUP) {
2016                 struct perf_counter *leader, *sub;
2017                 u64 nr = counter->nr_siblings;
2018
2019                 perf_output_put(&handle, nr);
2020
2021                 leader = counter->group_leader;
2022                 list_for_each_entry(sub, &leader->sibling_list, list_entry) {
2023                         if (sub != counter)
2024                                 sub->pmu->read(sub);
2025
2026                         group_entry.event = sub->hw_event.config;
2027                         group_entry.counter = atomic64_read(&sub->count);
2028
2029                         perf_output_put(&handle, group_entry);
2030                 }
2031         }
2032
2033         if (callchain)
2034                 perf_output_copy(&handle, callchain, callchain_size);
2035
2036         perf_output_end(&handle);
2037 }
2038
2039 /*
2040  * comm tracking
2041  */
2042
2043 struct perf_comm_event {
2044         struct task_struct      *task;
2045         char                    *comm;
2046         int                     comm_size;
2047
2048         struct {
2049                 struct perf_event_header        header;
2050
2051                 u32                             pid;
2052                 u32                             tid;
2053         } event;
2054 };
2055
2056 static void perf_counter_comm_output(struct perf_counter *counter,
2057                                      struct perf_comm_event *comm_event)
2058 {
2059         struct perf_output_handle handle;
2060         int size = comm_event->event.header.size;
2061         int ret = perf_output_begin(&handle, counter, size, 0, 0);
2062
2063         if (ret)
2064                 return;
2065
2066         perf_output_put(&handle, comm_event->event);
2067         perf_output_copy(&handle, comm_event->comm,
2068                                    comm_event->comm_size);
2069         perf_output_end(&handle);
2070 }
2071
2072 static int perf_counter_comm_match(struct perf_counter *counter,
2073                                    struct perf_comm_event *comm_event)
2074 {
2075         if (counter->hw_event.comm &&
2076             comm_event->event.header.type == PERF_EVENT_COMM)
2077                 return 1;
2078
2079         return 0;
2080 }
2081
2082 static void perf_counter_comm_ctx(struct perf_counter_context *ctx,
2083                                   struct perf_comm_event *comm_event)
2084 {
2085         struct perf_counter *counter;
2086
2087         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2088                 return;
2089
2090         rcu_read_lock();
2091         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2092                 if (perf_counter_comm_match(counter, comm_event))
2093                         perf_counter_comm_output(counter, comm_event);
2094         }
2095         rcu_read_unlock();
2096 }
2097
2098 static void perf_counter_comm_event(struct perf_comm_event *comm_event)
2099 {
2100         struct perf_cpu_context *cpuctx;
2101         unsigned int size;
2102         char *comm = comm_event->task->comm;
2103
2104         size = ALIGN(strlen(comm)+1, sizeof(u64));
2105
2106         comm_event->comm = comm;
2107         comm_event->comm_size = size;
2108
2109         comm_event->event.header.size = sizeof(comm_event->event) + size;
2110
2111         cpuctx = &get_cpu_var(perf_cpu_context);
2112         perf_counter_comm_ctx(&cpuctx->ctx, comm_event);
2113         put_cpu_var(perf_cpu_context);
2114
2115         perf_counter_comm_ctx(&current->perf_counter_ctx, comm_event);
2116 }
2117
2118 void perf_counter_comm(struct task_struct *task)
2119 {
2120         struct perf_comm_event comm_event;
2121
2122         if (!atomic_read(&nr_comm_tracking))
2123                 return;
2124
2125         comm_event = (struct perf_comm_event){
2126                 .task   = task,
2127                 .event  = {
2128                         .header = { .type = PERF_EVENT_COMM, },
2129                         .pid    = task->group_leader->pid,
2130                         .tid    = task->pid,
2131                 },
2132         };
2133
2134         perf_counter_comm_event(&comm_event);
2135 }
2136
2137 /*
2138  * mmap tracking
2139  */
2140
2141 struct perf_mmap_event {
2142         struct file     *file;
2143         char            *file_name;
2144         int             file_size;
2145
2146         struct {
2147                 struct perf_event_header        header;
2148
2149                 u32                             pid;
2150                 u32                             tid;
2151                 u64                             start;
2152                 u64                             len;
2153                 u64                             pgoff;
2154         } event;
2155 };
2156
2157 static void perf_counter_mmap_output(struct perf_counter *counter,
2158                                      struct perf_mmap_event *mmap_event)
2159 {
2160         struct perf_output_handle handle;
2161         int size = mmap_event->event.header.size;
2162         int ret = perf_output_begin(&handle, counter, size, 0, 0);
2163
2164         if (ret)
2165                 return;
2166
2167         perf_output_put(&handle, mmap_event->event);
2168         perf_output_copy(&handle, mmap_event->file_name,
2169                                    mmap_event->file_size);
2170         perf_output_end(&handle);
2171 }
2172
2173 static int perf_counter_mmap_match(struct perf_counter *counter,
2174                                    struct perf_mmap_event *mmap_event)
2175 {
2176         if (counter->hw_event.mmap &&
2177             mmap_event->event.header.type == PERF_EVENT_MMAP)
2178                 return 1;
2179
2180         if (counter->hw_event.munmap &&
2181             mmap_event->event.header.type == PERF_EVENT_MUNMAP)
2182                 return 1;
2183
2184         return 0;
2185 }
2186
2187 static void perf_counter_mmap_ctx(struct perf_counter_context *ctx,
2188                                   struct perf_mmap_event *mmap_event)
2189 {
2190         struct perf_counter *counter;
2191
2192         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2193                 return;
2194
2195         rcu_read_lock();
2196         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2197                 if (perf_counter_mmap_match(counter, mmap_event))
2198                         perf_counter_mmap_output(counter, mmap_event);
2199         }
2200         rcu_read_unlock();
2201 }
2202
2203 static void perf_counter_mmap_event(struct perf_mmap_event *mmap_event)
2204 {
2205         struct perf_cpu_context *cpuctx;
2206         struct file *file = mmap_event->file;
2207         unsigned int size;
2208         char tmp[16];
2209         char *buf = NULL;
2210         char *name;
2211
2212         if (file) {
2213                 buf = kzalloc(PATH_MAX, GFP_KERNEL);
2214                 if (!buf) {
2215                         name = strncpy(tmp, "//enomem", sizeof(tmp));
2216                         goto got_name;
2217                 }
2218                 name = d_path(&file->f_path, buf, PATH_MAX);
2219                 if (IS_ERR(name)) {
2220                         name = strncpy(tmp, "//toolong", sizeof(tmp));
2221                         goto got_name;
2222                 }
2223         } else {
2224                 name = strncpy(tmp, "//anon", sizeof(tmp));
2225                 goto got_name;
2226         }
2227
2228 got_name:
2229         size = ALIGN(strlen(name)+1, sizeof(u64));
2230
2231         mmap_event->file_name = name;
2232         mmap_event->file_size = size;
2233
2234         mmap_event->event.header.size = sizeof(mmap_event->event) + size;
2235
2236         cpuctx = &get_cpu_var(perf_cpu_context);
2237         perf_counter_mmap_ctx(&cpuctx->ctx, mmap_event);
2238         put_cpu_var(perf_cpu_context);
2239
2240         perf_counter_mmap_ctx(&current->perf_counter_ctx, mmap_event);
2241
2242         kfree(buf);
2243 }
2244
2245 void perf_counter_mmap(unsigned long addr, unsigned long len,
2246                        unsigned long pgoff, struct file *file)
2247 {
2248         struct perf_mmap_event mmap_event;
2249
2250         if (!atomic_read(&nr_mmap_tracking))
2251                 return;
2252
2253         mmap_event = (struct perf_mmap_event){
2254                 .file   = file,
2255                 .event  = {
2256                         .header = { .type = PERF_EVENT_MMAP, },
2257                         .pid    = current->group_leader->pid,
2258                         .tid    = current->pid,
2259                         .start  = addr,
2260                         .len    = len,
2261                         .pgoff  = pgoff,
2262                 },
2263         };
2264
2265         perf_counter_mmap_event(&mmap_event);
2266 }
2267
2268 void perf_counter_munmap(unsigned long addr, unsigned long len,
2269                          unsigned long pgoff, struct file *file)
2270 {
2271         struct perf_mmap_event mmap_event;
2272
2273         if (!atomic_read(&nr_munmap_tracking))
2274                 return;
2275
2276         mmap_event = (struct perf_mmap_event){
2277                 .file   = file,
2278                 .event  = {
2279                         .header = { .type = PERF_EVENT_MUNMAP, },
2280                         .pid    = current->group_leader->pid,
2281                         .tid    = current->pid,
2282                         .start  = addr,
2283                         .len    = len,
2284                         .pgoff  = pgoff,
2285                 },
2286         };
2287
2288         perf_counter_mmap_event(&mmap_event);
2289 }
2290
2291 /*
2292  * Generic counter overflow handling.
2293  */
2294
2295 int perf_counter_overflow(struct perf_counter *counter,
2296                           int nmi, struct pt_regs *regs, u64 addr)
2297 {
2298         int events = atomic_read(&counter->event_limit);
2299         int ret = 0;
2300
2301         /*
2302          * XXX event_limit might not quite work as expected on inherited
2303          * counters
2304          */
2305
2306         counter->pending_kill = POLL_IN;
2307         if (events && atomic_dec_and_test(&counter->event_limit)) {
2308                 ret = 1;
2309                 counter->pending_kill = POLL_HUP;
2310                 if (nmi) {
2311                         counter->pending_disable = 1;
2312                         perf_pending_queue(&counter->pending,
2313                                            perf_pending_counter);
2314                 } else
2315                         perf_counter_disable(counter);
2316         }
2317
2318         perf_counter_output(counter, nmi, regs, addr);
2319         return ret;
2320 }
2321
2322 /*
2323  * Generic software counter infrastructure
2324  */
2325
2326 static void perf_swcounter_update(struct perf_counter *counter)
2327 {
2328         struct hw_perf_counter *hwc = &counter->hw;
2329         u64 prev, now;
2330         s64 delta;
2331
2332 again:
2333         prev = atomic64_read(&hwc->prev_count);
2334         now = atomic64_read(&hwc->count);
2335         if (atomic64_cmpxchg(&hwc->prev_count, prev, now) != prev)
2336                 goto again;
2337
2338         delta = now - prev;
2339
2340         atomic64_add(delta, &counter->count);
2341         atomic64_sub(delta, &hwc->period_left);
2342 }
2343
2344 static void perf_swcounter_set_period(struct perf_counter *counter)
2345 {
2346         struct hw_perf_counter *hwc = &counter->hw;
2347         s64 left = atomic64_read(&hwc->period_left);
2348         s64 period = hwc->irq_period;
2349
2350         if (unlikely(left <= -period)) {
2351                 left = period;
2352                 atomic64_set(&hwc->period_left, left);
2353         }
2354
2355         if (unlikely(left <= 0)) {
2356                 left += period;
2357                 atomic64_add(period, &hwc->period_left);
2358         }
2359
2360         atomic64_set(&hwc->prev_count, -left);
2361         atomic64_set(&hwc->count, -left);
2362 }
2363
2364 static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer)
2365 {
2366         enum hrtimer_restart ret = HRTIMER_RESTART;
2367         struct perf_counter *counter;
2368         struct pt_regs *regs;
2369
2370         counter = container_of(hrtimer, struct perf_counter, hw.hrtimer);
2371         counter->pmu->read(counter);
2372
2373         regs = get_irq_regs();
2374         /*
2375          * In case we exclude kernel IPs or are somehow not in interrupt
2376          * context, provide the next best thing, the user IP.
2377          */
2378         if ((counter->hw_event.exclude_kernel || !regs) &&
2379                         !counter->hw_event.exclude_user)
2380                 regs = task_pt_regs(current);
2381
2382         if (regs) {
2383                 if (perf_counter_overflow(counter, 0, regs, 0))
2384                         ret = HRTIMER_NORESTART;
2385         }
2386
2387         hrtimer_forward_now(hrtimer, ns_to_ktime(counter->hw.irq_period));
2388
2389         return ret;
2390 }
2391
2392 static void perf_swcounter_overflow(struct perf_counter *counter,
2393                                     int nmi, struct pt_regs *regs, u64 addr)
2394 {
2395         perf_swcounter_update(counter);
2396         perf_swcounter_set_period(counter);
2397         if (perf_counter_overflow(counter, nmi, regs, addr))
2398                 /* soft-disable the counter */
2399                 ;
2400
2401 }
2402
2403 static int perf_swcounter_match(struct perf_counter *counter,
2404                                 enum perf_event_types type,
2405                                 u32 event, struct pt_regs *regs)
2406 {
2407         if (counter->state != PERF_COUNTER_STATE_ACTIVE)
2408                 return 0;
2409
2410         if (perf_event_raw(&counter->hw_event))
2411                 return 0;
2412
2413         if (perf_event_type(&counter->hw_event) != type)
2414                 return 0;
2415
2416         if (perf_event_id(&counter->hw_event) != event)
2417                 return 0;
2418
2419         if (counter->hw_event.exclude_user && user_mode(regs))
2420                 return 0;
2421
2422         if (counter->hw_event.exclude_kernel && !user_mode(regs))
2423                 return 0;
2424
2425         return 1;
2426 }
2427
2428 static void perf_swcounter_add(struct perf_counter *counter, u64 nr,
2429                                int nmi, struct pt_regs *regs, u64 addr)
2430 {
2431         int neg = atomic64_add_negative(nr, &counter->hw.count);
2432         if (counter->hw.irq_period && !neg)
2433                 perf_swcounter_overflow(counter, nmi, regs, addr);
2434 }
2435
2436 static void perf_swcounter_ctx_event(struct perf_counter_context *ctx,
2437                                      enum perf_event_types type, u32 event,
2438                                      u64 nr, int nmi, struct pt_regs *regs,
2439                                      u64 addr)
2440 {
2441         struct perf_counter *counter;
2442
2443         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2444                 return;
2445
2446         rcu_read_lock();
2447         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2448                 if (perf_swcounter_match(counter, type, event, regs))
2449                         perf_swcounter_add(counter, nr, nmi, regs, addr);
2450         }
2451         rcu_read_unlock();
2452 }
2453
2454 static int *perf_swcounter_recursion_context(struct perf_cpu_context *cpuctx)
2455 {
2456         if (in_nmi())
2457                 return &cpuctx->recursion[3];
2458
2459         if (in_irq())
2460                 return &cpuctx->recursion[2];
2461
2462         if (in_softirq())
2463                 return &cpuctx->recursion[1];
2464
2465         return &cpuctx->recursion[0];
2466 }
2467
2468 static void __perf_swcounter_event(enum perf_event_types type, u32 event,
2469                                    u64 nr, int nmi, struct pt_regs *regs,
2470                                    u64 addr)
2471 {
2472         struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
2473         int *recursion = perf_swcounter_recursion_context(cpuctx);
2474
2475         if (*recursion)
2476                 goto out;
2477
2478         (*recursion)++;
2479         barrier();
2480
2481         perf_swcounter_ctx_event(&cpuctx->ctx, type, event,
2482                                  nr, nmi, regs, addr);
2483         if (cpuctx->task_ctx) {
2484                 perf_swcounter_ctx_event(cpuctx->task_ctx, type, event,
2485                                          nr, nmi, regs, addr);
2486         }
2487
2488         barrier();
2489         (*recursion)--;
2490
2491 out:
2492         put_cpu_var(perf_cpu_context);
2493 }
2494
2495 void
2496 perf_swcounter_event(u32 event, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
2497 {
2498         __perf_swcounter_event(PERF_TYPE_SOFTWARE, event, nr, nmi, regs, addr);
2499 }
2500
2501 static void perf_swcounter_read(struct perf_counter *counter)
2502 {
2503         perf_swcounter_update(counter);
2504 }
2505
2506 static int perf_swcounter_enable(struct perf_counter *counter)
2507 {
2508         perf_swcounter_set_period(counter);
2509         return 0;
2510 }
2511
2512 static void perf_swcounter_disable(struct perf_counter *counter)
2513 {
2514         perf_swcounter_update(counter);
2515 }
2516
2517 static const struct pmu perf_ops_generic = {
2518         .enable         = perf_swcounter_enable,
2519         .disable        = perf_swcounter_disable,
2520         .read           = perf_swcounter_read,
2521 };
2522
2523 /*
2524  * Software counter: cpu wall time clock
2525  */
2526
2527 static void cpu_clock_perf_counter_update(struct perf_counter *counter)
2528 {
2529         int cpu = raw_smp_processor_id();
2530         s64 prev;
2531         u64 now;
2532
2533         now = cpu_clock(cpu);
2534         prev = atomic64_read(&counter->hw.prev_count);
2535         atomic64_set(&counter->hw.prev_count, now);
2536         atomic64_add(now - prev, &counter->count);
2537 }
2538
2539 static int cpu_clock_perf_counter_enable(struct perf_counter *counter)
2540 {
2541         struct hw_perf_counter *hwc = &counter->hw;
2542         int cpu = raw_smp_processor_id();
2543
2544         atomic64_set(&hwc->prev_count, cpu_clock(cpu));
2545         hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2546         hwc->hrtimer.function = perf_swcounter_hrtimer;
2547         if (hwc->irq_period) {
2548                 __hrtimer_start_range_ns(&hwc->hrtimer,
2549                                 ns_to_ktime(hwc->irq_period), 0,
2550                                 HRTIMER_MODE_REL, 0);
2551         }
2552
2553         return 0;
2554 }
2555
2556 static void cpu_clock_perf_counter_disable(struct perf_counter *counter)
2557 {
2558         hrtimer_cancel(&counter->hw.hrtimer);
2559         cpu_clock_perf_counter_update(counter);
2560 }
2561
2562 static void cpu_clock_perf_counter_read(struct perf_counter *counter)
2563 {
2564         cpu_clock_perf_counter_update(counter);
2565 }
2566
2567 static const struct pmu perf_ops_cpu_clock = {
2568         .enable         = cpu_clock_perf_counter_enable,
2569         .disable        = cpu_clock_perf_counter_disable,
2570         .read           = cpu_clock_perf_counter_read,
2571 };
2572
2573 /*
2574  * Software counter: task time clock
2575  */
2576
2577 static void task_clock_perf_counter_update(struct perf_counter *counter, u64 now)
2578 {
2579         u64 prev;
2580         s64 delta;
2581
2582         prev = atomic64_xchg(&counter->hw.prev_count, now);
2583         delta = now - prev;
2584         atomic64_add(delta, &counter->count);
2585 }
2586
2587 static int task_clock_perf_counter_enable(struct perf_counter *counter)
2588 {
2589         struct hw_perf_counter *hwc = &counter->hw;
2590         u64 now;
2591
2592         now = counter->ctx->time;
2593
2594         atomic64_set(&hwc->prev_count, now);
2595         hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2596         hwc->hrtimer.function = perf_swcounter_hrtimer;
2597         if (hwc->irq_period) {
2598                 __hrtimer_start_range_ns(&hwc->hrtimer,
2599                                 ns_to_ktime(hwc->irq_period), 0,
2600                                 HRTIMER_MODE_REL, 0);
2601         }
2602
2603         return 0;
2604 }
2605
2606 static void task_clock_perf_counter_disable(struct perf_counter *counter)
2607 {
2608         hrtimer_cancel(&counter->hw.hrtimer);
2609         task_clock_perf_counter_update(counter, counter->ctx->time);
2610
2611 }
2612
2613 static void task_clock_perf_counter_read(struct perf_counter *counter)
2614 {
2615         u64 time;
2616
2617         if (!in_nmi()) {
2618                 update_context_time(counter->ctx);
2619                 time = counter->ctx->time;
2620         } else {
2621                 u64 now = perf_clock();
2622                 u64 delta = now - counter->ctx->timestamp;
2623                 time = counter->ctx->time + delta;
2624         }
2625
2626         task_clock_perf_counter_update(counter, time);
2627 }
2628
2629 static const struct pmu perf_ops_task_clock = {
2630         .enable         = task_clock_perf_counter_enable,
2631         .disable        = task_clock_perf_counter_disable,
2632         .read           = task_clock_perf_counter_read,
2633 };
2634
2635 /*
2636  * Software counter: cpu migrations
2637  */
2638
2639 static inline u64 get_cpu_migrations(struct perf_counter *counter)
2640 {
2641         struct task_struct *curr = counter->ctx->task;
2642
2643         if (curr)
2644                 return curr->se.nr_migrations;
2645         return cpu_nr_migrations(smp_processor_id());
2646 }
2647
2648 static void cpu_migrations_perf_counter_update(struct perf_counter *counter)
2649 {
2650         u64 prev, now;
2651         s64 delta;
2652
2653         prev = atomic64_read(&counter->hw.prev_count);
2654         now = get_cpu_migrations(counter);
2655
2656         atomic64_set(&counter->hw.prev_count, now);
2657
2658         delta = now - prev;
2659
2660         atomic64_add(delta, &counter->count);
2661 }
2662
2663 static void cpu_migrations_perf_counter_read(struct perf_counter *counter)
2664 {
2665         cpu_migrations_perf_counter_update(counter);
2666 }
2667
2668 static int cpu_migrations_perf_counter_enable(struct perf_counter *counter)
2669 {
2670         if (counter->prev_state <= PERF_COUNTER_STATE_OFF)
2671                 atomic64_set(&counter->hw.prev_count,
2672                              get_cpu_migrations(counter));
2673         return 0;
2674 }
2675
2676 static void cpu_migrations_perf_counter_disable(struct perf_counter *counter)
2677 {
2678         cpu_migrations_perf_counter_update(counter);
2679 }
2680
2681 static const struct pmu perf_ops_cpu_migrations = {
2682         .enable         = cpu_migrations_perf_counter_enable,
2683         .disable        = cpu_migrations_perf_counter_disable,
2684         .read           = cpu_migrations_perf_counter_read,
2685 };
2686
2687 #ifdef CONFIG_EVENT_PROFILE
2688 void perf_tpcounter_event(int event_id)
2689 {
2690         struct pt_regs *regs = get_irq_regs();
2691
2692         if (!regs)
2693                 regs = task_pt_regs(current);
2694
2695         __perf_swcounter_event(PERF_TYPE_TRACEPOINT, event_id, 1, 1, regs, 0);
2696 }
2697 EXPORT_SYMBOL_GPL(perf_tpcounter_event);
2698
2699 extern int ftrace_profile_enable(int);
2700 extern void ftrace_profile_disable(int);
2701
2702 static void tp_perf_counter_destroy(struct perf_counter *counter)
2703 {
2704         ftrace_profile_disable(perf_event_id(&counter->hw_event));
2705 }
2706
2707 static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
2708 {
2709         int event_id = perf_event_id(&counter->hw_event);
2710         int ret;
2711
2712         ret = ftrace_profile_enable(event_id);
2713         if (ret)
2714                 return NULL;
2715
2716         counter->destroy = tp_perf_counter_destroy;
2717         counter->hw.irq_period = counter->hw_event.irq_period;
2718
2719         return &perf_ops_generic;
2720 }
2721 #else
2722 static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
2723 {
2724         return NULL;
2725 }
2726 #endif
2727
2728 static const struct pmu *sw_perf_counter_init(struct perf_counter *counter)
2729 {
2730         struct perf_counter_hw_event *hw_event = &counter->hw_event;
2731         const struct pmu *pmu = NULL;
2732         struct hw_perf_counter *hwc = &counter->hw;
2733
2734         /*
2735          * Software counters (currently) can't in general distinguish
2736          * between user, kernel and hypervisor events.
2737          * However, context switches and cpu migrations are considered
2738          * to be kernel events, and page faults are never hypervisor
2739          * events.
2740          */
2741         switch (perf_event_id(&counter->hw_event)) {
2742         case PERF_COUNT_CPU_CLOCK:
2743                 pmu = &perf_ops_cpu_clock;
2744
2745                 if (hw_event->irq_period && hw_event->irq_period < 10000)
2746                         hw_event->irq_period = 10000;
2747                 break;
2748         case PERF_COUNT_TASK_CLOCK:
2749                 /*
2750                  * If the user instantiates this as a per-cpu counter,
2751                  * use the cpu_clock counter instead.
2752                  */
2753                 if (counter->ctx->task)
2754                         pmu = &perf_ops_task_clock;
2755                 else
2756                         pmu = &perf_ops_cpu_clock;
2757
2758                 if (hw_event->irq_period && hw_event->irq_period < 10000)
2759                         hw_event->irq_period = 10000;
2760                 break;
2761         case PERF_COUNT_PAGE_FAULTS:
2762         case PERF_COUNT_PAGE_FAULTS_MIN:
2763         case PERF_COUNT_PAGE_FAULTS_MAJ:
2764         case PERF_COUNT_CONTEXT_SWITCHES:
2765                 pmu = &perf_ops_generic;
2766                 break;
2767         case PERF_COUNT_CPU_MIGRATIONS:
2768                 if (!counter->hw_event.exclude_kernel)
2769                         pmu = &perf_ops_cpu_migrations;
2770                 break;
2771         }
2772
2773         if (pmu)
2774                 hwc->irq_period = hw_event->irq_period;
2775
2776         return pmu;
2777 }
2778
2779 /*
2780  * Allocate and initialize a counter structure
2781  */
2782 static struct perf_counter *
2783 perf_counter_alloc(struct perf_counter_hw_event *hw_event,
2784                    int cpu,
2785                    struct perf_counter_context *ctx,
2786                    struct perf_counter *group_leader,
2787                    gfp_t gfpflags)
2788 {
2789         const struct pmu *pmu;
2790         struct perf_counter *counter;
2791         long err;
2792
2793         counter = kzalloc(sizeof(*counter), gfpflags);
2794         if (!counter)
2795                 return ERR_PTR(-ENOMEM);
2796
2797         /*
2798          * Single counters are their own group leaders, with an
2799          * empty sibling list:
2800          */
2801         if (!group_leader)
2802                 group_leader = counter;
2803
2804         mutex_init(&counter->mutex);
2805         INIT_LIST_HEAD(&counter->list_entry);
2806         INIT_LIST_HEAD(&counter->event_entry);
2807         INIT_LIST_HEAD(&counter->sibling_list);
2808         init_waitqueue_head(&counter->waitq);
2809
2810         mutex_init(&counter->mmap_mutex);
2811
2812         INIT_LIST_HEAD(&counter->child_list);
2813
2814         counter->cpu                    = cpu;
2815         counter->hw_event               = *hw_event;
2816         counter->group_leader           = group_leader;
2817         counter->pmu                    = NULL;
2818         counter->ctx                    = ctx;
2819
2820         counter->state = PERF_COUNTER_STATE_INACTIVE;
2821         if (hw_event->disabled)
2822                 counter->state = PERF_COUNTER_STATE_OFF;
2823
2824         pmu = NULL;
2825
2826         /*
2827          * we currently do not support PERF_RECORD_GROUP on inherited counters
2828          */
2829         if (hw_event->inherit && (hw_event->record_type & PERF_RECORD_GROUP))
2830                 goto done;
2831
2832         if (perf_event_raw(hw_event)) {
2833                 pmu = hw_perf_counter_init(counter);
2834                 goto done;
2835         }
2836
2837         switch (perf_event_type(hw_event)) {
2838         case PERF_TYPE_HARDWARE:
2839                 pmu = hw_perf_counter_init(counter);
2840                 break;
2841
2842         case PERF_TYPE_SOFTWARE:
2843                 pmu = sw_perf_counter_init(counter);
2844                 break;
2845
2846         case PERF_TYPE_TRACEPOINT:
2847                 pmu = tp_perf_counter_init(counter);
2848                 break;
2849         }
2850 done:
2851         err = 0;
2852         if (!pmu)
2853                 err = -EINVAL;
2854         else if (IS_ERR(pmu))
2855                 err = PTR_ERR(pmu);
2856
2857         if (err) {
2858                 kfree(counter);
2859                 return ERR_PTR(err);
2860         }
2861
2862         counter->pmu = pmu;
2863
2864         if (counter->hw_event.mmap)
2865                 atomic_inc(&nr_mmap_tracking);
2866         if (counter->hw_event.munmap)
2867                 atomic_inc(&nr_munmap_tracking);
2868         if (counter->hw_event.comm)
2869                 atomic_inc(&nr_comm_tracking);
2870
2871         return counter;
2872 }
2873
2874 /**
2875  * sys_perf_counter_open - open a performance counter, associate it to a task/cpu
2876  *
2877  * @hw_event_uptr:      event type attributes for monitoring/sampling
2878  * @pid:                target pid
2879  * @cpu:                target cpu
2880  * @group_fd:           group leader counter fd
2881  */
2882 SYSCALL_DEFINE5(perf_counter_open,
2883                 const struct perf_counter_hw_event __user *, hw_event_uptr,
2884                 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
2885 {
2886         struct perf_counter *counter, *group_leader;
2887         struct perf_counter_hw_event hw_event;
2888         struct perf_counter_context *ctx;
2889         struct file *counter_file = NULL;
2890         struct file *group_file = NULL;
2891         int fput_needed = 0;
2892         int fput_needed2 = 0;
2893         int ret;
2894
2895         /* for future expandability... */
2896         if (flags)
2897                 return -EINVAL;
2898
2899         if (copy_from_user(&hw_event, hw_event_uptr, sizeof(hw_event)) != 0)
2900                 return -EFAULT;
2901
2902         /*
2903          * Get the target context (task or percpu):
2904          */
2905         ctx = find_get_context(pid, cpu);
2906         if (IS_ERR(ctx))
2907                 return PTR_ERR(ctx);
2908
2909         /*
2910          * Look up the group leader (we will attach this counter to it):
2911          */
2912         group_leader = NULL;
2913         if (group_fd != -1) {
2914                 ret = -EINVAL;
2915                 group_file = fget_light(group_fd, &fput_needed);
2916                 if (!group_file)
2917                         goto err_put_context;
2918                 if (group_file->f_op != &perf_fops)
2919                         goto err_put_context;
2920
2921                 group_leader = group_file->private_data;
2922                 /*
2923                  * Do not allow a recursive hierarchy (this new sibling
2924                  * becoming part of another group-sibling):
2925                  */
2926                 if (group_leader->group_leader != group_leader)
2927                         goto err_put_context;
2928                 /*
2929                  * Do not allow to attach to a group in a different
2930                  * task or CPU context:
2931                  */
2932                 if (group_leader->ctx != ctx)
2933                         goto err_put_context;
2934                 /*
2935                  * Only a group leader can be exclusive or pinned
2936                  */
2937                 if (hw_event.exclusive || hw_event.pinned)
2938                         goto err_put_context;
2939         }
2940
2941         counter = perf_counter_alloc(&hw_event, cpu, ctx, group_leader,
2942                                      GFP_KERNEL);
2943         ret = PTR_ERR(counter);
2944         if (IS_ERR(counter))
2945                 goto err_put_context;
2946
2947         ret = anon_inode_getfd("[perf_counter]", &perf_fops, counter, 0);
2948         if (ret < 0)
2949                 goto err_free_put_context;
2950
2951         counter_file = fget_light(ret, &fput_needed2);
2952         if (!counter_file)
2953                 goto err_free_put_context;
2954
2955         counter->filp = counter_file;
2956         mutex_lock(&ctx->mutex);
2957         perf_install_in_context(ctx, counter, cpu);
2958         mutex_unlock(&ctx->mutex);
2959
2960         fput_light(counter_file, fput_needed2);
2961
2962 out_fput:
2963         fput_light(group_file, fput_needed);
2964
2965         return ret;
2966
2967 err_free_put_context:
2968         kfree(counter);
2969
2970 err_put_context:
2971         put_context(ctx);
2972
2973         goto out_fput;
2974 }
2975
2976 /*
2977  * Initialize the perf_counter context in a task_struct:
2978  */
2979 static void
2980 __perf_counter_init_context(struct perf_counter_context *ctx,
2981                             struct task_struct *task)
2982 {
2983         memset(ctx, 0, sizeof(*ctx));
2984         spin_lock_init(&ctx->lock);
2985         mutex_init(&ctx->mutex);
2986         INIT_LIST_HEAD(&ctx->counter_list);
2987         INIT_LIST_HEAD(&ctx->event_list);
2988         ctx->task = task;
2989 }
2990
2991 /*
2992  * inherit a counter from parent task to child task:
2993  */
2994 static struct perf_counter *
2995 inherit_counter(struct perf_counter *parent_counter,
2996               struct task_struct *parent,
2997               struct perf_counter_context *parent_ctx,
2998               struct task_struct *child,
2999               struct perf_counter *group_leader,
3000               struct perf_counter_context *child_ctx)
3001 {
3002         struct perf_counter *child_counter;
3003
3004         /*
3005          * Instead of creating recursive hierarchies of counters,
3006          * we link inherited counters back to the original parent,
3007          * which has a filp for sure, which we use as the reference
3008          * count:
3009          */
3010         if (parent_counter->parent)
3011                 parent_counter = parent_counter->parent;
3012
3013         child_counter = perf_counter_alloc(&parent_counter->hw_event,
3014                                            parent_counter->cpu, child_ctx,
3015                                            group_leader, GFP_KERNEL);
3016         if (IS_ERR(child_counter))
3017                 return child_counter;
3018
3019         /*
3020          * Link it up in the child's context:
3021          */
3022         child_counter->task = child;
3023         add_counter_to_ctx(child_counter, child_ctx);
3024
3025         child_counter->parent = parent_counter;
3026         /*
3027          * inherit into child's child as well:
3028          */
3029         child_counter->hw_event.inherit = 1;
3030
3031         /*
3032          * Get a reference to the parent filp - we will fput it
3033          * when the child counter exits. This is safe to do because
3034          * we are in the parent and we know that the filp still
3035          * exists and has a nonzero count:
3036          */
3037         atomic_long_inc(&parent_counter->filp->f_count);
3038
3039         /*
3040          * Link this into the parent counter's child list
3041          */
3042         mutex_lock(&parent_counter->mutex);
3043         list_add_tail(&child_counter->child_list, &parent_counter->child_list);
3044
3045         /*
3046          * Make the child state follow the state of the parent counter,
3047          * not its hw_event.disabled bit.  We hold the parent's mutex,
3048          * so we won't race with perf_counter_{en,dis}able_family.
3049          */
3050         if (parent_counter->state >= PERF_COUNTER_STATE_INACTIVE)
3051                 child_counter->state = PERF_COUNTER_STATE_INACTIVE;
3052         else
3053                 child_counter->state = PERF_COUNTER_STATE_OFF;
3054
3055         mutex_unlock(&parent_counter->mutex);
3056
3057         return child_counter;
3058 }
3059
3060 static int inherit_group(struct perf_counter *parent_counter,
3061               struct task_struct *parent,
3062               struct perf_counter_context *parent_ctx,
3063               struct task_struct *child,
3064               struct perf_counter_context *child_ctx)
3065 {
3066         struct perf_counter *leader;
3067         struct perf_counter *sub;
3068         struct perf_counter *child_ctr;
3069
3070         leader = inherit_counter(parent_counter, parent, parent_ctx,
3071                                  child, NULL, child_ctx);
3072         if (IS_ERR(leader))
3073                 return PTR_ERR(leader);
3074         list_for_each_entry(sub, &parent_counter->sibling_list, list_entry) {
3075                 child_ctr = inherit_counter(sub, parent, parent_ctx,
3076                                             child, leader, child_ctx);
3077                 if (IS_ERR(child_ctr))
3078                         return PTR_ERR(child_ctr);
3079         }
3080         return 0;
3081 }
3082
3083 static void sync_child_counter(struct perf_counter *child_counter,
3084                                struct perf_counter *parent_counter)
3085 {
3086         u64 parent_val, child_val;
3087
3088         parent_val = atomic64_read(&parent_counter->count);
3089         child_val = atomic64_read(&child_counter->count);
3090
3091         /*
3092          * Add back the child's count to the parent's count:
3093          */
3094         atomic64_add(child_val, &parent_counter->count);
3095         atomic64_add(child_counter->total_time_enabled,
3096                      &parent_counter->child_total_time_enabled);
3097         atomic64_add(child_counter->total_time_running,
3098                      &parent_counter->child_total_time_running);
3099
3100         /*
3101          * Remove this counter from the parent's list
3102          */
3103         mutex_lock(&parent_counter->mutex);
3104         list_del_init(&child_counter->child_list);
3105         mutex_unlock(&parent_counter->mutex);
3106
3107         /*
3108          * Release the parent counter, if this was the last
3109          * reference to it.
3110          */
3111         fput(parent_counter->filp);
3112 }
3113
3114 static void
3115 __perf_counter_exit_task(struct task_struct *child,
3116                          struct perf_counter *child_counter,
3117                          struct perf_counter_context *child_ctx)
3118 {
3119         struct perf_counter *parent_counter;
3120         struct perf_counter *sub, *tmp;
3121
3122         /*
3123          * If we do not self-reap then we have to wait for the
3124          * child task to unschedule (it will happen for sure),
3125          * so that its counter is at its final count. (This
3126          * condition triggers rarely - child tasks usually get
3127          * off their CPU before the parent has a chance to
3128          * get this far into the reaping action)
3129          */
3130         if (child != current) {
3131                 wait_task_inactive(child, 0);
3132                 list_del_init(&child_counter->list_entry);
3133                 update_counter_times(child_counter);
3134         } else {
3135                 struct perf_cpu_context *cpuctx;
3136                 unsigned long flags;
3137                 u64 perf_flags;
3138
3139                 /*
3140                  * Disable and unlink this counter.
3141                  *
3142                  * Be careful about zapping the list - IRQ/NMI context
3143                  * could still be processing it:
3144                  */
3145                 local_irq_save(flags);
3146                 perf_flags = hw_perf_save_disable();
3147
3148                 cpuctx = &__get_cpu_var(perf_cpu_context);
3149
3150                 group_sched_out(child_counter, cpuctx, child_ctx);
3151                 update_counter_times(child_counter);
3152
3153                 list_del_init(&child_counter->list_entry);
3154
3155                 child_ctx->nr_counters--;
3156
3157                 hw_perf_restore(perf_flags);
3158                 local_irq_restore(flags);
3159         }
3160
3161         parent_counter = child_counter->parent;
3162         /*
3163          * It can happen that parent exits first, and has counters
3164          * that are still around due to the child reference. These
3165          * counters need to be zapped - but otherwise linger.
3166          */
3167         if (parent_counter) {
3168                 sync_child_counter(child_counter, parent_counter);
3169                 list_for_each_entry_safe(sub, tmp, &child_counter->sibling_list,
3170                                          list_entry) {
3171                         if (sub->parent) {
3172                                 sync_child_counter(sub, sub->parent);
3173                                 free_counter(sub);
3174                         }
3175                 }
3176                 free_counter(child_counter);
3177         }
3178 }
3179
3180 /*
3181  * When a child task exits, feed back counter values to parent counters.
3182  *
3183  * Note: we may be running in child context, but the PID is not hashed
3184  * anymore so new counters will not be added.
3185  */
3186 void perf_counter_exit_task(struct task_struct *child)
3187 {
3188         struct perf_counter *child_counter, *tmp;
3189         struct perf_counter_context *child_ctx;
3190
3191         child_ctx = &child->perf_counter_ctx;
3192
3193         if (likely(!child_ctx->nr_counters))
3194                 return;
3195
3196         list_for_each_entry_safe(child_counter, tmp, &child_ctx->counter_list,
3197                                  list_entry)
3198                 __perf_counter_exit_task(child, child_counter, child_ctx);
3199 }
3200
3201 /*
3202  * Initialize the perf_counter context in task_struct
3203  */
3204 void perf_counter_init_task(struct task_struct *child)
3205 {
3206         struct perf_counter_context *child_ctx, *parent_ctx;
3207         struct perf_counter *counter;
3208         struct task_struct *parent = current;
3209
3210         child_ctx  =  &child->perf_counter_ctx;
3211         parent_ctx = &parent->perf_counter_ctx;
3212
3213         __perf_counter_init_context(child_ctx, child);
3214
3215         /*
3216          * This is executed from the parent task context, so inherit
3217          * counters that have been marked for cloning:
3218          */
3219
3220         if (likely(!parent_ctx->nr_counters))
3221                 return;
3222
3223         /*
3224          * Lock the parent list. No need to lock the child - not PID
3225          * hashed yet and not running, so nobody can access it.
3226          */
3227         mutex_lock(&parent_ctx->mutex);
3228
3229         /*
3230          * We dont have to disable NMIs - we are only looking at
3231          * the list, not manipulating it:
3232          */
3233         list_for_each_entry(counter, &parent_ctx->counter_list, list_entry) {
3234                 if (!counter->hw_event.inherit)
3235                         continue;
3236
3237                 if (inherit_group(counter, parent,
3238                                   parent_ctx, child, child_ctx))
3239                         break;
3240         }
3241
3242         mutex_unlock(&parent_ctx->mutex);
3243 }
3244
3245 static void __cpuinit perf_counter_init_cpu(int cpu)
3246 {
3247         struct perf_cpu_context *cpuctx;
3248
3249         cpuctx = &per_cpu(perf_cpu_context, cpu);
3250         __perf_counter_init_context(&cpuctx->ctx, NULL);
3251
3252         spin_lock(&perf_resource_lock);
3253         cpuctx->max_pertask = perf_max_counters - perf_reserved_percpu;
3254         spin_unlock(&perf_resource_lock);
3255
3256         hw_perf_counter_setup(cpu);
3257 }
3258
3259 #ifdef CONFIG_HOTPLUG_CPU
3260 static void __perf_counter_exit_cpu(void *info)
3261 {
3262         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
3263         struct perf_counter_context *ctx = &cpuctx->ctx;
3264         struct perf_counter *counter, *tmp;
3265
3266         list_for_each_entry_safe(counter, tmp, &ctx->counter_list, list_entry)
3267                 __perf_counter_remove_from_context(counter);
3268 }
3269 static void perf_counter_exit_cpu(int cpu)
3270 {
3271         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
3272         struct perf_counter_context *ctx = &cpuctx->ctx;
3273
3274         mutex_lock(&ctx->mutex);
3275         smp_call_function_single(cpu, __perf_counter_exit_cpu, NULL, 1);
3276         mutex_unlock(&ctx->mutex);
3277 }
3278 #else
3279 static inline void perf_counter_exit_cpu(int cpu) { }
3280 #endif
3281
3282 static int __cpuinit
3283 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
3284 {
3285         unsigned int cpu = (long)hcpu;
3286
3287         switch (action) {
3288
3289         case CPU_UP_PREPARE:
3290         case CPU_UP_PREPARE_FROZEN:
3291                 perf_counter_init_cpu(cpu);
3292                 break;
3293
3294         case CPU_DOWN_PREPARE:
3295         case CPU_DOWN_PREPARE_FROZEN:
3296                 perf_counter_exit_cpu(cpu);
3297                 break;
3298
3299         default:
3300                 break;
3301         }
3302
3303         return NOTIFY_OK;
3304 }
3305
3306 static struct notifier_block __cpuinitdata perf_cpu_nb = {
3307         .notifier_call          = perf_cpu_notify,
3308 };
3309
3310 void __init perf_counter_init(void)
3311 {
3312         perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
3313                         (void *)(long)smp_processor_id());
3314         register_cpu_notifier(&perf_cpu_nb);
3315 }
3316
3317 static ssize_t perf_show_reserve_percpu(struct sysdev_class *class, char *buf)
3318 {
3319         return sprintf(buf, "%d\n", perf_reserved_percpu);
3320 }
3321
3322 static ssize_t
3323 perf_set_reserve_percpu(struct sysdev_class *class,
3324                         const char *buf,
3325                         size_t count)
3326 {
3327         struct perf_cpu_context *cpuctx;
3328         unsigned long val;
3329         int err, cpu, mpt;
3330
3331         err = strict_strtoul(buf, 10, &val);
3332         if (err)
3333                 return err;
3334         if (val > perf_max_counters)
3335                 return -EINVAL;
3336
3337         spin_lock(&perf_resource_lock);
3338         perf_reserved_percpu = val;
3339         for_each_online_cpu(cpu) {
3340                 cpuctx = &per_cpu(perf_cpu_context, cpu);
3341                 spin_lock_irq(&cpuctx->ctx.lock);
3342                 mpt = min(perf_max_counters - cpuctx->ctx.nr_counters,
3343                           perf_max_counters - perf_reserved_percpu);
3344                 cpuctx->max_pertask = mpt;
3345                 spin_unlock_irq(&cpuctx->ctx.lock);
3346         }
3347         spin_unlock(&perf_resource_lock);
3348
3349         return count;
3350 }
3351
3352 static ssize_t perf_show_overcommit(struct sysdev_class *class, char *buf)
3353 {
3354         return sprintf(buf, "%d\n", perf_overcommit);
3355 }
3356
3357 static ssize_t
3358 perf_set_overcommit(struct sysdev_class *class, const char *buf, size_t count)
3359 {
3360         unsigned long val;
3361         int err;
3362
3363         err = strict_strtoul(buf, 10, &val);
3364         if (err)
3365                 return err;
3366         if (val > 1)
3367                 return -EINVAL;
3368
3369         spin_lock(&perf_resource_lock);
3370         perf_overcommit = val;
3371         spin_unlock(&perf_resource_lock);
3372
3373         return count;
3374 }
3375
3376 static SYSDEV_CLASS_ATTR(
3377                                 reserve_percpu,
3378                                 0644,
3379                                 perf_show_reserve_percpu,
3380                                 perf_set_reserve_percpu
3381                         );
3382
3383 static SYSDEV_CLASS_ATTR(
3384                                 overcommit,
3385                                 0644,
3386                                 perf_show_overcommit,
3387                                 perf_set_overcommit
3388                         );
3389
3390 static struct attribute *perfclass_attrs[] = {
3391         &attr_reserve_percpu.attr,
3392         &attr_overcommit.attr,
3393         NULL
3394 };
3395
3396 static struct attribute_group perfclass_attr_group = {
3397         .attrs                  = perfclass_attrs,
3398         .name                   = "perf_counters",
3399 };
3400
3401 static int __init perf_counter_sysfs_init(void)
3402 {
3403         return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
3404                                   &perfclass_attr_group);
3405 }
3406 device_initcall(perf_counter_sysfs_init);