kernel/perf_counter.c

   1 /*
   2  * Performance counter core code
   3  *
   4  *  Copyright(C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5  *  Copyright(C) 2008 Red Hat, Inc., Ingo Molnar
   6  *
   7  *
   8  *  For licensing details see kernel-base/COPYING
   9  */
  10
  11 #include <linux/fs.h>
  12 #include <linux/mm.h>
  13 #include <linux/cpu.h>
  14 #include <linux/smp.h>
  15 #include <linux/file.h>
  16 #include <linux/poll.h>
  17 #include <linux/sysfs.h>
  18 #include <linux/ptrace.h>
  19 #include <linux/percpu.h>
  20 #include <linux/vmstat.h>
  21 #include <linux/hardirq.h>
  22 #include <linux/rculist.h>
  23 #include <linux/uaccess.h>
  24 #include <linux/syscalls.h>
  25 #include <linux/anon_inodes.h>
  26 #include <linux/kernel_stat.h>
  27 #include <linux/perf_counter.h>
  28
  29 #include <asm/irq_regs.h>
  30
  31 /*
  32  * Each CPU has a list of per CPU counters:
  33  */
  34 DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
  35
  36 int perf_max_counters __read_mostly = 1;
  37 static int perf_reserved_percpu __read_mostly;
  38 static int perf_overcommit __read_mostly = 1;
  39
  40 /*
  41  * Mutex for (sysadmin-configurable) counter reservations:
  42  */
  43 static DEFINE_MUTEX(perf_resource_mutex);
  44
  45 /*
  46  * Architecture provided APIs - weak aliases:
  47  */
  48 extern __weak const struct hw_perf_counter_ops *
  49 hw_perf_counter_init(struct perf_counter *counter)
  50 {
  51         return NULL;
  52 }
  53
  54 u64 __weak hw_perf_save_disable(void)           { return 0; }
  55 void __weak hw_perf_restore(u64 ctrl)           { barrier(); }
  56 void __weak hw_perf_counter_setup(int cpu)      { barrier(); }
  57 int __weak hw_perf_group_sched_in(struct perf_counter *group_leader,
  58                struct perf_cpu_context *cpuctx,
  59                struct perf_counter_context *ctx, int cpu)
  60 {
  61         return 0;
  62 }
  63
  64 void __weak perf_counter_print_debug(void)      { }
  65
  66 static void
  67 list_add_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
  68 {
  69         struct perf_counter *group_leader = counter->group_leader;
  70
  71         /*
  72          * Depending on whether it is a standalone or sibling counter,
  73          * add it straight to the context's counter list, or to the group
  74          * leader's sibling list:
  75          */
  76         if (counter->group_leader == counter)
  77                 list_add_tail(&counter->list_entry, &ctx->counter_list);
  78         else {
  79                 list_add_tail(&counter->list_entry, &group_leader->sibling_list);
  80                 group_leader->nr_siblings++;
  81         }
  82
  83         list_add_rcu(&counter->event_entry, &ctx->event_list);
  84 }
  85
  86 static void
  87 list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
  88 {
  89         struct perf_counter *sibling, *tmp;
  90
  91         list_del_init(&counter->list_entry);
  92         list_del_rcu(&counter->event_entry);
  93
  94         if (counter->group_leader != counter)
  95                 counter->group_leader->nr_siblings--;
  96
  97         /*
  98          * If this was a group counter with sibling counters then
  99          * upgrade the siblings to singleton counters by adding them
 100          * to the context list directly:
 101          */
 102         list_for_each_entry_safe(sibling, tmp,
 103                                  &counter->sibling_list, list_entry) {
 104
 105                 list_move_tail(&sibling->list_entry, &ctx->counter_list);
 106                 sibling->group_leader = sibling;
 107         }
 108 }
 109
 110 static void
 111 counter_sched_out(struct perf_counter *counter,
 112                   struct perf_cpu_context *cpuctx,
 113                   struct perf_counter_context *ctx)
 114 {
 115         if (counter->state != PERF_COUNTER_STATE_ACTIVE)
 116                 return;
 117
 118         counter->state = PERF_COUNTER_STATE_INACTIVE;
 119         counter->hw_ops->disable(counter);
 120         counter->oncpu = -1;
 121
 122         if (!is_software_counter(counter))
 123                 cpuctx->active_oncpu--;
 124         ctx->nr_active--;
 125         if (counter->hw_event.exclusive || !cpuctx->active_oncpu)
 126                 cpuctx->exclusive = 0;
 127 }
 128
 129 static void
 130 group_sched_out(struct perf_counter *group_counter,
 131                 struct perf_cpu_context *cpuctx,
 132                 struct perf_counter_context *ctx)
 133 {
 134         struct perf_counter *counter;
 135
 136         if (group_counter->state != PERF_COUNTER_STATE_ACTIVE)
 137                 return;
 138
 139         counter_sched_out(group_counter, cpuctx, ctx);
 140
 141         /*
 142          * Schedule out siblings (if any):
 143          */
 144         list_for_each_entry(counter, &group_counter->sibling_list, list_entry)
 145                 counter_sched_out(counter, cpuctx, ctx);
 146
 147         if (group_counter->hw_event.exclusive)
 148                 cpuctx->exclusive = 0;
 149 }
 150
 151 /*
 152  * Cross CPU call to remove a performance counter
 153  *
 154  * We disable the counter on the hardware level first. After that we
 155  * remove it from the context list.
 156  */
 157 static void __perf_counter_remove_from_context(void *info)
 158 {
 159         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 160         struct perf_counter *counter = info;
 161         struct perf_counter_context *ctx = counter->ctx;
 162         unsigned long flags;
 163         u64 perf_flags;
 164
 165         /*
 166          * If this is a task context, we need to check whether it is
 167          * the current task context of this cpu. If not it has been
 168          * scheduled out before the smp call arrived.
 169          */
 170         if (ctx->task && cpuctx->task_ctx != ctx)
 171                 return;
 172
 173         curr_rq_lock_irq_save(&flags);
 174         spin_lock(&ctx->lock);
 175
 176         counter_sched_out(counter, cpuctx, ctx);
 177
 178         counter->task = NULL;
 179         ctx->nr_counters--;
 180
 181         /*
 182          * Protect the list operation against NMI by disabling the
 183          * counters on a global level. NOP for non NMI based counters.
 184          */
 185         perf_flags = hw_perf_save_disable();
 186         list_del_counter(counter, ctx);
 187         hw_perf_restore(perf_flags);
 188
 189         if (!ctx->task) {
 190                 /*
 191                  * Allow more per task counters with respect to the
 192                  * reservation:
 193                  */
 194                 cpuctx->max_pertask =
 195                         min(perf_max_counters - ctx->nr_counters,
 196                             perf_max_counters - perf_reserved_percpu);
 197         }
 198
 199         spin_unlock(&ctx->lock);
 200         curr_rq_unlock_irq_restore(&flags);
 201 }
 202
 203
 204 /*
 205  * Remove the counter from a task's (or a CPU's) list of counters.
 206  *
 207  * Must be called with counter->mutex and ctx->mutex held.
 208  *
 209  * CPU counters are removed with a smp call. For task counters we only
 210  * call when the task is on a CPU.
 211  */
 212 static void perf_counter_remove_from_context(struct perf_counter *counter)
 213 {
 214         struct perf_counter_context *ctx = counter->ctx;
 215         struct task_struct *task = ctx->task;
 216
 217         if (!task) {
 218                 /*
 219                  * Per cpu counters are removed via an smp call and
 220                  * the removal is always sucessful.
 221                  */
 222                 smp_call_function_single(counter->cpu,
 223                                          __perf_counter_remove_from_context,
 224                                          counter, 1);
 225                 return;
 226         }
 227
 228 retry:
 229         task_oncpu_function_call(task, __perf_counter_remove_from_context,
 230                                  counter);
 231
 232         spin_lock_irq(&ctx->lock);
 233         /*
 234          * If the context is active we need to retry the smp call.
 235          */
 236         if (ctx->nr_active && !list_empty(&counter->list_entry)) {
 237                 spin_unlock_irq(&ctx->lock);
 238                 goto retry;
 239         }
 240
 241         /*
 242          * The lock prevents that this context is scheduled in so we
 243          * can remove the counter safely, if the call above did not
 244          * succeed.
 245          */
 246         if (!list_empty(&counter->list_entry)) {
 247                 ctx->nr_counters--;
 248                 list_del_counter(counter, ctx);
 249                 counter->task = NULL;
 250         }
 251         spin_unlock_irq(&ctx->lock);
 252 }
 253
 254 /*
 255  * Cross CPU call to disable a performance counter
 256  */
 257 static void __perf_counter_disable(void *info)
 258 {
 259         struct perf_counter *counter = info;
 260         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 261         struct perf_counter_context *ctx = counter->ctx;
 262         unsigned long flags;
 263
 264         /*
 265          * If this is a per-task counter, need to check whether this
 266          * counter's task is the current task on this cpu.
 267          */
 268         if (ctx->task && cpuctx->task_ctx != ctx)
 269                 return;
 270
 271         curr_rq_lock_irq_save(&flags);
 272         spin_lock(&ctx->lock);
 273
 274         /*
 275          * If the counter is on, turn it off.
 276          * If it is in error state, leave it in error state.
 277          */
 278         if (counter->state >= PERF_COUNTER_STATE_INACTIVE) {
 279                 if (counter == counter->group_leader)
 280                         group_sched_out(counter, cpuctx, ctx);
 281                 else
 282                         counter_sched_out(counter, cpuctx, ctx);
 283                 counter->state = PERF_COUNTER_STATE_OFF;
 284         }
 285
 286         spin_unlock(&ctx->lock);
 287         curr_rq_unlock_irq_restore(&flags);
 288 }
 289
 290 /*
 291  * Disable a counter.
 292  */
 293 static void perf_counter_disable(struct perf_counter *counter)
 294 {
 295         struct perf_counter_context *ctx = counter->ctx;
 296         struct task_struct *task = ctx->task;
 297
 298         if (!task) {
 299                 /*
 300                  * Disable the counter on the cpu that it's on
 301                  */
 302                 smp_call_function_single(counter->cpu, __perf_counter_disable,
 303                                          counter, 1);
 304                 return;
 305         }
 306
 307  retry:
 308         task_oncpu_function_call(task, __perf_counter_disable, counter);
 309
 310         spin_lock_irq(&ctx->lock);
 311         /*
 312          * If the counter is still active, we need to retry the cross-call.
 313          */
 314         if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
 315                 spin_unlock_irq(&ctx->lock);
 316                 goto retry;
 317         }
 318
 319         /*
 320          * Since we have the lock this context can't be scheduled
 321          * in, so we can change the state safely.
 322          */
 323         if (counter->state == PERF_COUNTER_STATE_INACTIVE)
 324                 counter->state = PERF_COUNTER_STATE_OFF;
 325
 326         spin_unlock_irq(&ctx->lock);
 327 }
 328
 329 /*
 330  * Disable a counter and all its children.
 331  */
 332 static void perf_counter_disable_family(struct perf_counter *counter)
 333 {
 334         struct perf_counter *child;
 335
 336         perf_counter_disable(counter);
 337
 338         /*
 339          * Lock the mutex to protect the list of children
 340          */
 341         mutex_lock(&counter->mutex);
 342         list_for_each_entry(child, &counter->child_list, child_list)
 343                 perf_counter_disable(child);
 344         mutex_unlock(&counter->mutex);
 345 }
 346
 347 static int
 348 counter_sched_in(struct perf_counter *counter,
 349                  struct perf_cpu_context *cpuctx,
 350                  struct perf_counter_context *ctx,
 351                  int cpu)
 352 {
 353         if (counter->state <= PERF_COUNTER_STATE_OFF)
 354                 return 0;
 355
 356         counter->state = PERF_COUNTER_STATE_ACTIVE;
 357         counter->oncpu = cpu;   /* TODO: put 'cpu' into cpuctx->cpu */
 358         /*
 359          * The new state must be visible before we turn it on in the hardware:
 360          */
 361         smp_wmb();
 362
 363         if (counter->hw_ops->enable(counter)) {
 364                 counter->state = PERF_COUNTER_STATE_INACTIVE;
 365                 counter->oncpu = -1;
 366                 return -EAGAIN;
 367         }
 368
 369         if (!is_software_counter(counter))
 370                 cpuctx->active_oncpu++;
 371         ctx->nr_active++;
 372
 373         if (counter->hw_event.exclusive)
 374                 cpuctx->exclusive = 1;
 375
 376         return 0;
 377 }
 378
 379 /*
 380  * Return 1 for a group consisting entirely of software counters,
 381  * 0 if the group contains any hardware counters.
 382  */
 383 static int is_software_only_group(struct perf_counter *leader)
 384 {
 385         struct perf_counter *counter;
 386
 387         if (!is_software_counter(leader))
 388                 return 0;
 389
 390         list_for_each_entry(counter, &leader->sibling_list, list_entry)
 391                 if (!is_software_counter(counter))
 392                         return 0;
 393
 394         return 1;
 395 }
 396
 397 /*
 398  * Work out whether we can put this counter group on the CPU now.
 399  */
 400 static int group_can_go_on(struct perf_counter *counter,
 401                            struct perf_cpu_context *cpuctx,
 402                            int can_add_hw)
 403 {
 404         /*
 405          * Groups consisting entirely of software counters can always go on.
 406          */
 407         if (is_software_only_group(counter))
 408                 return 1;
 409         /*
 410          * If an exclusive group is already on, no other hardware
 411          * counters can go on.
 412          */
 413         if (cpuctx->exclusive)
 414                 return 0;
 415         /*
 416          * If this group is exclusive and there are already
 417          * counters on the CPU, it can't go on.
 418          */
 419         if (counter->hw_event.exclusive && cpuctx->active_oncpu)
 420                 return 0;
 421         /*
 422          * Otherwise, try to add it if all previous groups were able
 423          * to go on.
 424          */
 425         return can_add_hw;
 426 }
 427
 428 /*
 429  * Cross CPU call to install and enable a performance counter
 430  */
 431 static void __perf_install_in_context(void *info)
 432 {
 433         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 434         struct perf_counter *counter = info;
 435         struct perf_counter_context *ctx = counter->ctx;
 436         struct perf_counter *leader = counter->group_leader;
 437         int cpu = smp_processor_id();
 438         unsigned long flags;
 439         u64 perf_flags;
 440         int err;
 441
 442         /*
 443          * If this is a task context, we need to check whether it is
 444          * the current task context of this cpu. If not it has been
 445          * scheduled out before the smp call arrived.
 446          */
 447         if (ctx->task && cpuctx->task_ctx != ctx)
 448                 return;
 449
 450         curr_rq_lock_irq_save(&flags);
 451         spin_lock(&ctx->lock);
 452
 453         /*
 454          * Protect the list operation against NMI by disabling the
 455          * counters on a global level. NOP for non NMI based counters.
 456          */
 457         perf_flags = hw_perf_save_disable();
 458
 459         list_add_counter(counter, ctx);
 460         ctx->nr_counters++;
 461         counter->prev_state = PERF_COUNTER_STATE_OFF;
 462
 463         /*
 464          * Don't put the counter on if it is disabled or if
 465          * it is in a group and the group isn't on.
 466          */
 467         if (counter->state != PERF_COUNTER_STATE_INACTIVE ||
 468             (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE))
 469                 goto unlock;
 470
 471         /*
 472          * An exclusive counter can't go on if there are already active
 473          * hardware counters, and no hardware counter can go on if there
 474          * is already an exclusive counter on.
 475          */
 476         if (!group_can_go_on(counter, cpuctx, 1))
 477                 err = -EEXIST;
 478         else
 479                 err = counter_sched_in(counter, cpuctx, ctx, cpu);
 480
 481         if (err) {
 482                 /*
 483                  * This counter couldn't go on.  If it is in a group
 484                  * then we have to pull the whole group off.
 485                  * If the counter group is pinned then put it in error state.
 486                  */
 487                 if (leader != counter)
 488                         group_sched_out(leader, cpuctx, ctx);
 489                 if (leader->hw_event.pinned)
 490                         leader->state = PERF_COUNTER_STATE_ERROR;
 491         }
 492
 493         if (!err && !ctx->task && cpuctx->max_pertask)
 494                 cpuctx->max_pertask--;
 495
 496  unlock:
 497         hw_perf_restore(perf_flags);
 498
 499         spin_unlock(&ctx->lock);
 500         curr_rq_unlock_irq_restore(&flags);
 501 }
 502
 503 /*
 504  * Attach a performance counter to a context
 505  *
 506  * First we add the counter to the list with the hardware enable bit
 507  * in counter->hw_config cleared.
 508  *
 509  * If the counter is attached to a task which is on a CPU we use a smp
 510  * call to enable it in the task context. The task might have been
 511  * scheduled away, but we check this in the smp call again.
 512  *
 513  * Must be called with ctx->mutex held.
 514  */
 515 static void
 516 perf_install_in_context(struct perf_counter_context *ctx,
 517                         struct perf_counter *counter,
 518                         int cpu)
 519 {
 520         struct task_struct *task = ctx->task;
 521
 522         if (!task) {
 523                 /*
 524                  * Per cpu counters are installed via an smp call and
 525                  * the install is always sucessful.
 526                  */
 527                 smp_call_function_single(cpu, __perf_install_in_context,
 528                                          counter, 1);
 529                 return;
 530         }
 531
 532         counter->task = task;
 533 retry:
 534         task_oncpu_function_call(task, __perf_install_in_context,
 535                                  counter);
 536
 537         spin_lock_irq(&ctx->lock);
 538         /*
 539          * we need to retry the smp call.
 540          */
 541         if (ctx->is_active && list_empty(&counter->list_entry)) {
 542                 spin_unlock_irq(&ctx->lock);
 543                 goto retry;
 544         }
 545
 546         /*
 547          * The lock prevents that this context is scheduled in so we
 548          * can add the counter safely, if it the call above did not
 549          * succeed.
 550          */
 551         if (list_empty(&counter->list_entry)) {
 552                 list_add_counter(counter, ctx);
 553                 ctx->nr_counters++;
 554         }
 555         spin_unlock_irq(&ctx->lock);
 556 }
 557
 558 /*
 559  * Cross CPU call to enable a performance counter
 560  */
 561 static void __perf_counter_enable(void *info)
 562 {
 563         struct perf_counter *counter = info;
 564         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 565         struct perf_counter_context *ctx = counter->ctx;
 566         struct perf_counter *leader = counter->group_leader;
 567         unsigned long flags;
 568         int err;
 569
 570         /*
 571          * If this is a per-task counter, need to check whether this
 572          * counter's task is the current task on this cpu.
 573          */
 574         if (ctx->task && cpuctx->task_ctx != ctx)
 575                 return;
 576
 577         curr_rq_lock_irq_save(&flags);
 578         spin_lock(&ctx->lock);
 579
 580         counter->prev_state = counter->state;
 581         if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
 582                 goto unlock;
 583         counter->state = PERF_COUNTER_STATE_INACTIVE;
 584
 585         /*
 586          * If the counter is in a group and isn't the group leader,
 587          * then don't put it on unless the group is on.
 588          */
 589         if (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE)
 590                 goto unlock;
 591
 592         if (!group_can_go_on(counter, cpuctx, 1))
 593                 err = -EEXIST;
 594         else
 595                 err = counter_sched_in(counter, cpuctx, ctx,
 596                                        smp_processor_id());
 597
 598         if (err) {
 599                 /*
 600                  * If this counter can't go on and it's part of a
 601                  * group, then the whole group has to come off.
 602                  */
 603                 if (leader != counter)
 604                         group_sched_out(leader, cpuctx, ctx);
 605                 if (leader->hw_event.pinned)
 606                         leader->state = PERF_COUNTER_STATE_ERROR;
 607         }
 608
 609  unlock:
 610         spin_unlock(&ctx->lock);
 611         curr_rq_unlock_irq_restore(&flags);
 612 }
 613
 614 /*
 615  * Enable a counter.
 616  */
 617 static void perf_counter_enable(struct perf_counter *counter)
 618 {
 619         struct perf_counter_context *ctx = counter->ctx;
 620         struct task_struct *task = ctx->task;
 621
 622         if (!task) {
 623                 /*
 624                  * Enable the counter on the cpu that it's on
 625                  */
 626                 smp_call_function_single(counter->cpu, __perf_counter_enable,
 627                                          counter, 1);
 628                 return;
 629         }
 630
 631         spin_lock_irq(&ctx->lock);
 632         if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
 633                 goto out;
 634
 635         /*
 636          * If the counter is in error state, clear that first.
 637          * That way, if we see the counter in error state below, we
 638          * know that it has gone back into error state, as distinct
 639          * from the task having been scheduled away before the
 640          * cross-call arrived.
 641          */
 642         if (counter->state == PERF_COUNTER_STATE_ERROR)
 643                 counter->state = PERF_COUNTER_STATE_OFF;
 644
 645  retry:
 646         spin_unlock_irq(&ctx->lock);
 647         task_oncpu_function_call(task, __perf_counter_enable, counter);
 648
 649         spin_lock_irq(&ctx->lock);
 650
 651         /*
 652          * If the context is active and the counter is still off,
 653          * we need to retry the cross-call.
 654          */
 655         if (ctx->is_active && counter->state == PERF_COUNTER_STATE_OFF)
 656                 goto retry;
 657
 658         /*
 659          * Since we have the lock this context can't be scheduled
 660          * in, so we can change the state safely.
 661          */
 662         if (counter->state == PERF_COUNTER_STATE_OFF)
 663                 counter->state = PERF_COUNTER_STATE_INACTIVE;
 664  out:
 665         spin_unlock_irq(&ctx->lock);
 666 }
 667
 668 /*
 669  * Enable a counter and all its children.
 670  */
 671 static void perf_counter_enable_family(struct perf_counter *counter)
 672 {
 673         struct perf_counter *child;
 674
 675         perf_counter_enable(counter);
 676
 677         /*
 678          * Lock the mutex to protect the list of children
 679          */
 680         mutex_lock(&counter->mutex);
 681         list_for_each_entry(child, &counter->child_list, child_list)
 682                 perf_counter_enable(child);
 683         mutex_unlock(&counter->mutex);
 684 }
 685
 686 void __perf_counter_sched_out(struct perf_counter_context *ctx,
 687                               struct perf_cpu_context *cpuctx)
 688 {
 689         struct perf_counter *counter;
 690         u64 flags;
 691
 692         spin_lock(&ctx->lock);
 693         ctx->is_active = 0;
 694         if (likely(!ctx->nr_counters))
 695                 goto out;
 696
 697         flags = hw_perf_save_disable();
 698         if (ctx->nr_active) {
 699                 list_for_each_entry(counter, &ctx->counter_list, list_entry)
 700                         group_sched_out(counter, cpuctx, ctx);
 701         }
 702         hw_perf_restore(flags);
 703  out:
 704         spin_unlock(&ctx->lock);
 705 }
 706
 707 /*
 708  * Called from scheduler to remove the counters of the current task,
 709  * with interrupts disabled.
 710  *
 711  * We stop each counter and update the counter value in counter->count.
 712  *
 713  * This does not protect us against NMI, but disable()
 714  * sets the disabled bit in the control field of counter _before_
 715  * accessing the counter control register. If a NMI hits, then it will
 716  * not restart the counter.
 717  */
 718 void perf_counter_task_sched_out(struct task_struct *task, int cpu)
 719 {
 720         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
 721         struct perf_counter_context *ctx = &task->perf_counter_ctx;
 722         struct pt_regs *regs;
 723
 724         if (likely(!cpuctx->task_ctx))
 725                 return;
 726
 727         regs = task_pt_regs(task);
 728         perf_swcounter_event(PERF_COUNT_CONTEXT_SWITCHES, 1, 1, regs);
 729         __perf_counter_sched_out(ctx, cpuctx);
 730
 731         cpuctx->task_ctx = NULL;
 732 }
 733
 734 static void perf_counter_cpu_sched_out(struct perf_cpu_context *cpuctx)
 735 {
 736         __perf_counter_sched_out(&cpuctx->ctx, cpuctx);
 737 }
 738
 739 static int
 740 group_sched_in(struct perf_counter *group_counter,
 741                struct perf_cpu_context *cpuctx,
 742                struct perf_counter_context *ctx,
 743                int cpu)
 744 {
 745         struct perf_counter *counter, *partial_group;
 746         int ret;
 747
 748         if (group_counter->state == PERF_COUNTER_STATE_OFF)
 749                 return 0;
 750
 751         ret = hw_perf_group_sched_in(group_counter, cpuctx, ctx, cpu);
 752         if (ret)
 753                 return ret < 0 ? ret : 0;
 754
 755         group_counter->prev_state = group_counter->state;
 756         if (counter_sched_in(group_counter, cpuctx, ctx, cpu))
 757                 return -EAGAIN;
 758
 759         /*
 760          * Schedule in siblings as one group (if any):
 761          */
 762         list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
 763                 counter->prev_state = counter->state;
 764                 if (counter_sched_in(counter, cpuctx, ctx, cpu)) {
 765                         partial_group = counter;
 766                         goto group_error;
 767                 }
 768         }
 769
 770         return 0;
 771
 772 group_error:
 773         /*
 774          * Groups can be scheduled in as one unit only, so undo any
 775          * partial group before returning:
 776          */
 777         list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
 778                 if (counter == partial_group)
 779                         break;
 780                 counter_sched_out(counter, cpuctx, ctx);
 781         }
 782         counter_sched_out(group_counter, cpuctx, ctx);
 783
 784         return -EAGAIN;
 785 }
 786
 787 static void
 788 __perf_counter_sched_in(struct perf_counter_context *ctx,
 789                         struct perf_cpu_context *cpuctx, int cpu)
 790 {
 791         struct perf_counter *counter;
 792         u64 flags;
 793         int can_add_hw = 1;
 794
 795         spin_lock(&ctx->lock);
 796         ctx->is_active = 1;
 797         if (likely(!ctx->nr_counters))
 798                 goto out;
 799
 800         flags = hw_perf_save_disable();
 801
 802         /*
 803          * First go through the list and put on any pinned groups
 804          * in order to give them the best chance of going on.
 805          */
 806         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
 807                 if (counter->state <= PERF_COUNTER_STATE_OFF ||
 808                     !counter->hw_event.pinned)
 809                         continue;
 810                 if (counter->cpu != -1 && counter->cpu != cpu)
 811                         continue;
 812
 813                 if (group_can_go_on(counter, cpuctx, 1))
 814                         group_sched_in(counter, cpuctx, ctx, cpu);
 815
 816                 /*
 817                  * If this pinned group hasn't been scheduled,
 818                  * put it in error state.
 819                  */
 820                 if (counter->state == PERF_COUNTER_STATE_INACTIVE)
 821                         counter->state = PERF_COUNTER_STATE_ERROR;
 822         }
 823
 824         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
 825                 /*
 826                  * Ignore counters in OFF or ERROR state, and
 827                  * ignore pinned counters since we did them already.
 828                  */
 829                 if (counter->state <= PERF_COUNTER_STATE_OFF ||
 830                     counter->hw_event.pinned)
 831                         continue;
 832
 833                 /*
 834                  * Listen to the 'cpu' scheduling filter constraint
 835                  * of counters:
 836                  */
 837                 if (counter->cpu != -1 && counter->cpu != cpu)
 838                         continue;
 839
 840                 if (group_can_go_on(counter, cpuctx, can_add_hw)) {
 841                         if (group_sched_in(counter, cpuctx, ctx, cpu))
 842                                 can_add_hw = 0;
 843                 }
 844         }
 845         hw_perf_restore(flags);
 846  out:
 847         spin_unlock(&ctx->lock);
 848 }
 849
 850 /*
 851  * Called from scheduler to add the counters of the current task
 852  * with interrupts disabled.
 853  *
 854  * We restore the counter value and then enable it.
 855  *
 856  * This does not protect us against NMI, but enable()
 857  * sets the enabled bit in the control field of counter _before_
 858  * accessing the counter control register. If a NMI hits, then it will
 859  * keep the counter running.
 860  */
 861 void perf_counter_task_sched_in(struct task_struct *task, int cpu)
 862 {
 863         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
 864         struct perf_counter_context *ctx = &task->perf_counter_ctx;
 865
 866         __perf_counter_sched_in(ctx, cpuctx, cpu);
 867         cpuctx->task_ctx = ctx;
 868 }
 869
 870 static void perf_counter_cpu_sched_in(struct perf_cpu_context *cpuctx, int cpu)
 871 {
 872         struct perf_counter_context *ctx = &cpuctx->ctx;
 873
 874         __perf_counter_sched_in(ctx, cpuctx, cpu);
 875 }
 876
 877 int perf_counter_task_disable(void)
 878 {
 879         struct task_struct *curr = current;
 880         struct perf_counter_context *ctx = &curr->perf_counter_ctx;
 881         struct perf_counter *counter;
 882         unsigned long flags;
 883         u64 perf_flags;
 884         int cpu;
 885
 886         if (likely(!ctx->nr_counters))
 887                 return 0;
 888
 889         curr_rq_lock_irq_save(&flags);
 890         cpu = smp_processor_id();
 891
 892         /* force the update of the task clock: */
 893         __task_delta_exec(curr, 1);
 894
 895         perf_counter_task_sched_out(curr, cpu);
 896
 897         spin_lock(&ctx->lock);
 898
 899         /*
 900          * Disable all the counters:
 901          */
 902         perf_flags = hw_perf_save_disable();
 903
 904         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
 905                 if (counter->state != PERF_COUNTER_STATE_ERROR)
 906                         counter->state = PERF_COUNTER_STATE_OFF;
 907         }
 908
 909         hw_perf_restore(perf_flags);
 910
 911         spin_unlock(&ctx->lock);
 912
 913         curr_rq_unlock_irq_restore(&flags);
 914
 915         return 0;
 916 }
 917
 918 int perf_counter_task_enable(void)
 919 {
 920         struct task_struct *curr = current;
 921         struct perf_counter_context *ctx = &curr->perf_counter_ctx;
 922         struct perf_counter *counter;
 923         unsigned long flags;
 924         u64 perf_flags;
 925         int cpu;
 926
 927         if (likely(!ctx->nr_counters))
 928                 return 0;
 929
 930         curr_rq_lock_irq_save(&flags);
 931         cpu = smp_processor_id();
 932
 933         /* force the update of the task clock: */
 934         __task_delta_exec(curr, 1);
 935
 936         perf_counter_task_sched_out(curr, cpu);
 937
 938         spin_lock(&ctx->lock);
 939
 940         /*
 941          * Disable all the counters:
 942          */
 943         perf_flags = hw_perf_save_disable();
 944
 945         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
 946                 if (counter->state > PERF_COUNTER_STATE_OFF)
 947                         continue;
 948                 counter->state = PERF_COUNTER_STATE_INACTIVE;
 949                 counter->hw_event.disabled = 0;
 950         }
 951         hw_perf_restore(perf_flags);
 952
 953         spin_unlock(&ctx->lock);
 954
 955         perf_counter_task_sched_in(curr, cpu);
 956
 957         curr_rq_unlock_irq_restore(&flags);
 958
 959         return 0;
 960 }
 961
 962 /*
 963  * Round-robin a context's counters:
 964  */
 965 static void rotate_ctx(struct perf_counter_context *ctx)
 966 {
 967         struct perf_counter *counter;
 968         u64 perf_flags;
 969
 970         if (!ctx->nr_counters)
 971                 return;
 972
 973         spin_lock(&ctx->lock);
 974         /*
 975          * Rotate the first entry last (works just fine for group counters too):
 976          */
 977         perf_flags = hw_perf_save_disable();
 978         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
 979                 list_move_tail(&counter->list_entry, &ctx->counter_list);
 980                 break;
 981         }
 982         hw_perf_restore(perf_flags);
 983
 984         spin_unlock(&ctx->lock);
 985 }
 986
 987 void perf_counter_task_tick(struct task_struct *curr, int cpu)
 988 {
 989         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
 990         struct perf_counter_context *ctx = &curr->perf_counter_ctx;
 991         const int rotate_percpu = 0;
 992
 993         if (rotate_percpu)
 994                 perf_counter_cpu_sched_out(cpuctx);
 995         perf_counter_task_sched_out(curr, cpu);
 996
 997         if (rotate_percpu)
 998                 rotate_ctx(&cpuctx->ctx);
 999         rotate_ctx(ctx);
1000
1001         if (rotate_percpu)
1002                 perf_counter_cpu_sched_in(cpuctx, cpu);
1003         perf_counter_task_sched_in(curr, cpu);
1004 }
1005
1006 /*
1007  * Cross CPU call to read the hardware counter
1008  */
1009 static void __read(void *info)
1010 {
1011         struct perf_counter *counter = info;
1012         unsigned long flags;
1013
1014         curr_rq_lock_irq_save(&flags);
1015         counter->hw_ops->read(counter);
1016         curr_rq_unlock_irq_restore(&flags);
1017 }
1018
1019 static u64 perf_counter_read(struct perf_counter *counter)
1020 {
1021         /*
1022          * If counter is enabled and currently active on a CPU, update the
1023          * value in the counter structure:
1024          */
1025         if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
1026                 smp_call_function_single(counter->oncpu,
1027                                          __read, counter, 1);
1028         }
1029
1030         return atomic64_read(&counter->count);
1031 }
1032
1033 static void put_context(struct perf_counter_context *ctx)
1034 {
1035         if (ctx->task)
1036                 put_task_struct(ctx->task);
1037 }
1038
1039 static struct perf_counter_context *find_get_context(pid_t pid, int cpu)
1040 {
1041         struct perf_cpu_context *cpuctx;
1042         struct perf_counter_context *ctx;
1043         struct task_struct *task;
1044
1045         /*
1046          * If cpu is not a wildcard then this is a percpu counter:
1047          */
1048         if (cpu != -1) {
1049                 /* Must be root to operate on a CPU counter: */
1050                 if (!capable(CAP_SYS_ADMIN))
1051                         return ERR_PTR(-EACCES);
1052
1053                 if (cpu < 0 || cpu > num_possible_cpus())
1054                         return ERR_PTR(-EINVAL);
1055
1056                 /*
1057                  * We could be clever and allow to attach a counter to an
1058                  * offline CPU and activate it when the CPU comes up, but
1059                  * that's for later.
1060                  */
1061                 if (!cpu_isset(cpu, cpu_online_map))
1062                         return ERR_PTR(-ENODEV);
1063
1064                 cpuctx = &per_cpu(perf_cpu_context, cpu);
1065                 ctx = &cpuctx->ctx;
1066
1067                 return ctx;
1068         }
1069
1070         rcu_read_lock();
1071         if (!pid)
1072                 task = current;
1073         else
1074                 task = find_task_by_vpid(pid);
1075         if (task)
1076                 get_task_struct(task);
1077         rcu_read_unlock();
1078
1079         if (!task)
1080                 return ERR_PTR(-ESRCH);
1081
1082         ctx = &task->perf_counter_ctx;
1083         ctx->task = task;
1084
1085         /* Reuse ptrace permission checks for now. */
1086         if (!ptrace_may_access(task, PTRACE_MODE_READ)) {
1087                 put_context(ctx);
1088                 return ERR_PTR(-EACCES);
1089         }
1090
1091         return ctx;
1092 }
1093
1094 static void free_counter_rcu(struct rcu_head *head)
1095 {
1096         struct perf_counter *counter;
1097
1098         counter = container_of(head, struct perf_counter, rcu_head);
1099         kfree(counter);
1100 }
1101
1102 static void free_counter(struct perf_counter *counter)
1103 {
1104         if (counter->destroy)
1105                 counter->destroy(counter);
1106
1107         call_rcu(&counter->rcu_head, free_counter_rcu);
1108 }
1109
1110 /*
1111  * Called when the last reference to the file is gone.
1112  */
1113 static int perf_release(struct inode *inode, struct file *file)
1114 {
1115         struct perf_counter *counter = file->private_data;
1116         struct perf_counter_context *ctx = counter->ctx;
1117
1118         file->private_data = NULL;
1119
1120         mutex_lock(&ctx->mutex);
1121         mutex_lock(&counter->mutex);
1122
1123         perf_counter_remove_from_context(counter);
1124
1125         mutex_unlock(&counter->mutex);
1126         mutex_unlock(&ctx->mutex);
1127
1128         free_counter(counter);
1129         put_context(ctx);
1130
1131         return 0;
1132 }
1133
1134 /*
1135  * Read the performance counter - simple non blocking version for now
1136  */
1137 static ssize_t
1138 perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count)
1139 {
1140         u64 cntval;
1141
1142         if (count < sizeof(cntval))
1143                 return -EINVAL;
1144
1145         /*
1146          * Return end-of-file for a read on a counter that is in
1147          * error state (i.e. because it was pinned but it couldn't be
1148          * scheduled on to the CPU at some point).
1149          */
1150         if (counter->state == PERF_COUNTER_STATE_ERROR)
1151                 return 0;
1152
1153         mutex_lock(&counter->mutex);
1154         cntval = perf_counter_read(counter);
1155         mutex_unlock(&counter->mutex);
1156
1157         return put_user(cntval, (u64 __user *) buf) ? -EFAULT : sizeof(cntval);
1158 }
1159
1160 static ssize_t
1161 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
1162 {
1163         struct perf_counter *counter = file->private_data;
1164
1165         return perf_read_hw(counter, buf, count);
1166 }
1167
1168 static unsigned int perf_poll(struct file *file, poll_table *wait)
1169 {
1170         struct perf_counter *counter = file->private_data;
1171         struct perf_mmap_data *data;
1172         unsigned int events;
1173
1174         rcu_read_lock();
1175         data = rcu_dereference(counter->data);
1176         if (data)
1177                 events = atomic_xchg(&data->wakeup, 0);
1178         else
1179                 events = POLL_HUP;
1180         rcu_read_unlock();
1181
1182         poll_wait(file, &counter->waitq, wait);
1183
1184         return events;
1185 }
1186
1187 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1188 {
1189         struct perf_counter *counter = file->private_data;
1190         int err = 0;
1191
1192         switch (cmd) {
1193         case PERF_COUNTER_IOC_ENABLE:
1194                 perf_counter_enable_family(counter);
1195                 break;
1196         case PERF_COUNTER_IOC_DISABLE:
1197                 perf_counter_disable_family(counter);
1198                 break;
1199         default:
1200                 err = -ENOTTY;
1201         }
1202         return err;
1203 }
1204
1205 static void __perf_counter_update_userpage(struct perf_counter *counter,
1206                                            struct perf_mmap_data *data)
1207 {
1208         struct perf_counter_mmap_page *userpg = data->user_page;
1209
1210         /*
1211          * Disable preemption so as to not let the corresponding user-space
1212          * spin too long if we get preempted.
1213          */
1214         preempt_disable();
1215         ++userpg->lock;
1216         smp_wmb();
1217         userpg->index = counter->hw.idx;
1218         userpg->offset = atomic64_read(&counter->count);
1219         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
1220                 userpg->offset -= atomic64_read(&counter->hw.prev_count);
1221
1222         userpg->data_head = atomic_read(&data->head);
1223         smp_wmb();
1224         ++userpg->lock;
1225         preempt_enable();
1226 }
1227
1228 void perf_counter_update_userpage(struct perf_counter *counter)
1229 {
1230         struct perf_mmap_data *data;
1231
1232         rcu_read_lock();
1233         data = rcu_dereference(counter->data);
1234         if (data)
1235                 __perf_counter_update_userpage(counter, data);
1236         rcu_read_unlock();
1237 }
1238
1239 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1240 {
1241         struct perf_counter *counter = vma->vm_file->private_data;
1242         struct perf_mmap_data *data;
1243         int ret = VM_FAULT_SIGBUS;
1244
1245         rcu_read_lock();
1246         data = rcu_dereference(counter->data);
1247         if (!data)
1248                 goto unlock;
1249
1250         if (vmf->pgoff == 0) {
1251                 vmf->page = virt_to_page(data->user_page);
1252         } else {
1253                 int nr = vmf->pgoff - 1;
1254
1255                 if ((unsigned)nr > data->nr_pages)
1256                         goto unlock;
1257
1258                 vmf->page = virt_to_page(data->data_pages[nr]);
1259         }
1260         get_page(vmf->page);
1261         ret = 0;
1262 unlock:
1263         rcu_read_unlock();
1264
1265         return ret;
1266 }
1267
1268 static int perf_mmap_data_alloc(struct perf_counter *counter, int nr_pages)
1269 {
1270         struct perf_mmap_data *data;
1271         unsigned long size;
1272         int i;
1273
1274         WARN_ON(atomic_read(&counter->mmap_count));
1275
1276         size = sizeof(struct perf_mmap_data);
1277         size += nr_pages * sizeof(void *);
1278
1279         data = kzalloc(size, GFP_KERNEL);
1280         if (!data)
1281                 goto fail;
1282
1283         data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
1284         if (!data->user_page)
1285                 goto fail_user_page;
1286
1287         for (i = 0; i < nr_pages; i++) {
1288                 data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
1289                 if (!data->data_pages[i])
1290                         goto fail_data_pages;
1291         }
1292
1293         data->nr_pages = nr_pages;
1294
1295         rcu_assign_pointer(counter->data, data);
1296
1297         return 0;
1298
1299 fail_data_pages:
1300         for (i--; i >= 0; i--)
1301                 free_page((unsigned long)data->data_pages[i]);
1302
1303         free_page((unsigned long)data->user_page);
1304
1305 fail_user_page:
1306         kfree(data);
1307
1308 fail:
1309         return -ENOMEM;
1310 }
1311
1312 static void __perf_mmap_data_free(struct rcu_head *rcu_head)
1313 {
1314         struct perf_mmap_data *data = container_of(rcu_head,
1315                         struct perf_mmap_data, rcu_head);
1316         int i;
1317
1318         free_page((unsigned long)data->user_page);
1319         for (i = 0; i < data->nr_pages; i++)
1320                 free_page((unsigned long)data->data_pages[i]);
1321         kfree(data);
1322 }
1323
1324 static void perf_mmap_data_free(struct perf_counter *counter)
1325 {
1326         struct perf_mmap_data *data = counter->data;
1327
1328         WARN_ON(atomic_read(&counter->mmap_count));
1329
1330         rcu_assign_pointer(counter->data, NULL);
1331         call_rcu(&data->rcu_head, __perf_mmap_data_free);
1332 }
1333
1334 static void perf_mmap_open(struct vm_area_struct *vma)
1335 {
1336         struct perf_counter *counter = vma->vm_file->private_data;
1337
1338         atomic_inc(&counter->mmap_count);
1339 }
1340
1341 static void perf_mmap_close(struct vm_area_struct *vma)
1342 {
1343         struct perf_counter *counter = vma->vm_file->private_data;
1344
1345         if (atomic_dec_and_mutex_lock(&counter->mmap_count,
1346                                       &counter->mmap_mutex)) {
1347                 perf_mmap_data_free(counter);
1348                 mutex_unlock(&counter->mmap_mutex);
1349         }
1350 }
1351
1352 static struct vm_operations_struct perf_mmap_vmops = {
1353         .open = perf_mmap_open,
1354         .close = perf_mmap_close,
1355         .fault = perf_mmap_fault,
1356 };
1357
1358 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
1359 {
1360         struct perf_counter *counter = file->private_data;
1361         unsigned long vma_size;
1362         unsigned long nr_pages;
1363         unsigned long locked, lock_limit;
1364         int ret = 0;
1365
1366         if (!(vma->vm_flags & VM_SHARED) || (vma->vm_flags & VM_WRITE))
1367                 return -EINVAL;
1368
1369         vma_size = vma->vm_end - vma->vm_start;
1370         nr_pages = (vma_size / PAGE_SIZE) - 1;
1371
1372         if (nr_pages == 0 || !is_power_of_2(nr_pages))
1373                 return -EINVAL;
1374
1375         if (vma_size != PAGE_SIZE * (1 + nr_pages))
1376                 return -EINVAL;
1377
1378         if (vma->vm_pgoff != 0)
1379                 return -EINVAL;
1380
1381         locked = vma_size >>  PAGE_SHIFT;
1382         locked += vma->vm_mm->locked_vm;
1383
1384         lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
1385         lock_limit >>= PAGE_SHIFT;
1386
1387         if ((locked > lock_limit) && !capable(CAP_IPC_LOCK))
1388                 return -EPERM;
1389
1390         mutex_lock(&counter->mmap_mutex);
1391         if (atomic_inc_not_zero(&counter->mmap_count))
1392                 goto out;
1393
1394         WARN_ON(counter->data);
1395         ret = perf_mmap_data_alloc(counter, nr_pages);
1396         if (!ret)
1397                 atomic_set(&counter->mmap_count, 1);
1398 out:
1399         mutex_unlock(&counter->mmap_mutex);
1400
1401         vma->vm_flags &= ~VM_MAYWRITE;
1402         vma->vm_flags |= VM_RESERVED;
1403         vma->vm_ops = &perf_mmap_vmops;
1404
1405         return ret;
1406 }
1407
1408 static const struct file_operations perf_fops = {
1409         .release                = perf_release,
1410         .read                   = perf_read,
1411         .poll                   = perf_poll,
1412         .unlocked_ioctl         = perf_ioctl,
1413         .compat_ioctl           = perf_ioctl,
1414         .mmap                   = perf_mmap,
1415 };
1416
1417 /*
1418  * Output
1419  */
1420
1421 struct perf_output_handle {
1422         struct perf_counter     *counter;
1423         struct perf_mmap_data   *data;
1424         unsigned int            offset;
1425         int                     wakeup;
1426 };
1427
1428 static int perf_output_begin(struct perf_output_handle *handle,
1429                              struct perf_counter *counter, unsigned int size)
1430 {
1431         struct perf_mmap_data *data;
1432         unsigned int offset, head;
1433
1434         rcu_read_lock();
1435         data = rcu_dereference(counter->data);
1436         if (!data)
1437                 goto out;
1438
1439         if (!data->nr_pages)
1440                 goto out;
1441
1442         do {
1443                 offset = head = atomic_read(&data->head);
1444                 head += size;
1445         } while (atomic_cmpxchg(&data->head, offset, head) != offset);
1446
1447         handle->counter = counter;
1448         handle->data    = data;
1449         handle->offset  = offset;
1450         handle->wakeup  = (offset >> PAGE_SHIFT) != (head >> PAGE_SHIFT);
1451
1452         return 0;
1453
1454 out:
1455         rcu_read_unlock();
1456
1457         return -ENOSPC;
1458 }
1459
1460 static void perf_output_copy(struct perf_output_handle *handle,
1461                              void *buf, unsigned int len)
1462 {
1463         unsigned int pages_mask;
1464         unsigned int offset;
1465         unsigned int size;
1466         void **pages;
1467
1468         offset          = handle->offset;
1469         pages_mask      = handle->data->nr_pages - 1;
1470         pages           = handle->data->data_pages;
1471
1472         do {
1473                 unsigned int page_offset;
1474                 int nr;
1475
1476                 nr          = (offset >> PAGE_SHIFT) & pages_mask;
1477                 page_offset = offset & (PAGE_SIZE - 1);
1478                 size        = min_t(unsigned int, PAGE_SIZE - page_offset, len);
1479
1480                 memcpy(pages[nr] + page_offset, buf, size);
1481
1482                 len         -= size;
1483                 buf         += size;
1484                 offset      += size;
1485         } while (len);
1486
1487         handle->offset = offset;
1488 }
1489
1490 #define perf_output_put(handle, x) \
1491         perf_output_copy((handle), &(x), sizeof(x))
1492
1493 static void perf_output_end(struct perf_output_handle *handle, int nmi)
1494 {
1495         if (handle->wakeup) {
1496                 (void)atomic_xchg(&handle->data->wakeup, POLL_IN);
1497                 __perf_counter_update_userpage(handle->counter, handle->data);
1498                 if (nmi) {
1499                         handle->counter->wakeup_pending = 1;
1500                         set_perf_counter_pending();
1501                 } else
1502                         wake_up(&handle->counter->waitq);
1503         }
1504         rcu_read_unlock();
1505 }
1506
1507 static int perf_output_write(struct perf_counter *counter, int nmi,
1508                              void *buf, ssize_t size)
1509 {
1510         struct perf_output_handle handle;
1511         int ret;
1512
1513         ret = perf_output_begin(&handle, counter, size);
1514         if (ret)
1515                 goto out;
1516
1517         perf_output_copy(&handle, buf, size);
1518         perf_output_end(&handle, nmi);
1519
1520 out:
1521         return ret;
1522 }
1523
1524 static void perf_output_simple(struct perf_counter *counter,
1525                                int nmi, struct pt_regs *regs)
1526 {
1527         struct {
1528                 struct perf_event_header header;
1529                 u64 ip;
1530         } event;
1531
1532         event.header.type = PERF_EVENT_IP;
1533         event.header.size = sizeof(event);
1534         event.ip = instruction_pointer(regs);
1535
1536         perf_output_write(counter, nmi, &event, sizeof(event));
1537 }
1538
1539 static void perf_output_group(struct perf_counter *counter, int nmi)
1540 {
1541         struct perf_output_handle handle;
1542         struct perf_event_header header;
1543         struct perf_counter *leader, *sub;
1544         unsigned int size;
1545         struct {
1546                 u64 event;
1547                 u64 counter;
1548         } entry;
1549         int ret;
1550
1551         size = sizeof(header) + counter->nr_siblings * sizeof(entry);
1552
1553         ret = perf_output_begin(&handle, counter, size);
1554         if (ret)
1555                 return;
1556
1557         header.type = PERF_EVENT_GROUP;
1558         header.size = size;
1559
1560         perf_output_put(&handle, header);
1561
1562         leader = counter->group_leader;
1563         list_for_each_entry(sub, &leader->sibling_list, list_entry) {
1564                 if (sub != counter)
1565                         sub->hw_ops->read(sub);
1566
1567                 entry.event = sub->hw_event.config;
1568                 entry.counter = atomic64_read(&sub->count);
1569
1570                 perf_output_put(&handle, entry);
1571         }
1572
1573         perf_output_end(&handle, nmi);
1574 }
1575
1576 void perf_counter_output(struct perf_counter *counter,
1577                          int nmi, struct pt_regs *regs)
1578 {
1579         switch (counter->hw_event.record_type) {
1580         case PERF_RECORD_SIMPLE:
1581                 return;
1582
1583         case PERF_RECORD_IRQ:
1584                 perf_output_simple(counter, nmi, regs);
1585                 break;
1586
1587         case PERF_RECORD_GROUP:
1588                 perf_output_group(counter, nmi);
1589                 break;
1590         }
1591 }
1592
1593 /*
1594  * Generic software counter infrastructure
1595  */
1596
1597 static void perf_swcounter_update(struct perf_counter *counter)
1598 {
1599         struct hw_perf_counter *hwc = &counter->hw;
1600         u64 prev, now;
1601         s64 delta;
1602
1603 again:
1604         prev = atomic64_read(&hwc->prev_count);
1605         now = atomic64_read(&hwc->count);
1606         if (atomic64_cmpxchg(&hwc->prev_count, prev, now) != prev)
1607                 goto again;
1608
1609         delta = now - prev;
1610
1611         atomic64_add(delta, &counter->count);
1612         atomic64_sub(delta, &hwc->period_left);
1613 }
1614
1615 static void perf_swcounter_set_period(struct perf_counter *counter)
1616 {
1617         struct hw_perf_counter *hwc = &counter->hw;
1618         s64 left = atomic64_read(&hwc->period_left);
1619         s64 period = hwc->irq_period;
1620
1621         if (unlikely(left <= -period)) {
1622                 left = period;
1623                 atomic64_set(&hwc->period_left, left);
1624         }
1625
1626         if (unlikely(left <= 0)) {
1627                 left += period;
1628                 atomic64_add(period, &hwc->period_left);
1629         }
1630
1631         atomic64_set(&hwc->prev_count, -left);
1632         atomic64_set(&hwc->count, -left);
1633 }
1634
1635 static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer)
1636 {
1637         struct perf_counter *counter;
1638         struct pt_regs *regs;
1639
1640         counter = container_of(hrtimer, struct perf_counter, hw.hrtimer);
1641         counter->hw_ops->read(counter);
1642
1643         regs = get_irq_regs();
1644         /*
1645          * In case we exclude kernel IPs or are somehow not in interrupt
1646          * context, provide the next best thing, the user IP.
1647          */
1648         if ((counter->hw_event.exclude_kernel || !regs) &&
1649                         !counter->hw_event.exclude_user)
1650                 regs = task_pt_regs(current);
1651
1652         if (regs)
1653                 perf_counter_output(counter, 0, regs);
1654
1655         hrtimer_forward_now(hrtimer, ns_to_ktime(counter->hw.irq_period));
1656
1657         return HRTIMER_RESTART;
1658 }
1659
1660 static void perf_swcounter_overflow(struct perf_counter *counter,
1661                                     int nmi, struct pt_regs *regs)
1662 {
1663         perf_swcounter_update(counter);
1664         perf_swcounter_set_period(counter);
1665         perf_counter_output(counter, nmi, regs);
1666 }
1667
1668 static int perf_swcounter_match(struct perf_counter *counter,
1669                                 enum perf_event_types type,
1670                                 u32 event, struct pt_regs *regs)
1671 {
1672         if (counter->state != PERF_COUNTER_STATE_ACTIVE)
1673                 return 0;
1674
1675         if (perf_event_raw(&counter->hw_event))
1676                 return 0;
1677
1678         if (perf_event_type(&counter->hw_event) != type)
1679                 return 0;
1680
1681         if (perf_event_id(&counter->hw_event) != event)
1682                 return 0;
1683
1684         if (counter->hw_event.exclude_user && user_mode(regs))
1685                 return 0;
1686
1687         if (counter->hw_event.exclude_kernel && !user_mode(regs))
1688                 return 0;
1689
1690         return 1;
1691 }
1692
1693 static void perf_swcounter_add(struct perf_counter *counter, u64 nr,
1694                                int nmi, struct pt_regs *regs)
1695 {
1696         int neg = atomic64_add_negative(nr, &counter->hw.count);
1697         if (counter->hw.irq_period && !neg)
1698                 perf_swcounter_overflow(counter, nmi, regs);
1699 }
1700
1701 static void perf_swcounter_ctx_event(struct perf_counter_context *ctx,
1702                                      enum perf_event_types type, u32 event,
1703                                      u64 nr, int nmi, struct pt_regs *regs)
1704 {
1705         struct perf_counter *counter;
1706
1707         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
1708                 return;
1709
1710         rcu_read_lock();
1711         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
1712                 if (perf_swcounter_match(counter, type, event, regs))
1713                         perf_swcounter_add(counter, nr, nmi, regs);
1714         }
1715         rcu_read_unlock();
1716 }
1717
1718 static int *perf_swcounter_recursion_context(struct perf_cpu_context *cpuctx)
1719 {
1720         if (in_nmi())
1721                 return &cpuctx->recursion[3];
1722
1723         if (in_irq())
1724                 return &cpuctx->recursion[2];
1725
1726         if (in_softirq())
1727                 return &cpuctx->recursion[1];
1728
1729         return &cpuctx->recursion[0];
1730 }
1731
1732 static void __perf_swcounter_event(enum perf_event_types type, u32 event,
1733                                    u64 nr, int nmi, struct pt_regs *regs)
1734 {
1735         struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
1736         int *recursion = perf_swcounter_recursion_context(cpuctx);
1737
1738         if (*recursion)
1739                 goto out;
1740
1741         (*recursion)++;
1742         barrier();
1743
1744         perf_swcounter_ctx_event(&cpuctx->ctx, type, event, nr, nmi, regs);
1745         if (cpuctx->task_ctx) {
1746                 perf_swcounter_ctx_event(cpuctx->task_ctx, type, event,
1747                                 nr, nmi, regs);
1748         }
1749
1750         barrier();
1751         (*recursion)--;
1752
1753 out:
1754         put_cpu_var(perf_cpu_context);
1755 }
1756
1757 void perf_swcounter_event(u32 event, u64 nr, int nmi, struct pt_regs *regs)
1758 {
1759         __perf_swcounter_event(PERF_TYPE_SOFTWARE, event, nr, nmi, regs);
1760 }
1761
1762 static void perf_swcounter_read(struct perf_counter *counter)
1763 {
1764         perf_swcounter_update(counter);
1765 }
1766
1767 static int perf_swcounter_enable(struct perf_counter *counter)
1768 {
1769         perf_swcounter_set_period(counter);
1770         return 0;
1771 }
1772
1773 static void perf_swcounter_disable(struct perf_counter *counter)
1774 {
1775         perf_swcounter_update(counter);
1776 }
1777
1778 static const struct hw_perf_counter_ops perf_ops_generic = {
1779         .enable         = perf_swcounter_enable,
1780         .disable        = perf_swcounter_disable,
1781         .read           = perf_swcounter_read,
1782 };
1783
1784 /*
1785  * Software counter: cpu wall time clock
1786  */
1787
1788 static void cpu_clock_perf_counter_update(struct perf_counter *counter)
1789 {
1790         int cpu = raw_smp_processor_id();
1791         s64 prev;
1792         u64 now;
1793
1794         now = cpu_clock(cpu);
1795         prev = atomic64_read(&counter->hw.prev_count);
1796         atomic64_set(&counter->hw.prev_count, now);
1797         atomic64_add(now - prev, &counter->count);
1798 }
1799
1800 static int cpu_clock_perf_counter_enable(struct perf_counter *counter)
1801 {
1802         struct hw_perf_counter *hwc = &counter->hw;
1803         int cpu = raw_smp_processor_id();
1804
1805         atomic64_set(&hwc->prev_count, cpu_clock(cpu));
1806         hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1807         hwc->hrtimer.function = perf_swcounter_hrtimer;
1808         if (hwc->irq_period) {
1809                 __hrtimer_start_range_ns(&hwc->hrtimer,
1810                                 ns_to_ktime(hwc->irq_period), 0,
1811                                 HRTIMER_MODE_REL, 0);
1812         }
1813
1814         return 0;
1815 }
1816
1817 static void cpu_clock_perf_counter_disable(struct perf_counter *counter)
1818 {
1819         hrtimer_cancel(&counter->hw.hrtimer);
1820         cpu_clock_perf_counter_update(counter);
1821 }
1822
1823 static void cpu_clock_perf_counter_read(struct perf_counter *counter)
1824 {
1825         cpu_clock_perf_counter_update(counter);
1826 }
1827
1828 static const struct hw_perf_counter_ops perf_ops_cpu_clock = {
1829         .enable         = cpu_clock_perf_counter_enable,
1830         .disable        = cpu_clock_perf_counter_disable,
1831         .read           = cpu_clock_perf_counter_read,
1832 };
1833
1834 /*
1835  * Software counter: task time clock
1836  */
1837
1838 /*
1839  * Called from within the scheduler:
1840  */
1841 static u64 task_clock_perf_counter_val(struct perf_counter *counter, int update)
1842 {
1843         struct task_struct *curr = counter->task;
1844         u64 delta;
1845
1846         delta = __task_delta_exec(curr, update);
1847
1848         return curr->se.sum_exec_runtime + delta;
1849 }
1850
1851 static void task_clock_perf_counter_update(struct perf_counter *counter, u64 now)
1852 {
1853         u64 prev;
1854         s64 delta;
1855
1856         prev = atomic64_read(&counter->hw.prev_count);
1857
1858         atomic64_set(&counter->hw.prev_count, now);
1859
1860         delta = now - prev;
1861
1862         atomic64_add(delta, &counter->count);
1863 }
1864
1865 static int task_clock_perf_counter_enable(struct perf_counter *counter)
1866 {
1867         struct hw_perf_counter *hwc = &counter->hw;
1868
1869         atomic64_set(&hwc->prev_count, task_clock_perf_counter_val(counter, 0));
1870         hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1871         hwc->hrtimer.function = perf_swcounter_hrtimer;
1872         if (hwc->irq_period) {
1873                 __hrtimer_start_range_ns(&hwc->hrtimer,
1874                                 ns_to_ktime(hwc->irq_period), 0,
1875                                 HRTIMER_MODE_REL, 0);
1876         }
1877
1878         return 0;
1879 }
1880
1881 static void task_clock_perf_counter_disable(struct perf_counter *counter)
1882 {
1883         hrtimer_cancel(&counter->hw.hrtimer);
1884         task_clock_perf_counter_update(counter,
1885                         task_clock_perf_counter_val(counter, 0));
1886 }
1887
1888 static void task_clock_perf_counter_read(struct perf_counter *counter)
1889 {
1890         task_clock_perf_counter_update(counter,
1891                         task_clock_perf_counter_val(counter, 1));
1892 }
1893
1894 static const struct hw_perf_counter_ops perf_ops_task_clock = {
1895         .enable         = task_clock_perf_counter_enable,
1896         .disable        = task_clock_perf_counter_disable,
1897         .read           = task_clock_perf_counter_read,
1898 };
1899
1900 /*
1901  * Software counter: cpu migrations
1902  */
1903
1904 static inline u64 get_cpu_migrations(struct perf_counter *counter)
1905 {
1906         struct task_struct *curr = counter->ctx->task;
1907
1908         if (curr)
1909                 return curr->se.nr_migrations;
1910         return cpu_nr_migrations(smp_processor_id());
1911 }
1912
1913 static void cpu_migrations_perf_counter_update(struct perf_counter *counter)
1914 {
1915         u64 prev, now;
1916         s64 delta;
1917
1918         prev = atomic64_read(&counter->hw.prev_count);
1919         now = get_cpu_migrations(counter);
1920
1921         atomic64_set(&counter->hw.prev_count, now);
1922
1923         delta = now - prev;
1924
1925         atomic64_add(delta, &counter->count);
1926 }
1927
1928 static void cpu_migrations_perf_counter_read(struct perf_counter *counter)
1929 {
1930         cpu_migrations_perf_counter_update(counter);
1931 }
1932
1933 static int cpu_migrations_perf_counter_enable(struct perf_counter *counter)
1934 {
1935         if (counter->prev_state <= PERF_COUNTER_STATE_OFF)
1936                 atomic64_set(&counter->hw.prev_count,
1937                              get_cpu_migrations(counter));
1938         return 0;
1939 }
1940
1941 static void cpu_migrations_perf_counter_disable(struct perf_counter *counter)
1942 {
1943         cpu_migrations_perf_counter_update(counter);
1944 }
1945
1946 static const struct hw_perf_counter_ops perf_ops_cpu_migrations = {
1947         .enable         = cpu_migrations_perf_counter_enable,
1948         .disable        = cpu_migrations_perf_counter_disable,
1949         .read           = cpu_migrations_perf_counter_read,
1950 };
1951
1952 #ifdef CONFIG_EVENT_PROFILE
1953 void perf_tpcounter_event(int event_id)
1954 {
1955         struct pt_regs *regs = get_irq_regs();
1956
1957         if (!regs)
1958                 regs = task_pt_regs(current);
1959
1960         __perf_swcounter_event(PERF_TYPE_TRACEPOINT, event_id, 1, 1, regs);
1961 }
1962
1963 extern int ftrace_profile_enable(int);
1964 extern void ftrace_profile_disable(int);
1965
1966 static void tp_perf_counter_destroy(struct perf_counter *counter)
1967 {
1968         ftrace_profile_disable(perf_event_id(&counter->hw_event));
1969 }
1970
1971 static const struct hw_perf_counter_ops *
1972 tp_perf_counter_init(struct perf_counter *counter)
1973 {
1974         int event_id = perf_event_id(&counter->hw_event);
1975         int ret;
1976
1977         ret = ftrace_profile_enable(event_id);
1978         if (ret)
1979                 return NULL;
1980
1981         counter->destroy = tp_perf_counter_destroy;
1982         counter->hw.irq_period = counter->hw_event.irq_period;
1983
1984         return &perf_ops_generic;
1985 }
1986 #else
1987 static const struct hw_perf_counter_ops *
1988 tp_perf_counter_init(struct perf_counter *counter)
1989 {
1990         return NULL;
1991 }
1992 #endif
1993
1994 static const struct hw_perf_counter_ops *
1995 sw_perf_counter_init(struct perf_counter *counter)
1996 {
1997         struct perf_counter_hw_event *hw_event = &counter->hw_event;
1998         const struct hw_perf_counter_ops *hw_ops = NULL;
1999         struct hw_perf_counter *hwc = &counter->hw;
2000
2001         /*
2002          * Software counters (currently) can't in general distinguish
2003          * between user, kernel and hypervisor events.
2004          * However, context switches and cpu migrations are considered
2005          * to be kernel events, and page faults are never hypervisor
2006          * events.
2007          */
2008         switch (perf_event_id(&counter->hw_event)) {
2009         case PERF_COUNT_CPU_CLOCK:
2010                 hw_ops = &perf_ops_cpu_clock;
2011
2012                 if (hw_event->irq_period && hw_event->irq_period < 10000)
2013                         hw_event->irq_period = 10000;
2014                 break;
2015         case PERF_COUNT_TASK_CLOCK:
2016                 /*
2017                  * If the user instantiates this as a per-cpu counter,
2018                  * use the cpu_clock counter instead.
2019                  */
2020                 if (counter->ctx->task)
2021                         hw_ops = &perf_ops_task_clock;
2022                 else
2023                         hw_ops = &perf_ops_cpu_clock;
2024
2025                 if (hw_event->irq_period && hw_event->irq_period < 10000)
2026                         hw_event->irq_period = 10000;
2027                 break;
2028         case PERF_COUNT_PAGE_FAULTS:
2029         case PERF_COUNT_PAGE_FAULTS_MIN:
2030         case PERF_COUNT_PAGE_FAULTS_MAJ:
2031         case PERF_COUNT_CONTEXT_SWITCHES:
2032                 hw_ops = &perf_ops_generic;
2033                 break;
2034         case PERF_COUNT_CPU_MIGRATIONS:
2035                 if (!counter->hw_event.exclude_kernel)
2036                         hw_ops = &perf_ops_cpu_migrations;
2037                 break;
2038         }
2039
2040         if (hw_ops)
2041                 hwc->irq_period = hw_event->irq_period;
2042
2043         return hw_ops;
2044 }
2045
2046 /*
2047  * Allocate and initialize a counter structure
2048  */
2049 static struct perf_counter *
2050 perf_counter_alloc(struct perf_counter_hw_event *hw_event,
2051                    int cpu,
2052                    struct perf_counter_context *ctx,
2053                    struct perf_counter *group_leader,
2054                    gfp_t gfpflags)
2055 {
2056         const struct hw_perf_counter_ops *hw_ops;
2057         struct perf_counter *counter;
2058
2059         counter = kzalloc(sizeof(*counter), gfpflags);
2060         if (!counter)
2061                 return NULL;
2062
2063         /*
2064          * Single counters are their own group leaders, with an
2065          * empty sibling list:
2066          */
2067         if (!group_leader)
2068                 group_leader = counter;
2069
2070         mutex_init(&counter->mutex);
2071         INIT_LIST_HEAD(&counter->list_entry);
2072         INIT_LIST_HEAD(&counter->event_entry);
2073         INIT_LIST_HEAD(&counter->sibling_list);
2074         init_waitqueue_head(&counter->waitq);
2075
2076         mutex_init(&counter->mmap_mutex);
2077
2078         INIT_LIST_HEAD(&counter->child_list);
2079
2080         counter->cpu                    = cpu;
2081         counter->hw_event               = *hw_event;
2082         counter->wakeup_pending         = 0;
2083         counter->group_leader           = group_leader;
2084         counter->hw_ops                 = NULL;
2085         counter->ctx                    = ctx;
2086
2087         counter->state = PERF_COUNTER_STATE_INACTIVE;
2088         if (hw_event->disabled)
2089                 counter->state = PERF_COUNTER_STATE_OFF;
2090
2091         hw_ops = NULL;
2092
2093         if (perf_event_raw(hw_event)) {
2094                 hw_ops = hw_perf_counter_init(counter);
2095                 goto done;
2096         }
2097
2098         switch (perf_event_type(hw_event)) {
2099         case PERF_TYPE_HARDWARE:
2100                 hw_ops = hw_perf_counter_init(counter);
2101                 break;
2102
2103         case PERF_TYPE_SOFTWARE:
2104                 hw_ops = sw_perf_counter_init(counter);
2105                 break;
2106
2107         case PERF_TYPE_TRACEPOINT:
2108                 hw_ops = tp_perf_counter_init(counter);
2109                 break;
2110         }
2111
2112         if (!hw_ops) {
2113                 kfree(counter);
2114                 return NULL;
2115         }
2116 done:
2117         counter->hw_ops = hw_ops;
2118
2119         return counter;
2120 }
2121
2122 /**
2123  * sys_perf_counter_open - open a performance counter, associate it to a task/cpu
2124  *
2125  * @hw_event_uptr:      event type attributes for monitoring/sampling
2126  * @pid:                target pid
2127  * @cpu:                target cpu
2128  * @group_fd:           group leader counter fd
2129  */
2130 SYSCALL_DEFINE5(perf_counter_open,
2131                 const struct perf_counter_hw_event __user *, hw_event_uptr,
2132                 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
2133 {
2134         struct perf_counter *counter, *group_leader;
2135         struct perf_counter_hw_event hw_event;
2136         struct perf_counter_context *ctx;
2137         struct file *counter_file = NULL;
2138         struct file *group_file = NULL;
2139         int fput_needed = 0;
2140         int fput_needed2 = 0;
2141         int ret;
2142
2143         /* for future expandability... */
2144         if (flags)
2145                 return -EINVAL;
2146
2147         if (copy_from_user(&hw_event, hw_event_uptr, sizeof(hw_event)) != 0)
2148                 return -EFAULT;
2149
2150         /*
2151          * Get the target context (task or percpu):
2152          */
2153         ctx = find_get_context(pid, cpu);
2154         if (IS_ERR(ctx))
2155                 return PTR_ERR(ctx);
2156
2157         /*
2158          * Look up the group leader (we will attach this counter to it):
2159          */
2160         group_leader = NULL;
2161         if (group_fd != -1) {
2162                 ret = -EINVAL;
2163                 group_file = fget_light(group_fd, &fput_needed);
2164                 if (!group_file)
2165                         goto err_put_context;
2166                 if (group_file->f_op != &perf_fops)
2167                         goto err_put_context;
2168
2169                 group_leader = group_file->private_data;
2170                 /*
2171                  * Do not allow a recursive hierarchy (this new sibling
2172                  * becoming part of another group-sibling):
2173                  */
2174                 if (group_leader->group_leader != group_leader)
2175                         goto err_put_context;
2176                 /*
2177                  * Do not allow to attach to a group in a different
2178                  * task or CPU context:
2179                  */
2180                 if (group_leader->ctx != ctx)
2181                         goto err_put_context;
2182                 /*
2183                  * Only a group leader can be exclusive or pinned
2184                  */
2185                 if (hw_event.exclusive || hw_event.pinned)
2186                         goto err_put_context;
2187         }
2188
2189         ret = -EINVAL;
2190         counter = perf_counter_alloc(&hw_event, cpu, ctx, group_leader,
2191                                      GFP_KERNEL);
2192         if (!counter)
2193                 goto err_put_context;
2194
2195         ret = anon_inode_getfd("[perf_counter]", &perf_fops, counter, 0);
2196         if (ret < 0)
2197                 goto err_free_put_context;
2198
2199         counter_file = fget_light(ret, &fput_needed2);
2200         if (!counter_file)
2201                 goto err_free_put_context;
2202
2203         counter->filp = counter_file;
2204         mutex_lock(&ctx->mutex);
2205         perf_install_in_context(ctx, counter, cpu);
2206         mutex_unlock(&ctx->mutex);
2207
2208         fput_light(counter_file, fput_needed2);
2209
2210 out_fput:
2211         fput_light(group_file, fput_needed);
2212
2213         return ret;
2214
2215 err_free_put_context:
2216         kfree(counter);
2217
2218 err_put_context:
2219         put_context(ctx);
2220
2221         goto out_fput;
2222 }
2223
2224 /*
2225  * Initialize the perf_counter context in a task_struct:
2226  */
2227 static void
2228 __perf_counter_init_context(struct perf_counter_context *ctx,
2229                             struct task_struct *task)
2230 {
2231         memset(ctx, 0, sizeof(*ctx));
2232         spin_lock_init(&ctx->lock);
2233         mutex_init(&ctx->mutex);
2234         INIT_LIST_HEAD(&ctx->counter_list);
2235         INIT_LIST_HEAD(&ctx->event_list);
2236         ctx->task = task;
2237 }
2238
2239 /*
2240  * inherit a counter from parent task to child task:
2241  */
2242 static struct perf_counter *
2243 inherit_counter(struct perf_counter *parent_counter,
2244               struct task_struct *parent,
2245               struct perf_counter_context *parent_ctx,
2246               struct task_struct *child,
2247               struct perf_counter *group_leader,
2248               struct perf_counter_context *child_ctx)
2249 {
2250         struct perf_counter *child_counter;
2251
2252         /*
2253          * Instead of creating recursive hierarchies of counters,
2254          * we link inherited counters back to the original parent,
2255          * which has a filp for sure, which we use as the reference
2256          * count:
2257          */
2258         if (parent_counter->parent)
2259                 parent_counter = parent_counter->parent;
2260
2261         child_counter = perf_counter_alloc(&parent_counter->hw_event,
2262                                            parent_counter->cpu, child_ctx,
2263                                            group_leader, GFP_KERNEL);
2264         if (!child_counter)
2265                 return NULL;
2266
2267         /*
2268          * Link it up in the child's context:
2269          */
2270         child_counter->task = child;
2271         list_add_counter(child_counter, child_ctx);
2272         child_ctx->nr_counters++;
2273
2274         child_counter->parent = parent_counter;
2275         /*
2276          * inherit into child's child as well:
2277          */
2278         child_counter->hw_event.inherit = 1;
2279
2280         /*
2281          * Get a reference to the parent filp - we will fput it
2282          * when the child counter exits. This is safe to do because
2283          * we are in the parent and we know that the filp still
2284          * exists and has a nonzero count:
2285          */
2286         atomic_long_inc(&parent_counter->filp->f_count);
2287
2288         /*
2289          * Link this into the parent counter's child list
2290          */
2291         mutex_lock(&parent_counter->mutex);
2292         list_add_tail(&child_counter->child_list, &parent_counter->child_list);
2293
2294         /*
2295          * Make the child state follow the state of the parent counter,
2296          * not its hw_event.disabled bit.  We hold the parent's mutex,
2297          * so we won't race with perf_counter_{en,dis}able_family.
2298          */
2299         if (parent_counter->state >= PERF_COUNTER_STATE_INACTIVE)
2300                 child_counter->state = PERF_COUNTER_STATE_INACTIVE;
2301         else
2302                 child_counter->state = PERF_COUNTER_STATE_OFF;
2303
2304         mutex_unlock(&parent_counter->mutex);
2305
2306         return child_counter;
2307 }
2308
2309 static int inherit_group(struct perf_counter *parent_counter,
2310               struct task_struct *parent,
2311               struct perf_counter_context *parent_ctx,
2312               struct task_struct *child,
2313               struct perf_counter_context *child_ctx)
2314 {
2315         struct perf_counter *leader;
2316         struct perf_counter *sub;
2317
2318         leader = inherit_counter(parent_counter, parent, parent_ctx,
2319                                  child, NULL, child_ctx);
2320         if (!leader)
2321                 return -ENOMEM;
2322         list_for_each_entry(sub, &parent_counter->sibling_list, list_entry) {
2323                 if (!inherit_counter(sub, parent, parent_ctx,
2324                                      child, leader, child_ctx))
2325                         return -ENOMEM;
2326         }
2327         return 0;
2328 }
2329
2330 static void sync_child_counter(struct perf_counter *child_counter,
2331                                struct perf_counter *parent_counter)
2332 {
2333         u64 parent_val, child_val;
2334
2335         parent_val = atomic64_read(&parent_counter->count);
2336         child_val = atomic64_read(&child_counter->count);
2337
2338         /*
2339          * Add back the child's count to the parent's count:
2340          */
2341         atomic64_add(child_val, &parent_counter->count);
2342
2343         /*
2344          * Remove this counter from the parent's list
2345          */
2346         mutex_lock(&parent_counter->mutex);
2347         list_del_init(&child_counter->child_list);
2348         mutex_unlock(&parent_counter->mutex);
2349
2350         /*
2351          * Release the parent counter, if this was the last
2352          * reference to it.
2353          */
2354         fput(parent_counter->filp);
2355 }
2356
2357 static void
2358 __perf_counter_exit_task(struct task_struct *child,
2359                          struct perf_counter *child_counter,
2360                          struct perf_counter_context *child_ctx)
2361 {
2362         struct perf_counter *parent_counter;
2363         struct perf_counter *sub, *tmp;
2364
2365         /*
2366          * If we do not self-reap then we have to wait for the
2367          * child task to unschedule (it will happen for sure),
2368          * so that its counter is at its final count. (This
2369          * condition triggers rarely - child tasks usually get
2370          * off their CPU before the parent has a chance to
2371          * get this far into the reaping action)
2372          */
2373         if (child != current) {
2374                 wait_task_inactive(child, 0);
2375                 list_del_init(&child_counter->list_entry);
2376         } else {
2377                 struct perf_cpu_context *cpuctx;
2378                 unsigned long flags;
2379                 u64 perf_flags;
2380
2381                 /*
2382                  * Disable and unlink this counter.
2383                  *
2384                  * Be careful about zapping the list - IRQ/NMI context
2385                  * could still be processing it:
2386                  */
2387                 curr_rq_lock_irq_save(&flags);
2388                 perf_flags = hw_perf_save_disable();
2389
2390                 cpuctx = &__get_cpu_var(perf_cpu_context);
2391
2392                 group_sched_out(child_counter, cpuctx, child_ctx);
2393
2394                 list_del_init(&child_counter->list_entry);
2395
2396                 child_ctx->nr_counters--;
2397
2398                 hw_perf_restore(perf_flags);
2399                 curr_rq_unlock_irq_restore(&flags);
2400         }
2401
2402         parent_counter = child_counter->parent;
2403         /*
2404          * It can happen that parent exits first, and has counters
2405          * that are still around due to the child reference. These
2406          * counters need to be zapped - but otherwise linger.
2407          */
2408         if (parent_counter) {
2409                 sync_child_counter(child_counter, parent_counter);
2410                 list_for_each_entry_safe(sub, tmp, &child_counter->sibling_list,
2411                                          list_entry) {
2412                         if (sub->parent) {
2413                                 sync_child_counter(sub, sub->parent);
2414                                 free_counter(sub);
2415                         }
2416                 }
2417                 free_counter(child_counter);
2418         }
2419 }
2420
2421 /*
2422  * When a child task exits, feed back counter values to parent counters.
2423  *
2424  * Note: we may be running in child context, but the PID is not hashed
2425  * anymore so new counters will not be added.
2426  */
2427 void perf_counter_exit_task(struct task_struct *child)
2428 {
2429         struct perf_counter *child_counter, *tmp;
2430         struct perf_counter_context *child_ctx;
2431
2432         child_ctx = &child->perf_counter_ctx;
2433
2434         if (likely(!child_ctx->nr_counters))
2435                 return;
2436
2437         list_for_each_entry_safe(child_counter, tmp, &child_ctx->counter_list,
2438                                  list_entry)
2439                 __perf_counter_exit_task(child, child_counter, child_ctx);
2440 }
2441
2442 /*
2443  * Initialize the perf_counter context in task_struct
2444  */
2445 void perf_counter_init_task(struct task_struct *child)
2446 {
2447         struct perf_counter_context *child_ctx, *parent_ctx;
2448         struct perf_counter *counter;
2449         struct task_struct *parent = current;
2450
2451         child_ctx  =  &child->perf_counter_ctx;
2452         parent_ctx = &parent->perf_counter_ctx;
2453
2454         __perf_counter_init_context(child_ctx, child);
2455
2456         /*
2457          * This is executed from the parent task context, so inherit
2458          * counters that have been marked for cloning:
2459          */
2460
2461         if (likely(!parent_ctx->nr_counters))
2462                 return;
2463
2464         /*
2465          * Lock the parent list. No need to lock the child - not PID
2466          * hashed yet and not running, so nobody can access it.
2467          */
2468         mutex_lock(&parent_ctx->mutex);
2469
2470         /*
2471          * We dont have to disable NMIs - we are only looking at
2472          * the list, not manipulating it:
2473          */
2474         list_for_each_entry(counter, &parent_ctx->counter_list, list_entry) {
2475                 if (!counter->hw_event.inherit)
2476                         continue;
2477
2478                 if (inherit_group(counter, parent,
2479                                   parent_ctx, child, child_ctx))
2480                         break;
2481         }
2482
2483         mutex_unlock(&parent_ctx->mutex);
2484 }
2485
2486 static void __cpuinit perf_counter_init_cpu(int cpu)
2487 {
2488         struct perf_cpu_context *cpuctx;
2489
2490         cpuctx = &per_cpu(perf_cpu_context, cpu);
2491         __perf_counter_init_context(&cpuctx->ctx, NULL);
2492
2493         mutex_lock(&perf_resource_mutex);
2494         cpuctx->max_pertask = perf_max_counters - perf_reserved_percpu;
2495         mutex_unlock(&perf_resource_mutex);
2496
2497         hw_perf_counter_setup(cpu);
2498 }
2499
2500 #ifdef CONFIG_HOTPLUG_CPU
2501 static void __perf_counter_exit_cpu(void *info)
2502 {
2503         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
2504         struct perf_counter_context *ctx = &cpuctx->ctx;
2505         struct perf_counter *counter, *tmp;
2506
2507         list_for_each_entry_safe(counter, tmp, &ctx->counter_list, list_entry)
2508                 __perf_counter_remove_from_context(counter);
2509 }
2510 static void perf_counter_exit_cpu(int cpu)
2511 {
2512         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
2513         struct perf_counter_context *ctx = &cpuctx->ctx;
2514
2515         mutex_lock(&ctx->mutex);
2516         smp_call_function_single(cpu, __perf_counter_exit_cpu, NULL, 1);
2517         mutex_unlock(&ctx->mutex);
2518 }
2519 #else
2520 static inline void perf_counter_exit_cpu(int cpu) { }
2521 #endif
2522
2523 static int __cpuinit
2524 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
2525 {
2526         unsigned int cpu = (long)hcpu;
2527
2528         switch (action) {
2529
2530         case CPU_UP_PREPARE:
2531         case CPU_UP_PREPARE_FROZEN:
2532                 perf_counter_init_cpu(cpu);
2533                 break;
2534
2535         case CPU_DOWN_PREPARE:
2536         case CPU_DOWN_PREPARE_FROZEN:
2537                 perf_counter_exit_cpu(cpu);
2538                 break;
2539
2540         default:
2541                 break;
2542         }
2543
2544         return NOTIFY_OK;
2545 }
2546
2547 static struct notifier_block __cpuinitdata perf_cpu_nb = {
2548         .notifier_call          = perf_cpu_notify,
2549 };
2550
2551 static int __init perf_counter_init(void)
2552 {
2553         perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
2554                         (void *)(long)smp_processor_id());
2555         register_cpu_notifier(&perf_cpu_nb);
2556
2557         return 0;
2558 }
2559 early_initcall(perf_counter_init);
2560
2561 static ssize_t perf_show_reserve_percpu(struct sysdev_class *class, char *buf)
2562 {
2563         return sprintf(buf, "%d\n", perf_reserved_percpu);
2564 }
2565
2566 static ssize_t
2567 perf_set_reserve_percpu(struct sysdev_class *class,
2568                         const char *buf,
2569                         size_t count)
2570 {
2571         struct perf_cpu_context *cpuctx;
2572         unsigned long val;
2573         int err, cpu, mpt;
2574
2575         err = strict_strtoul(buf, 10, &val);
2576         if (err)
2577                 return err;
2578         if (val > perf_max_counters)
2579                 return -EINVAL;
2580
2581         mutex_lock(&perf_resource_mutex);
2582         perf_reserved_percpu = val;
2583         for_each_online_cpu(cpu) {
2584                 cpuctx = &per_cpu(perf_cpu_context, cpu);
2585                 spin_lock_irq(&cpuctx->ctx.lock);
2586                 mpt = min(perf_max_counters - cpuctx->ctx.nr_counters,
2587                           perf_max_counters - perf_reserved_percpu);
2588                 cpuctx->max_pertask = mpt;
2589                 spin_unlock_irq(&cpuctx->ctx.lock);
2590         }
2591         mutex_unlock(&perf_resource_mutex);
2592
2593         return count;
2594 }
2595
2596 static ssize_t perf_show_overcommit(struct sysdev_class *class, char *buf)
2597 {
2598         return sprintf(buf, "%d\n", perf_overcommit);
2599 }
2600
2601 static ssize_t
2602 perf_set_overcommit(struct sysdev_class *class, const char *buf, size_t count)
2603 {
2604         unsigned long val;
2605         int err;
2606
2607         err = strict_strtoul(buf, 10, &val);
2608         if (err)
2609                 return err;
2610         if (val > 1)
2611                 return -EINVAL;
2612
2613         mutex_lock(&perf_resource_mutex);
2614         perf_overcommit = val;
2615         mutex_unlock(&perf_resource_mutex);
2616
2617         return count;
2618 }
2619
2620 static SYSDEV_CLASS_ATTR(
2621                                 reserve_percpu,
2622                                 0644,
2623                                 perf_show_reserve_percpu,
2624                                 perf_set_reserve_percpu
2625                         );
2626
2627 static SYSDEV_CLASS_ATTR(
2628                                 overcommit,
2629                                 0644,
2630                                 perf_show_overcommit,
2631                                 perf_set_overcommit
2632                         );
2633
2634 static struct attribute *perfclass_attrs[] = {
2635         &attr_reserve_percpu.attr,
2636         &attr_overcommit.attr,
2637         NULL
2638 };
2639
2640 static struct attribute_group perfclass_attr_group = {
2641         .attrs                  = perfclass_attrs,
2642         .name                   = "perf_counters",
2643 };
2644
2645 static int __init perf_counter_sysfs_init(void)
2646 {
2647         return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
2648                                   &perfclass_attr_group);
2649 }
2650 device_initcall(perf_counter_sysfs_init);