Merge branch 'perfcounters-fixes-for-linus' of git://git.kernel.org/pub/scm/linux...
[pandora-kernel.git] / kernel / perf_counter.c
1 /*
2  * Performance counter core code
3  *
4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7  *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8  *
9  *  For licensing details see kernel-base/COPYING
10  */
11
12 #include <linux/fs.h>
13 #include <linux/mm.h>
14 #include <linux/cpu.h>
15 #include <linux/smp.h>
16 #include <linux/file.h>
17 #include <linux/poll.h>
18 #include <linux/sysfs.h>
19 #include <linux/dcache.h>
20 #include <linux/percpu.h>
21 #include <linux/ptrace.h>
22 #include <linux/vmstat.h>
23 #include <linux/hardirq.h>
24 #include <linux/rculist.h>
25 #include <linux/uaccess.h>
26 #include <linux/syscalls.h>
27 #include <linux/anon_inodes.h>
28 #include <linux/kernel_stat.h>
29 #include <linux/perf_counter.h>
30
31 #include <asm/irq_regs.h>
32
33 /*
34  * Each CPU has a list of per CPU counters:
35  */
36 DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
37
38 int perf_max_counters __read_mostly = 1;
39 static int perf_reserved_percpu __read_mostly;
40 static int perf_overcommit __read_mostly = 1;
41
42 static atomic_t nr_counters __read_mostly;
43 static atomic_t nr_mmap_counters __read_mostly;
44 static atomic_t nr_comm_counters __read_mostly;
45 static atomic_t nr_task_counters __read_mostly;
46
47 /*
48  * perf counter paranoia level:
49  *  0 - not paranoid
50  *  1 - disallow cpu counters to unpriv
51  *  2 - disallow kernel profiling to unpriv
52  */
53 int sysctl_perf_counter_paranoid __read_mostly = 1;
54
55 static inline bool perf_paranoid_cpu(void)
56 {
57         return sysctl_perf_counter_paranoid > 0;
58 }
59
60 static inline bool perf_paranoid_kernel(void)
61 {
62         return sysctl_perf_counter_paranoid > 1;
63 }
64
65 int sysctl_perf_counter_mlock __read_mostly = 512; /* 'free' kb per user */
66
67 /*
68  * max perf counter sample rate
69  */
70 int sysctl_perf_counter_sample_rate __read_mostly = 100000;
71
72 static atomic64_t perf_counter_id;
73
74 /*
75  * Lock for (sysadmin-configurable) counter reservations:
76  */
77 static DEFINE_SPINLOCK(perf_resource_lock);
78
79 /*
80  * Architecture provided APIs - weak aliases:
81  */
82 extern __weak const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
83 {
84         return NULL;
85 }
86
87 void __weak hw_perf_disable(void)               { barrier(); }
88 void __weak hw_perf_enable(void)                { barrier(); }
89
90 void __weak hw_perf_counter_setup(int cpu)      { barrier(); }
91 void __weak hw_perf_counter_setup_online(int cpu)       { barrier(); }
92
93 int __weak
94 hw_perf_group_sched_in(struct perf_counter *group_leader,
95                struct perf_cpu_context *cpuctx,
96                struct perf_counter_context *ctx, int cpu)
97 {
98         return 0;
99 }
100
101 void __weak perf_counter_print_debug(void)      { }
102
103 static DEFINE_PER_CPU(int, disable_count);
104
105 void __perf_disable(void)
106 {
107         __get_cpu_var(disable_count)++;
108 }
109
110 bool __perf_enable(void)
111 {
112         return !--__get_cpu_var(disable_count);
113 }
114
115 void perf_disable(void)
116 {
117         __perf_disable();
118         hw_perf_disable();
119 }
120
121 void perf_enable(void)
122 {
123         if (__perf_enable())
124                 hw_perf_enable();
125 }
126
127 static void get_ctx(struct perf_counter_context *ctx)
128 {
129         WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
130 }
131
132 static void free_ctx(struct rcu_head *head)
133 {
134         struct perf_counter_context *ctx;
135
136         ctx = container_of(head, struct perf_counter_context, rcu_head);
137         kfree(ctx);
138 }
139
140 static void put_ctx(struct perf_counter_context *ctx)
141 {
142         if (atomic_dec_and_test(&ctx->refcount)) {
143                 if (ctx->parent_ctx)
144                         put_ctx(ctx->parent_ctx);
145                 if (ctx->task)
146                         put_task_struct(ctx->task);
147                 call_rcu(&ctx->rcu_head, free_ctx);
148         }
149 }
150
151 static void unclone_ctx(struct perf_counter_context *ctx)
152 {
153         if (ctx->parent_ctx) {
154                 put_ctx(ctx->parent_ctx);
155                 ctx->parent_ctx = NULL;
156         }
157 }
158
159 /*
160  * If we inherit counters we want to return the parent counter id
161  * to userspace.
162  */
163 static u64 primary_counter_id(struct perf_counter *counter)
164 {
165         u64 id = counter->id;
166
167         if (counter->parent)
168                 id = counter->parent->id;
169
170         return id;
171 }
172
173 /*
174  * Get the perf_counter_context for a task and lock it.
175  * This has to cope with with the fact that until it is locked,
176  * the context could get moved to another task.
177  */
178 static struct perf_counter_context *
179 perf_lock_task_context(struct task_struct *task, unsigned long *flags)
180 {
181         struct perf_counter_context *ctx;
182
183         rcu_read_lock();
184  retry:
185         ctx = rcu_dereference(task->perf_counter_ctxp);
186         if (ctx) {
187                 /*
188                  * If this context is a clone of another, it might
189                  * get swapped for another underneath us by
190                  * perf_counter_task_sched_out, though the
191                  * rcu_read_lock() protects us from any context
192                  * getting freed.  Lock the context and check if it
193                  * got swapped before we could get the lock, and retry
194                  * if so.  If we locked the right context, then it
195                  * can't get swapped on us any more.
196                  */
197                 spin_lock_irqsave(&ctx->lock, *flags);
198                 if (ctx != rcu_dereference(task->perf_counter_ctxp)) {
199                         spin_unlock_irqrestore(&ctx->lock, *flags);
200                         goto retry;
201                 }
202
203                 if (!atomic_inc_not_zero(&ctx->refcount)) {
204                         spin_unlock_irqrestore(&ctx->lock, *flags);
205                         ctx = NULL;
206                 }
207         }
208         rcu_read_unlock();
209         return ctx;
210 }
211
212 /*
213  * Get the context for a task and increment its pin_count so it
214  * can't get swapped to another task.  This also increments its
215  * reference count so that the context can't get freed.
216  */
217 static struct perf_counter_context *perf_pin_task_context(struct task_struct *task)
218 {
219         struct perf_counter_context *ctx;
220         unsigned long flags;
221
222         ctx = perf_lock_task_context(task, &flags);
223         if (ctx) {
224                 ++ctx->pin_count;
225                 spin_unlock_irqrestore(&ctx->lock, flags);
226         }
227         return ctx;
228 }
229
230 static void perf_unpin_context(struct perf_counter_context *ctx)
231 {
232         unsigned long flags;
233
234         spin_lock_irqsave(&ctx->lock, flags);
235         --ctx->pin_count;
236         spin_unlock_irqrestore(&ctx->lock, flags);
237         put_ctx(ctx);
238 }
239
240 /*
241  * Add a counter from the lists for its context.
242  * Must be called with ctx->mutex and ctx->lock held.
243  */
244 static void
245 list_add_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
246 {
247         struct perf_counter *group_leader = counter->group_leader;
248
249         /*
250          * Depending on whether it is a standalone or sibling counter,
251          * add it straight to the context's counter list, or to the group
252          * leader's sibling list:
253          */
254         if (group_leader == counter)
255                 list_add_tail(&counter->list_entry, &ctx->counter_list);
256         else {
257                 list_add_tail(&counter->list_entry, &group_leader->sibling_list);
258                 group_leader->nr_siblings++;
259         }
260
261         list_add_rcu(&counter->event_entry, &ctx->event_list);
262         ctx->nr_counters++;
263         if (counter->attr.inherit_stat)
264                 ctx->nr_stat++;
265 }
266
267 /*
268  * Remove a counter from the lists for its context.
269  * Must be called with ctx->mutex and ctx->lock held.
270  */
271 static void
272 list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
273 {
274         struct perf_counter *sibling, *tmp;
275
276         if (list_empty(&counter->list_entry))
277                 return;
278         ctx->nr_counters--;
279         if (counter->attr.inherit_stat)
280                 ctx->nr_stat--;
281
282         list_del_init(&counter->list_entry);
283         list_del_rcu(&counter->event_entry);
284
285         if (counter->group_leader != counter)
286                 counter->group_leader->nr_siblings--;
287
288         /*
289          * If this was a group counter with sibling counters then
290          * upgrade the siblings to singleton counters by adding them
291          * to the context list directly:
292          */
293         list_for_each_entry_safe(sibling, tmp,
294                                  &counter->sibling_list, list_entry) {
295
296                 list_move_tail(&sibling->list_entry, &ctx->counter_list);
297                 sibling->group_leader = sibling;
298         }
299 }
300
301 static void
302 counter_sched_out(struct perf_counter *counter,
303                   struct perf_cpu_context *cpuctx,
304                   struct perf_counter_context *ctx)
305 {
306         if (counter->state != PERF_COUNTER_STATE_ACTIVE)
307                 return;
308
309         counter->state = PERF_COUNTER_STATE_INACTIVE;
310         if (counter->pending_disable) {
311                 counter->pending_disable = 0;
312                 counter->state = PERF_COUNTER_STATE_OFF;
313         }
314         counter->tstamp_stopped = ctx->time;
315         counter->pmu->disable(counter);
316         counter->oncpu = -1;
317
318         if (!is_software_counter(counter))
319                 cpuctx->active_oncpu--;
320         ctx->nr_active--;
321         if (counter->attr.exclusive || !cpuctx->active_oncpu)
322                 cpuctx->exclusive = 0;
323 }
324
325 static void
326 group_sched_out(struct perf_counter *group_counter,
327                 struct perf_cpu_context *cpuctx,
328                 struct perf_counter_context *ctx)
329 {
330         struct perf_counter *counter;
331
332         if (group_counter->state != PERF_COUNTER_STATE_ACTIVE)
333                 return;
334
335         counter_sched_out(group_counter, cpuctx, ctx);
336
337         /*
338          * Schedule out siblings (if any):
339          */
340         list_for_each_entry(counter, &group_counter->sibling_list, list_entry)
341                 counter_sched_out(counter, cpuctx, ctx);
342
343         if (group_counter->attr.exclusive)
344                 cpuctx->exclusive = 0;
345 }
346
347 /*
348  * Cross CPU call to remove a performance counter
349  *
350  * We disable the counter on the hardware level first. After that we
351  * remove it from the context list.
352  */
353 static void __perf_counter_remove_from_context(void *info)
354 {
355         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
356         struct perf_counter *counter = info;
357         struct perf_counter_context *ctx = counter->ctx;
358
359         /*
360          * If this is a task context, we need to check whether it is
361          * the current task context of this cpu. If not it has been
362          * scheduled out before the smp call arrived.
363          */
364         if (ctx->task && cpuctx->task_ctx != ctx)
365                 return;
366
367         spin_lock(&ctx->lock);
368         /*
369          * Protect the list operation against NMI by disabling the
370          * counters on a global level.
371          */
372         perf_disable();
373
374         counter_sched_out(counter, cpuctx, ctx);
375
376         list_del_counter(counter, ctx);
377
378         if (!ctx->task) {
379                 /*
380                  * Allow more per task counters with respect to the
381                  * reservation:
382                  */
383                 cpuctx->max_pertask =
384                         min(perf_max_counters - ctx->nr_counters,
385                             perf_max_counters - perf_reserved_percpu);
386         }
387
388         perf_enable();
389         spin_unlock(&ctx->lock);
390 }
391
392
393 /*
394  * Remove the counter from a task's (or a CPU's) list of counters.
395  *
396  * Must be called with ctx->mutex held.
397  *
398  * CPU counters are removed with a smp call. For task counters we only
399  * call when the task is on a CPU.
400  *
401  * If counter->ctx is a cloned context, callers must make sure that
402  * every task struct that counter->ctx->task could possibly point to
403  * remains valid.  This is OK when called from perf_release since
404  * that only calls us on the top-level context, which can't be a clone.
405  * When called from perf_counter_exit_task, it's OK because the
406  * context has been detached from its task.
407  */
408 static void perf_counter_remove_from_context(struct perf_counter *counter)
409 {
410         struct perf_counter_context *ctx = counter->ctx;
411         struct task_struct *task = ctx->task;
412
413         if (!task) {
414                 /*
415                  * Per cpu counters are removed via an smp call and
416                  * the removal is always sucessful.
417                  */
418                 smp_call_function_single(counter->cpu,
419                                          __perf_counter_remove_from_context,
420                                          counter, 1);
421                 return;
422         }
423
424 retry:
425         task_oncpu_function_call(task, __perf_counter_remove_from_context,
426                                  counter);
427
428         spin_lock_irq(&ctx->lock);
429         /*
430          * If the context is active we need to retry the smp call.
431          */
432         if (ctx->nr_active && !list_empty(&counter->list_entry)) {
433                 spin_unlock_irq(&ctx->lock);
434                 goto retry;
435         }
436
437         /*
438          * The lock prevents that this context is scheduled in so we
439          * can remove the counter safely, if the call above did not
440          * succeed.
441          */
442         if (!list_empty(&counter->list_entry)) {
443                 list_del_counter(counter, ctx);
444         }
445         spin_unlock_irq(&ctx->lock);
446 }
447
448 static inline u64 perf_clock(void)
449 {
450         return cpu_clock(smp_processor_id());
451 }
452
453 /*
454  * Update the record of the current time in a context.
455  */
456 static void update_context_time(struct perf_counter_context *ctx)
457 {
458         u64 now = perf_clock();
459
460         ctx->time += now - ctx->timestamp;
461         ctx->timestamp = now;
462 }
463
464 /*
465  * Update the total_time_enabled and total_time_running fields for a counter.
466  */
467 static void update_counter_times(struct perf_counter *counter)
468 {
469         struct perf_counter_context *ctx = counter->ctx;
470         u64 run_end;
471
472         if (counter->state < PERF_COUNTER_STATE_INACTIVE)
473                 return;
474
475         counter->total_time_enabled = ctx->time - counter->tstamp_enabled;
476
477         if (counter->state == PERF_COUNTER_STATE_INACTIVE)
478                 run_end = counter->tstamp_stopped;
479         else
480                 run_end = ctx->time;
481
482         counter->total_time_running = run_end - counter->tstamp_running;
483 }
484
485 /*
486  * Update total_time_enabled and total_time_running for all counters in a group.
487  */
488 static void update_group_times(struct perf_counter *leader)
489 {
490         struct perf_counter *counter;
491
492         update_counter_times(leader);
493         list_for_each_entry(counter, &leader->sibling_list, list_entry)
494                 update_counter_times(counter);
495 }
496
497 /*
498  * Cross CPU call to disable a performance counter
499  */
500 static void __perf_counter_disable(void *info)
501 {
502         struct perf_counter *counter = info;
503         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
504         struct perf_counter_context *ctx = counter->ctx;
505
506         /*
507          * If this is a per-task counter, need to check whether this
508          * counter's task is the current task on this cpu.
509          */
510         if (ctx->task && cpuctx->task_ctx != ctx)
511                 return;
512
513         spin_lock(&ctx->lock);
514
515         /*
516          * If the counter is on, turn it off.
517          * If it is in error state, leave it in error state.
518          */
519         if (counter->state >= PERF_COUNTER_STATE_INACTIVE) {
520                 update_context_time(ctx);
521                 update_counter_times(counter);
522                 if (counter == counter->group_leader)
523                         group_sched_out(counter, cpuctx, ctx);
524                 else
525                         counter_sched_out(counter, cpuctx, ctx);
526                 counter->state = PERF_COUNTER_STATE_OFF;
527         }
528
529         spin_unlock(&ctx->lock);
530 }
531
532 /*
533  * Disable a counter.
534  *
535  * If counter->ctx is a cloned context, callers must make sure that
536  * every task struct that counter->ctx->task could possibly point to
537  * remains valid.  This condition is satisifed when called through
538  * perf_counter_for_each_child or perf_counter_for_each because they
539  * hold the top-level counter's child_mutex, so any descendant that
540  * goes to exit will block in sync_child_counter.
541  * When called from perf_pending_counter it's OK because counter->ctx
542  * is the current context on this CPU and preemption is disabled,
543  * hence we can't get into perf_counter_task_sched_out for this context.
544  */
545 static void perf_counter_disable(struct perf_counter *counter)
546 {
547         struct perf_counter_context *ctx = counter->ctx;
548         struct task_struct *task = ctx->task;
549
550         if (!task) {
551                 /*
552                  * Disable the counter on the cpu that it's on
553                  */
554                 smp_call_function_single(counter->cpu, __perf_counter_disable,
555                                          counter, 1);
556                 return;
557         }
558
559  retry:
560         task_oncpu_function_call(task, __perf_counter_disable, counter);
561
562         spin_lock_irq(&ctx->lock);
563         /*
564          * If the counter is still active, we need to retry the cross-call.
565          */
566         if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
567                 spin_unlock_irq(&ctx->lock);
568                 goto retry;
569         }
570
571         /*
572          * Since we have the lock this context can't be scheduled
573          * in, so we can change the state safely.
574          */
575         if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
576                 update_counter_times(counter);
577                 counter->state = PERF_COUNTER_STATE_OFF;
578         }
579
580         spin_unlock_irq(&ctx->lock);
581 }
582
583 static int
584 counter_sched_in(struct perf_counter *counter,
585                  struct perf_cpu_context *cpuctx,
586                  struct perf_counter_context *ctx,
587                  int cpu)
588 {
589         if (counter->state <= PERF_COUNTER_STATE_OFF)
590                 return 0;
591
592         counter->state = PERF_COUNTER_STATE_ACTIVE;
593         counter->oncpu = cpu;   /* TODO: put 'cpu' into cpuctx->cpu */
594         /*
595          * The new state must be visible before we turn it on in the hardware:
596          */
597         smp_wmb();
598
599         if (counter->pmu->enable(counter)) {
600                 counter->state = PERF_COUNTER_STATE_INACTIVE;
601                 counter->oncpu = -1;
602                 return -EAGAIN;
603         }
604
605         counter->tstamp_running += ctx->time - counter->tstamp_stopped;
606
607         if (!is_software_counter(counter))
608                 cpuctx->active_oncpu++;
609         ctx->nr_active++;
610
611         if (counter->attr.exclusive)
612                 cpuctx->exclusive = 1;
613
614         return 0;
615 }
616
617 static int
618 group_sched_in(struct perf_counter *group_counter,
619                struct perf_cpu_context *cpuctx,
620                struct perf_counter_context *ctx,
621                int cpu)
622 {
623         struct perf_counter *counter, *partial_group;
624         int ret;
625
626         if (group_counter->state == PERF_COUNTER_STATE_OFF)
627                 return 0;
628
629         ret = hw_perf_group_sched_in(group_counter, cpuctx, ctx, cpu);
630         if (ret)
631                 return ret < 0 ? ret : 0;
632
633         if (counter_sched_in(group_counter, cpuctx, ctx, cpu))
634                 return -EAGAIN;
635
636         /*
637          * Schedule in siblings as one group (if any):
638          */
639         list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
640                 if (counter_sched_in(counter, cpuctx, ctx, cpu)) {
641                         partial_group = counter;
642                         goto group_error;
643                 }
644         }
645
646         return 0;
647
648 group_error:
649         /*
650          * Groups can be scheduled in as one unit only, so undo any
651          * partial group before returning:
652          */
653         list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
654                 if (counter == partial_group)
655                         break;
656                 counter_sched_out(counter, cpuctx, ctx);
657         }
658         counter_sched_out(group_counter, cpuctx, ctx);
659
660         return -EAGAIN;
661 }
662
663 /*
664  * Return 1 for a group consisting entirely of software counters,
665  * 0 if the group contains any hardware counters.
666  */
667 static int is_software_only_group(struct perf_counter *leader)
668 {
669         struct perf_counter *counter;
670
671         if (!is_software_counter(leader))
672                 return 0;
673
674         list_for_each_entry(counter, &leader->sibling_list, list_entry)
675                 if (!is_software_counter(counter))
676                         return 0;
677
678         return 1;
679 }
680
681 /*
682  * Work out whether we can put this counter group on the CPU now.
683  */
684 static int group_can_go_on(struct perf_counter *counter,
685                            struct perf_cpu_context *cpuctx,
686                            int can_add_hw)
687 {
688         /*
689          * Groups consisting entirely of software counters can always go on.
690          */
691         if (is_software_only_group(counter))
692                 return 1;
693         /*
694          * If an exclusive group is already on, no other hardware
695          * counters can go on.
696          */
697         if (cpuctx->exclusive)
698                 return 0;
699         /*
700          * If this group is exclusive and there are already
701          * counters on the CPU, it can't go on.
702          */
703         if (counter->attr.exclusive && cpuctx->active_oncpu)
704                 return 0;
705         /*
706          * Otherwise, try to add it if all previous groups were able
707          * to go on.
708          */
709         return can_add_hw;
710 }
711
712 static void add_counter_to_ctx(struct perf_counter *counter,
713                                struct perf_counter_context *ctx)
714 {
715         list_add_counter(counter, ctx);
716         counter->tstamp_enabled = ctx->time;
717         counter->tstamp_running = ctx->time;
718         counter->tstamp_stopped = ctx->time;
719 }
720
721 /*
722  * Cross CPU call to install and enable a performance counter
723  *
724  * Must be called with ctx->mutex held
725  */
726 static void __perf_install_in_context(void *info)
727 {
728         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
729         struct perf_counter *counter = info;
730         struct perf_counter_context *ctx = counter->ctx;
731         struct perf_counter *leader = counter->group_leader;
732         int cpu = smp_processor_id();
733         int err;
734
735         /*
736          * If this is a task context, we need to check whether it is
737          * the current task context of this cpu. If not it has been
738          * scheduled out before the smp call arrived.
739          * Or possibly this is the right context but it isn't
740          * on this cpu because it had no counters.
741          */
742         if (ctx->task && cpuctx->task_ctx != ctx) {
743                 if (cpuctx->task_ctx || ctx->task != current)
744                         return;
745                 cpuctx->task_ctx = ctx;
746         }
747
748         spin_lock(&ctx->lock);
749         ctx->is_active = 1;
750         update_context_time(ctx);
751
752         /*
753          * Protect the list operation against NMI by disabling the
754          * counters on a global level. NOP for non NMI based counters.
755          */
756         perf_disable();
757
758         add_counter_to_ctx(counter, ctx);
759
760         /*
761          * Don't put the counter on if it is disabled or if
762          * it is in a group and the group isn't on.
763          */
764         if (counter->state != PERF_COUNTER_STATE_INACTIVE ||
765             (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE))
766                 goto unlock;
767
768         /*
769          * An exclusive counter can't go on if there are already active
770          * hardware counters, and no hardware counter can go on if there
771          * is already an exclusive counter on.
772          */
773         if (!group_can_go_on(counter, cpuctx, 1))
774                 err = -EEXIST;
775         else
776                 err = counter_sched_in(counter, cpuctx, ctx, cpu);
777
778         if (err) {
779                 /*
780                  * This counter couldn't go on.  If it is in a group
781                  * then we have to pull the whole group off.
782                  * If the counter group is pinned then put it in error state.
783                  */
784                 if (leader != counter)
785                         group_sched_out(leader, cpuctx, ctx);
786                 if (leader->attr.pinned) {
787                         update_group_times(leader);
788                         leader->state = PERF_COUNTER_STATE_ERROR;
789                 }
790         }
791
792         if (!err && !ctx->task && cpuctx->max_pertask)
793                 cpuctx->max_pertask--;
794
795  unlock:
796         perf_enable();
797
798         spin_unlock(&ctx->lock);
799 }
800
801 /*
802  * Attach a performance counter to a context
803  *
804  * First we add the counter to the list with the hardware enable bit
805  * in counter->hw_config cleared.
806  *
807  * If the counter is attached to a task which is on a CPU we use a smp
808  * call to enable it in the task context. The task might have been
809  * scheduled away, but we check this in the smp call again.
810  *
811  * Must be called with ctx->mutex held.
812  */
813 static void
814 perf_install_in_context(struct perf_counter_context *ctx,
815                         struct perf_counter *counter,
816                         int cpu)
817 {
818         struct task_struct *task = ctx->task;
819
820         if (!task) {
821                 /*
822                  * Per cpu counters are installed via an smp call and
823                  * the install is always sucessful.
824                  */
825                 smp_call_function_single(cpu, __perf_install_in_context,
826                                          counter, 1);
827                 return;
828         }
829
830 retry:
831         task_oncpu_function_call(task, __perf_install_in_context,
832                                  counter);
833
834         spin_lock_irq(&ctx->lock);
835         /*
836          * we need to retry the smp call.
837          */
838         if (ctx->is_active && list_empty(&counter->list_entry)) {
839                 spin_unlock_irq(&ctx->lock);
840                 goto retry;
841         }
842
843         /*
844          * The lock prevents that this context is scheduled in so we
845          * can add the counter safely, if it the call above did not
846          * succeed.
847          */
848         if (list_empty(&counter->list_entry))
849                 add_counter_to_ctx(counter, ctx);
850         spin_unlock_irq(&ctx->lock);
851 }
852
853 /*
854  * Cross CPU call to enable a performance counter
855  */
856 static void __perf_counter_enable(void *info)
857 {
858         struct perf_counter *counter = info;
859         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
860         struct perf_counter_context *ctx = counter->ctx;
861         struct perf_counter *leader = counter->group_leader;
862         int err;
863
864         /*
865          * If this is a per-task counter, need to check whether this
866          * counter's task is the current task on this cpu.
867          */
868         if (ctx->task && cpuctx->task_ctx != ctx) {
869                 if (cpuctx->task_ctx || ctx->task != current)
870                         return;
871                 cpuctx->task_ctx = ctx;
872         }
873
874         spin_lock(&ctx->lock);
875         ctx->is_active = 1;
876         update_context_time(ctx);
877
878         if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
879                 goto unlock;
880         counter->state = PERF_COUNTER_STATE_INACTIVE;
881         counter->tstamp_enabled = ctx->time - counter->total_time_enabled;
882
883         /*
884          * If the counter is in a group and isn't the group leader,
885          * then don't put it on unless the group is on.
886          */
887         if (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE)
888                 goto unlock;
889
890         if (!group_can_go_on(counter, cpuctx, 1)) {
891                 err = -EEXIST;
892         } else {
893                 perf_disable();
894                 if (counter == leader)
895                         err = group_sched_in(counter, cpuctx, ctx,
896                                              smp_processor_id());
897                 else
898                         err = counter_sched_in(counter, cpuctx, ctx,
899                                                smp_processor_id());
900                 perf_enable();
901         }
902
903         if (err) {
904                 /*
905                  * If this counter can't go on and it's part of a
906                  * group, then the whole group has to come off.
907                  */
908                 if (leader != counter)
909                         group_sched_out(leader, cpuctx, ctx);
910                 if (leader->attr.pinned) {
911                         update_group_times(leader);
912                         leader->state = PERF_COUNTER_STATE_ERROR;
913                 }
914         }
915
916  unlock:
917         spin_unlock(&ctx->lock);
918 }
919
920 /*
921  * Enable a counter.
922  *
923  * If counter->ctx is a cloned context, callers must make sure that
924  * every task struct that counter->ctx->task could possibly point to
925  * remains valid.  This condition is satisfied when called through
926  * perf_counter_for_each_child or perf_counter_for_each as described
927  * for perf_counter_disable.
928  */
929 static void perf_counter_enable(struct perf_counter *counter)
930 {
931         struct perf_counter_context *ctx = counter->ctx;
932         struct task_struct *task = ctx->task;
933
934         if (!task) {
935                 /*
936                  * Enable the counter on the cpu that it's on
937                  */
938                 smp_call_function_single(counter->cpu, __perf_counter_enable,
939                                          counter, 1);
940                 return;
941         }
942
943         spin_lock_irq(&ctx->lock);
944         if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
945                 goto out;
946
947         /*
948          * If the counter is in error state, clear that first.
949          * That way, if we see the counter in error state below, we
950          * know that it has gone back into error state, as distinct
951          * from the task having been scheduled away before the
952          * cross-call arrived.
953          */
954         if (counter->state == PERF_COUNTER_STATE_ERROR)
955                 counter->state = PERF_COUNTER_STATE_OFF;
956
957  retry:
958         spin_unlock_irq(&ctx->lock);
959         task_oncpu_function_call(task, __perf_counter_enable, counter);
960
961         spin_lock_irq(&ctx->lock);
962
963         /*
964          * If the context is active and the counter is still off,
965          * we need to retry the cross-call.
966          */
967         if (ctx->is_active && counter->state == PERF_COUNTER_STATE_OFF)
968                 goto retry;
969
970         /*
971          * Since we have the lock this context can't be scheduled
972          * in, so we can change the state safely.
973          */
974         if (counter->state == PERF_COUNTER_STATE_OFF) {
975                 counter->state = PERF_COUNTER_STATE_INACTIVE;
976                 counter->tstamp_enabled =
977                         ctx->time - counter->total_time_enabled;
978         }
979  out:
980         spin_unlock_irq(&ctx->lock);
981 }
982
983 static int perf_counter_refresh(struct perf_counter *counter, int refresh)
984 {
985         /*
986          * not supported on inherited counters
987          */
988         if (counter->attr.inherit)
989                 return -EINVAL;
990
991         atomic_add(refresh, &counter->event_limit);
992         perf_counter_enable(counter);
993
994         return 0;
995 }
996
997 void __perf_counter_sched_out(struct perf_counter_context *ctx,
998                               struct perf_cpu_context *cpuctx)
999 {
1000         struct perf_counter *counter;
1001
1002         spin_lock(&ctx->lock);
1003         ctx->is_active = 0;
1004         if (likely(!ctx->nr_counters))
1005                 goto out;
1006         update_context_time(ctx);
1007
1008         perf_disable();
1009         if (ctx->nr_active) {
1010                 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1011                         if (counter != counter->group_leader)
1012                                 counter_sched_out(counter, cpuctx, ctx);
1013                         else
1014                                 group_sched_out(counter, cpuctx, ctx);
1015                 }
1016         }
1017         perf_enable();
1018  out:
1019         spin_unlock(&ctx->lock);
1020 }
1021
1022 /*
1023  * Test whether two contexts are equivalent, i.e. whether they
1024  * have both been cloned from the same version of the same context
1025  * and they both have the same number of enabled counters.
1026  * If the number of enabled counters is the same, then the set
1027  * of enabled counters should be the same, because these are both
1028  * inherited contexts, therefore we can't access individual counters
1029  * in them directly with an fd; we can only enable/disable all
1030  * counters via prctl, or enable/disable all counters in a family
1031  * via ioctl, which will have the same effect on both contexts.
1032  */
1033 static int context_equiv(struct perf_counter_context *ctx1,
1034                          struct perf_counter_context *ctx2)
1035 {
1036         return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1037                 && ctx1->parent_gen == ctx2->parent_gen
1038                 && !ctx1->pin_count && !ctx2->pin_count;
1039 }
1040
1041 static void __perf_counter_read(void *counter);
1042
1043 static void __perf_counter_sync_stat(struct perf_counter *counter,
1044                                      struct perf_counter *next_counter)
1045 {
1046         u64 value;
1047
1048         if (!counter->attr.inherit_stat)
1049                 return;
1050
1051         /*
1052          * Update the counter value, we cannot use perf_counter_read()
1053          * because we're in the middle of a context switch and have IRQs
1054          * disabled, which upsets smp_call_function_single(), however
1055          * we know the counter must be on the current CPU, therefore we
1056          * don't need to use it.
1057          */
1058         switch (counter->state) {
1059         case PERF_COUNTER_STATE_ACTIVE:
1060                 __perf_counter_read(counter);
1061                 break;
1062
1063         case PERF_COUNTER_STATE_INACTIVE:
1064                 update_counter_times(counter);
1065                 break;
1066
1067         default:
1068                 break;
1069         }
1070
1071         /*
1072          * In order to keep per-task stats reliable we need to flip the counter
1073          * values when we flip the contexts.
1074          */
1075         value = atomic64_read(&next_counter->count);
1076         value = atomic64_xchg(&counter->count, value);
1077         atomic64_set(&next_counter->count, value);
1078
1079         swap(counter->total_time_enabled, next_counter->total_time_enabled);
1080         swap(counter->total_time_running, next_counter->total_time_running);
1081
1082         /*
1083          * Since we swizzled the values, update the user visible data too.
1084          */
1085         perf_counter_update_userpage(counter);
1086         perf_counter_update_userpage(next_counter);
1087 }
1088
1089 #define list_next_entry(pos, member) \
1090         list_entry(pos->member.next, typeof(*pos), member)
1091
1092 static void perf_counter_sync_stat(struct perf_counter_context *ctx,
1093                                    struct perf_counter_context *next_ctx)
1094 {
1095         struct perf_counter *counter, *next_counter;
1096
1097         if (!ctx->nr_stat)
1098                 return;
1099
1100         counter = list_first_entry(&ctx->event_list,
1101                                    struct perf_counter, event_entry);
1102
1103         next_counter = list_first_entry(&next_ctx->event_list,
1104                                         struct perf_counter, event_entry);
1105
1106         while (&counter->event_entry != &ctx->event_list &&
1107                &next_counter->event_entry != &next_ctx->event_list) {
1108
1109                 __perf_counter_sync_stat(counter, next_counter);
1110
1111                 counter = list_next_entry(counter, event_entry);
1112                 next_counter = list_next_entry(next_counter, event_entry);
1113         }
1114 }
1115
1116 /*
1117  * Called from scheduler to remove the counters of the current task,
1118  * with interrupts disabled.
1119  *
1120  * We stop each counter and update the counter value in counter->count.
1121  *
1122  * This does not protect us against NMI, but disable()
1123  * sets the disabled bit in the control field of counter _before_
1124  * accessing the counter control register. If a NMI hits, then it will
1125  * not restart the counter.
1126  */
1127 void perf_counter_task_sched_out(struct task_struct *task,
1128                                  struct task_struct *next, int cpu)
1129 {
1130         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
1131         struct perf_counter_context *ctx = task->perf_counter_ctxp;
1132         struct perf_counter_context *next_ctx;
1133         struct perf_counter_context *parent;
1134         struct pt_regs *regs;
1135         int do_switch = 1;
1136
1137         regs = task_pt_regs(task);
1138         perf_swcounter_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, regs, 0);
1139
1140         if (likely(!ctx || !cpuctx->task_ctx))
1141                 return;
1142
1143         update_context_time(ctx);
1144
1145         rcu_read_lock();
1146         parent = rcu_dereference(ctx->parent_ctx);
1147         next_ctx = next->perf_counter_ctxp;
1148         if (parent && next_ctx &&
1149             rcu_dereference(next_ctx->parent_ctx) == parent) {
1150                 /*
1151                  * Looks like the two contexts are clones, so we might be
1152                  * able to optimize the context switch.  We lock both
1153                  * contexts and check that they are clones under the
1154                  * lock (including re-checking that neither has been
1155                  * uncloned in the meantime).  It doesn't matter which
1156                  * order we take the locks because no other cpu could
1157                  * be trying to lock both of these tasks.
1158                  */
1159                 spin_lock(&ctx->lock);
1160                 spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
1161                 if (context_equiv(ctx, next_ctx)) {
1162                         /*
1163                          * XXX do we need a memory barrier of sorts
1164                          * wrt to rcu_dereference() of perf_counter_ctxp
1165                          */
1166                         task->perf_counter_ctxp = next_ctx;
1167                         next->perf_counter_ctxp = ctx;
1168                         ctx->task = next;
1169                         next_ctx->task = task;
1170                         do_switch = 0;
1171
1172                         perf_counter_sync_stat(ctx, next_ctx);
1173                 }
1174                 spin_unlock(&next_ctx->lock);
1175                 spin_unlock(&ctx->lock);
1176         }
1177         rcu_read_unlock();
1178
1179         if (do_switch) {
1180                 __perf_counter_sched_out(ctx, cpuctx);
1181                 cpuctx->task_ctx = NULL;
1182         }
1183 }
1184
1185 /*
1186  * Called with IRQs disabled
1187  */
1188 static void __perf_counter_task_sched_out(struct perf_counter_context *ctx)
1189 {
1190         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1191
1192         if (!cpuctx->task_ctx)
1193                 return;
1194
1195         if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
1196                 return;
1197
1198         __perf_counter_sched_out(ctx, cpuctx);
1199         cpuctx->task_ctx = NULL;
1200 }
1201
1202 /*
1203  * Called with IRQs disabled
1204  */
1205 static void perf_counter_cpu_sched_out(struct perf_cpu_context *cpuctx)
1206 {
1207         __perf_counter_sched_out(&cpuctx->ctx, cpuctx);
1208 }
1209
1210 static void
1211 __perf_counter_sched_in(struct perf_counter_context *ctx,
1212                         struct perf_cpu_context *cpuctx, int cpu)
1213 {
1214         struct perf_counter *counter;
1215         int can_add_hw = 1;
1216
1217         spin_lock(&ctx->lock);
1218         ctx->is_active = 1;
1219         if (likely(!ctx->nr_counters))
1220                 goto out;
1221
1222         ctx->timestamp = perf_clock();
1223
1224         perf_disable();
1225
1226         /*
1227          * First go through the list and put on any pinned groups
1228          * in order to give them the best chance of going on.
1229          */
1230         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1231                 if (counter->state <= PERF_COUNTER_STATE_OFF ||
1232                     !counter->attr.pinned)
1233                         continue;
1234                 if (counter->cpu != -1 && counter->cpu != cpu)
1235                         continue;
1236
1237                 if (counter != counter->group_leader)
1238                         counter_sched_in(counter, cpuctx, ctx, cpu);
1239                 else {
1240                         if (group_can_go_on(counter, cpuctx, 1))
1241                                 group_sched_in(counter, cpuctx, ctx, cpu);
1242                 }
1243
1244                 /*
1245                  * If this pinned group hasn't been scheduled,
1246                  * put it in error state.
1247                  */
1248                 if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
1249                         update_group_times(counter);
1250                         counter->state = PERF_COUNTER_STATE_ERROR;
1251                 }
1252         }
1253
1254         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1255                 /*
1256                  * Ignore counters in OFF or ERROR state, and
1257                  * ignore pinned counters since we did them already.
1258                  */
1259                 if (counter->state <= PERF_COUNTER_STATE_OFF ||
1260                     counter->attr.pinned)
1261                         continue;
1262
1263                 /*
1264                  * Listen to the 'cpu' scheduling filter constraint
1265                  * of counters:
1266                  */
1267                 if (counter->cpu != -1 && counter->cpu != cpu)
1268                         continue;
1269
1270                 if (counter != counter->group_leader) {
1271                         if (counter_sched_in(counter, cpuctx, ctx, cpu))
1272                                 can_add_hw = 0;
1273                 } else {
1274                         if (group_can_go_on(counter, cpuctx, can_add_hw)) {
1275                                 if (group_sched_in(counter, cpuctx, ctx, cpu))
1276                                         can_add_hw = 0;
1277                         }
1278                 }
1279         }
1280         perf_enable();
1281  out:
1282         spin_unlock(&ctx->lock);
1283 }
1284
1285 /*
1286  * Called from scheduler to add the counters of the current task
1287  * with interrupts disabled.
1288  *
1289  * We restore the counter value and then enable it.
1290  *
1291  * This does not protect us against NMI, but enable()
1292  * sets the enabled bit in the control field of counter _before_
1293  * accessing the counter control register. If a NMI hits, then it will
1294  * keep the counter running.
1295  */
1296 void perf_counter_task_sched_in(struct task_struct *task, int cpu)
1297 {
1298         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
1299         struct perf_counter_context *ctx = task->perf_counter_ctxp;
1300
1301         if (likely(!ctx))
1302                 return;
1303         if (cpuctx->task_ctx == ctx)
1304                 return;
1305         __perf_counter_sched_in(ctx, cpuctx, cpu);
1306         cpuctx->task_ctx = ctx;
1307 }
1308
1309 static void perf_counter_cpu_sched_in(struct perf_cpu_context *cpuctx, int cpu)
1310 {
1311         struct perf_counter_context *ctx = &cpuctx->ctx;
1312
1313         __perf_counter_sched_in(ctx, cpuctx, cpu);
1314 }
1315
1316 #define MAX_INTERRUPTS (~0ULL)
1317
1318 static void perf_log_throttle(struct perf_counter *counter, int enable);
1319
1320 static void perf_adjust_period(struct perf_counter *counter, u64 events)
1321 {
1322         struct hw_perf_counter *hwc = &counter->hw;
1323         u64 period, sample_period;
1324         s64 delta;
1325
1326         events *= hwc->sample_period;
1327         period = div64_u64(events, counter->attr.sample_freq);
1328
1329         delta = (s64)(period - hwc->sample_period);
1330         delta = (delta + 7) / 8; /* low pass filter */
1331
1332         sample_period = hwc->sample_period + delta;
1333
1334         if (!sample_period)
1335                 sample_period = 1;
1336
1337         hwc->sample_period = sample_period;
1338 }
1339
1340 static void perf_ctx_adjust_freq(struct perf_counter_context *ctx)
1341 {
1342         struct perf_counter *counter;
1343         struct hw_perf_counter *hwc;
1344         u64 interrupts, freq;
1345
1346         spin_lock(&ctx->lock);
1347         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1348                 if (counter->state != PERF_COUNTER_STATE_ACTIVE)
1349                         continue;
1350
1351                 hwc = &counter->hw;
1352
1353                 interrupts = hwc->interrupts;
1354                 hwc->interrupts = 0;
1355
1356                 /*
1357                  * unthrottle counters on the tick
1358                  */
1359                 if (interrupts == MAX_INTERRUPTS) {
1360                         perf_log_throttle(counter, 1);
1361                         counter->pmu->unthrottle(counter);
1362                         interrupts = 2*sysctl_perf_counter_sample_rate/HZ;
1363                 }
1364
1365                 if (!counter->attr.freq || !counter->attr.sample_freq)
1366                         continue;
1367
1368                 /*
1369                  * if the specified freq < HZ then we need to skip ticks
1370                  */
1371                 if (counter->attr.sample_freq < HZ) {
1372                         freq = counter->attr.sample_freq;
1373
1374                         hwc->freq_count += freq;
1375                         hwc->freq_interrupts += interrupts;
1376
1377                         if (hwc->freq_count < HZ)
1378                                 continue;
1379
1380                         interrupts = hwc->freq_interrupts;
1381                         hwc->freq_interrupts = 0;
1382                         hwc->freq_count -= HZ;
1383                 } else
1384                         freq = HZ;
1385
1386                 perf_adjust_period(counter, freq * interrupts);
1387
1388                 /*
1389                  * In order to avoid being stalled by an (accidental) huge
1390                  * sample period, force reset the sample period if we didn't
1391                  * get any events in this freq period.
1392                  */
1393                 if (!interrupts) {
1394                         perf_disable();
1395                         counter->pmu->disable(counter);
1396                         atomic64_set(&hwc->period_left, 0);
1397                         counter->pmu->enable(counter);
1398                         perf_enable();
1399                 }
1400         }
1401         spin_unlock(&ctx->lock);
1402 }
1403
1404 /*
1405  * Round-robin a context's counters:
1406  */
1407 static void rotate_ctx(struct perf_counter_context *ctx)
1408 {
1409         struct perf_counter *counter;
1410
1411         if (!ctx->nr_counters)
1412                 return;
1413
1414         spin_lock(&ctx->lock);
1415         /*
1416          * Rotate the first entry last (works just fine for group counters too):
1417          */
1418         perf_disable();
1419         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1420                 list_move_tail(&counter->list_entry, &ctx->counter_list);
1421                 break;
1422         }
1423         perf_enable();
1424
1425         spin_unlock(&ctx->lock);
1426 }
1427
1428 void perf_counter_task_tick(struct task_struct *curr, int cpu)
1429 {
1430         struct perf_cpu_context *cpuctx;
1431         struct perf_counter_context *ctx;
1432
1433         if (!atomic_read(&nr_counters))
1434                 return;
1435
1436         cpuctx = &per_cpu(perf_cpu_context, cpu);
1437         ctx = curr->perf_counter_ctxp;
1438
1439         perf_ctx_adjust_freq(&cpuctx->ctx);
1440         if (ctx)
1441                 perf_ctx_adjust_freq(ctx);
1442
1443         perf_counter_cpu_sched_out(cpuctx);
1444         if (ctx)
1445                 __perf_counter_task_sched_out(ctx);
1446
1447         rotate_ctx(&cpuctx->ctx);
1448         if (ctx)
1449                 rotate_ctx(ctx);
1450
1451         perf_counter_cpu_sched_in(cpuctx, cpu);
1452         if (ctx)
1453                 perf_counter_task_sched_in(curr, cpu);
1454 }
1455
1456 /*
1457  * Enable all of a task's counters that have been marked enable-on-exec.
1458  * This expects task == current.
1459  */
1460 static void perf_counter_enable_on_exec(struct task_struct *task)
1461 {
1462         struct perf_counter_context *ctx;
1463         struct perf_counter *counter;
1464         unsigned long flags;
1465         int enabled = 0;
1466
1467         local_irq_save(flags);
1468         ctx = task->perf_counter_ctxp;
1469         if (!ctx || !ctx->nr_counters)
1470                 goto out;
1471
1472         __perf_counter_task_sched_out(ctx);
1473
1474         spin_lock(&ctx->lock);
1475
1476         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1477                 if (!counter->attr.enable_on_exec)
1478                         continue;
1479                 counter->attr.enable_on_exec = 0;
1480                 if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
1481                         continue;
1482                 counter->state = PERF_COUNTER_STATE_INACTIVE;
1483                 counter->tstamp_enabled =
1484                         ctx->time - counter->total_time_enabled;
1485                 enabled = 1;
1486         }
1487
1488         /*
1489          * Unclone this context if we enabled any counter.
1490          */
1491         if (enabled)
1492                 unclone_ctx(ctx);
1493
1494         spin_unlock(&ctx->lock);
1495
1496         perf_counter_task_sched_in(task, smp_processor_id());
1497  out:
1498         local_irq_restore(flags);
1499 }
1500
1501 /*
1502  * Cross CPU call to read the hardware counter
1503  */
1504 static void __perf_counter_read(void *info)
1505 {
1506         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1507         struct perf_counter *counter = info;
1508         struct perf_counter_context *ctx = counter->ctx;
1509         unsigned long flags;
1510
1511         /*
1512          * If this is a task context, we need to check whether it is
1513          * the current task context of this cpu.  If not it has been
1514          * scheduled out before the smp call arrived.  In that case
1515          * counter->count would have been updated to a recent sample
1516          * when the counter was scheduled out.
1517          */
1518         if (ctx->task && cpuctx->task_ctx != ctx)
1519                 return;
1520
1521         local_irq_save(flags);
1522         if (ctx->is_active)
1523                 update_context_time(ctx);
1524         counter->pmu->read(counter);
1525         update_counter_times(counter);
1526         local_irq_restore(flags);
1527 }
1528
1529 static u64 perf_counter_read(struct perf_counter *counter)
1530 {
1531         /*
1532          * If counter is enabled and currently active on a CPU, update the
1533          * value in the counter structure:
1534          */
1535         if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
1536                 smp_call_function_single(counter->oncpu,
1537                                          __perf_counter_read, counter, 1);
1538         } else if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
1539                 update_counter_times(counter);
1540         }
1541
1542         return atomic64_read(&counter->count);
1543 }
1544
1545 /*
1546  * Initialize the perf_counter context in a task_struct:
1547  */
1548 static void
1549 __perf_counter_init_context(struct perf_counter_context *ctx,
1550                             struct task_struct *task)
1551 {
1552         memset(ctx, 0, sizeof(*ctx));
1553         spin_lock_init(&ctx->lock);
1554         mutex_init(&ctx->mutex);
1555         INIT_LIST_HEAD(&ctx->counter_list);
1556         INIT_LIST_HEAD(&ctx->event_list);
1557         atomic_set(&ctx->refcount, 1);
1558         ctx->task = task;
1559 }
1560
1561 static struct perf_counter_context *find_get_context(pid_t pid, int cpu)
1562 {
1563         struct perf_counter_context *ctx;
1564         struct perf_cpu_context *cpuctx;
1565         struct task_struct *task;
1566         unsigned long flags;
1567         int err;
1568
1569         /*
1570          * If cpu is not a wildcard then this is a percpu counter:
1571          */
1572         if (cpu != -1) {
1573                 /* Must be root to operate on a CPU counter: */
1574                 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
1575                         return ERR_PTR(-EACCES);
1576
1577                 if (cpu < 0 || cpu > num_possible_cpus())
1578                         return ERR_PTR(-EINVAL);
1579
1580                 /*
1581                  * We could be clever and allow to attach a counter to an
1582                  * offline CPU and activate it when the CPU comes up, but
1583                  * that's for later.
1584                  */
1585                 if (!cpu_isset(cpu, cpu_online_map))
1586                         return ERR_PTR(-ENODEV);
1587
1588                 cpuctx = &per_cpu(perf_cpu_context, cpu);
1589                 ctx = &cpuctx->ctx;
1590                 get_ctx(ctx);
1591
1592                 return ctx;
1593         }
1594
1595         rcu_read_lock();
1596         if (!pid)
1597                 task = current;
1598         else
1599                 task = find_task_by_vpid(pid);
1600         if (task)
1601                 get_task_struct(task);
1602         rcu_read_unlock();
1603
1604         if (!task)
1605                 return ERR_PTR(-ESRCH);
1606
1607         /*
1608          * Can't attach counters to a dying task.
1609          */
1610         err = -ESRCH;
1611         if (task->flags & PF_EXITING)
1612                 goto errout;
1613
1614         /* Reuse ptrace permission checks for now. */
1615         err = -EACCES;
1616         if (!ptrace_may_access(task, PTRACE_MODE_READ))
1617                 goto errout;
1618
1619  retry:
1620         ctx = perf_lock_task_context(task, &flags);
1621         if (ctx) {
1622                 unclone_ctx(ctx);
1623                 spin_unlock_irqrestore(&ctx->lock, flags);
1624         }
1625
1626         if (!ctx) {
1627                 ctx = kmalloc(sizeof(struct perf_counter_context), GFP_KERNEL);
1628                 err = -ENOMEM;
1629                 if (!ctx)
1630                         goto errout;
1631                 __perf_counter_init_context(ctx, task);
1632                 get_ctx(ctx);
1633                 if (cmpxchg(&task->perf_counter_ctxp, NULL, ctx)) {
1634                         /*
1635                          * We raced with some other task; use
1636                          * the context they set.
1637                          */
1638                         kfree(ctx);
1639                         goto retry;
1640                 }
1641                 get_task_struct(task);
1642         }
1643
1644         put_task_struct(task);
1645         return ctx;
1646
1647  errout:
1648         put_task_struct(task);
1649         return ERR_PTR(err);
1650 }
1651
1652 static void free_counter_rcu(struct rcu_head *head)
1653 {
1654         struct perf_counter *counter;
1655
1656         counter = container_of(head, struct perf_counter, rcu_head);
1657         if (counter->ns)
1658                 put_pid_ns(counter->ns);
1659         kfree(counter);
1660 }
1661
1662 static void perf_pending_sync(struct perf_counter *counter);
1663
1664 static void free_counter(struct perf_counter *counter)
1665 {
1666         perf_pending_sync(counter);
1667
1668         if (!counter->parent) {
1669                 atomic_dec(&nr_counters);
1670                 if (counter->attr.mmap)
1671                         atomic_dec(&nr_mmap_counters);
1672                 if (counter->attr.comm)
1673                         atomic_dec(&nr_comm_counters);
1674                 if (counter->attr.task)
1675                         atomic_dec(&nr_task_counters);
1676         }
1677
1678         if (counter->destroy)
1679                 counter->destroy(counter);
1680
1681         put_ctx(counter->ctx);
1682         call_rcu(&counter->rcu_head, free_counter_rcu);
1683 }
1684
1685 /*
1686  * Called when the last reference to the file is gone.
1687  */
1688 static int perf_release(struct inode *inode, struct file *file)
1689 {
1690         struct perf_counter *counter = file->private_data;
1691         struct perf_counter_context *ctx = counter->ctx;
1692
1693         file->private_data = NULL;
1694
1695         WARN_ON_ONCE(ctx->parent_ctx);
1696         mutex_lock(&ctx->mutex);
1697         perf_counter_remove_from_context(counter);
1698         mutex_unlock(&ctx->mutex);
1699
1700         mutex_lock(&counter->owner->perf_counter_mutex);
1701         list_del_init(&counter->owner_entry);
1702         mutex_unlock(&counter->owner->perf_counter_mutex);
1703         put_task_struct(counter->owner);
1704
1705         free_counter(counter);
1706
1707         return 0;
1708 }
1709
1710 static int perf_counter_read_size(struct perf_counter *counter)
1711 {
1712         int entry = sizeof(u64); /* value */
1713         int size = 0;
1714         int nr = 1;
1715
1716         if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1717                 size += sizeof(u64);
1718
1719         if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1720                 size += sizeof(u64);
1721
1722         if (counter->attr.read_format & PERF_FORMAT_ID)
1723                 entry += sizeof(u64);
1724
1725         if (counter->attr.read_format & PERF_FORMAT_GROUP) {
1726                 nr += counter->group_leader->nr_siblings;
1727                 size += sizeof(u64);
1728         }
1729
1730         size += entry * nr;
1731
1732         return size;
1733 }
1734
1735 static u64 perf_counter_read_value(struct perf_counter *counter)
1736 {
1737         struct perf_counter *child;
1738         u64 total = 0;
1739
1740         total += perf_counter_read(counter);
1741         list_for_each_entry(child, &counter->child_list, child_list)
1742                 total += perf_counter_read(child);
1743
1744         return total;
1745 }
1746
1747 static int perf_counter_read_entry(struct perf_counter *counter,
1748                                    u64 read_format, char __user *buf)
1749 {
1750         int n = 0, count = 0;
1751         u64 values[2];
1752
1753         values[n++] = perf_counter_read_value(counter);
1754         if (read_format & PERF_FORMAT_ID)
1755                 values[n++] = primary_counter_id(counter);
1756
1757         count = n * sizeof(u64);
1758
1759         if (copy_to_user(buf, values, count))
1760                 return -EFAULT;
1761
1762         return count;
1763 }
1764
1765 static int perf_counter_read_group(struct perf_counter *counter,
1766                                    u64 read_format, char __user *buf)
1767 {
1768         struct perf_counter *leader = counter->group_leader, *sub;
1769         int n = 0, size = 0, err = -EFAULT;
1770         u64 values[3];
1771
1772         values[n++] = 1 + leader->nr_siblings;
1773         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
1774                 values[n++] = leader->total_time_enabled +
1775                         atomic64_read(&leader->child_total_time_enabled);
1776         }
1777         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
1778                 values[n++] = leader->total_time_running +
1779                         atomic64_read(&leader->child_total_time_running);
1780         }
1781
1782         size = n * sizeof(u64);
1783
1784         if (copy_to_user(buf, values, size))
1785                 return -EFAULT;
1786
1787         err = perf_counter_read_entry(leader, read_format, buf + size);
1788         if (err < 0)
1789                 return err;
1790
1791         size += err;
1792
1793         list_for_each_entry(sub, &leader->sibling_list, list_entry) {
1794                 err = perf_counter_read_entry(sub, read_format,
1795                                 buf + size);
1796                 if (err < 0)
1797                         return err;
1798
1799                 size += err;
1800         }
1801
1802         return size;
1803 }
1804
1805 static int perf_counter_read_one(struct perf_counter *counter,
1806                                  u64 read_format, char __user *buf)
1807 {
1808         u64 values[4];
1809         int n = 0;
1810
1811         values[n++] = perf_counter_read_value(counter);
1812         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
1813                 values[n++] = counter->total_time_enabled +
1814                         atomic64_read(&counter->child_total_time_enabled);
1815         }
1816         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
1817                 values[n++] = counter->total_time_running +
1818                         atomic64_read(&counter->child_total_time_running);
1819         }
1820         if (read_format & PERF_FORMAT_ID)
1821                 values[n++] = primary_counter_id(counter);
1822
1823         if (copy_to_user(buf, values, n * sizeof(u64)))
1824                 return -EFAULT;
1825
1826         return n * sizeof(u64);
1827 }
1828
1829 /*
1830  * Read the performance counter - simple non blocking version for now
1831  */
1832 static ssize_t
1833 perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count)
1834 {
1835         u64 read_format = counter->attr.read_format;
1836         int ret;
1837
1838         /*
1839          * Return end-of-file for a read on a counter that is in
1840          * error state (i.e. because it was pinned but it couldn't be
1841          * scheduled on to the CPU at some point).
1842          */
1843         if (counter->state == PERF_COUNTER_STATE_ERROR)
1844                 return 0;
1845
1846         if (count < perf_counter_read_size(counter))
1847                 return -ENOSPC;
1848
1849         WARN_ON_ONCE(counter->ctx->parent_ctx);
1850         mutex_lock(&counter->child_mutex);
1851         if (read_format & PERF_FORMAT_GROUP)
1852                 ret = perf_counter_read_group(counter, read_format, buf);
1853         else
1854                 ret = perf_counter_read_one(counter, read_format, buf);
1855         mutex_unlock(&counter->child_mutex);
1856
1857         return ret;
1858 }
1859
1860 static ssize_t
1861 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
1862 {
1863         struct perf_counter *counter = file->private_data;
1864
1865         return perf_read_hw(counter, buf, count);
1866 }
1867
1868 static unsigned int perf_poll(struct file *file, poll_table *wait)
1869 {
1870         struct perf_counter *counter = file->private_data;
1871         struct perf_mmap_data *data;
1872         unsigned int events = POLL_HUP;
1873
1874         rcu_read_lock();
1875         data = rcu_dereference(counter->data);
1876         if (data)
1877                 events = atomic_xchg(&data->poll, 0);
1878         rcu_read_unlock();
1879
1880         poll_wait(file, &counter->waitq, wait);
1881
1882         return events;
1883 }
1884
1885 static void perf_counter_reset(struct perf_counter *counter)
1886 {
1887         (void)perf_counter_read(counter);
1888         atomic64_set(&counter->count, 0);
1889         perf_counter_update_userpage(counter);
1890 }
1891
1892 /*
1893  * Holding the top-level counter's child_mutex means that any
1894  * descendant process that has inherited this counter will block
1895  * in sync_child_counter if it goes to exit, thus satisfying the
1896  * task existence requirements of perf_counter_enable/disable.
1897  */
1898 static void perf_counter_for_each_child(struct perf_counter *counter,
1899                                         void (*func)(struct perf_counter *))
1900 {
1901         struct perf_counter *child;
1902
1903         WARN_ON_ONCE(counter->ctx->parent_ctx);
1904         mutex_lock(&counter->child_mutex);
1905         func(counter);
1906         list_for_each_entry(child, &counter->child_list, child_list)
1907                 func(child);
1908         mutex_unlock(&counter->child_mutex);
1909 }
1910
1911 static void perf_counter_for_each(struct perf_counter *counter,
1912                                   void (*func)(struct perf_counter *))
1913 {
1914         struct perf_counter_context *ctx = counter->ctx;
1915         struct perf_counter *sibling;
1916
1917         WARN_ON_ONCE(ctx->parent_ctx);
1918         mutex_lock(&ctx->mutex);
1919         counter = counter->group_leader;
1920
1921         perf_counter_for_each_child(counter, func);
1922         func(counter);
1923         list_for_each_entry(sibling, &counter->sibling_list, list_entry)
1924                 perf_counter_for_each_child(counter, func);
1925         mutex_unlock(&ctx->mutex);
1926 }
1927
1928 static int perf_counter_period(struct perf_counter *counter, u64 __user *arg)
1929 {
1930         struct perf_counter_context *ctx = counter->ctx;
1931         unsigned long size;
1932         int ret = 0;
1933         u64 value;
1934
1935         if (!counter->attr.sample_period)
1936                 return -EINVAL;
1937
1938         size = copy_from_user(&value, arg, sizeof(value));
1939         if (size != sizeof(value))
1940                 return -EFAULT;
1941
1942         if (!value)
1943                 return -EINVAL;
1944
1945         spin_lock_irq(&ctx->lock);
1946         if (counter->attr.freq) {
1947                 if (value > sysctl_perf_counter_sample_rate) {
1948                         ret = -EINVAL;
1949                         goto unlock;
1950                 }
1951
1952                 counter->attr.sample_freq = value;
1953         } else {
1954                 counter->attr.sample_period = value;
1955                 counter->hw.sample_period = value;
1956         }
1957 unlock:
1958         spin_unlock_irq(&ctx->lock);
1959
1960         return ret;
1961 }
1962
1963 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1964 {
1965         struct perf_counter *counter = file->private_data;
1966         void (*func)(struct perf_counter *);
1967         u32 flags = arg;
1968
1969         switch (cmd) {
1970         case PERF_COUNTER_IOC_ENABLE:
1971                 func = perf_counter_enable;
1972                 break;
1973         case PERF_COUNTER_IOC_DISABLE:
1974                 func = perf_counter_disable;
1975                 break;
1976         case PERF_COUNTER_IOC_RESET:
1977                 func = perf_counter_reset;
1978                 break;
1979
1980         case PERF_COUNTER_IOC_REFRESH:
1981                 return perf_counter_refresh(counter, arg);
1982
1983         case PERF_COUNTER_IOC_PERIOD:
1984                 return perf_counter_period(counter, (u64 __user *)arg);
1985
1986         default:
1987                 return -ENOTTY;
1988         }
1989
1990         if (flags & PERF_IOC_FLAG_GROUP)
1991                 perf_counter_for_each(counter, func);
1992         else
1993                 perf_counter_for_each_child(counter, func);
1994
1995         return 0;
1996 }
1997
1998 int perf_counter_task_enable(void)
1999 {
2000         struct perf_counter *counter;
2001
2002         mutex_lock(&current->perf_counter_mutex);
2003         list_for_each_entry(counter, &current->perf_counter_list, owner_entry)
2004                 perf_counter_for_each_child(counter, perf_counter_enable);
2005         mutex_unlock(&current->perf_counter_mutex);
2006
2007         return 0;
2008 }
2009
2010 int perf_counter_task_disable(void)
2011 {
2012         struct perf_counter *counter;
2013
2014         mutex_lock(&current->perf_counter_mutex);
2015         list_for_each_entry(counter, &current->perf_counter_list, owner_entry)
2016                 perf_counter_for_each_child(counter, perf_counter_disable);
2017         mutex_unlock(&current->perf_counter_mutex);
2018
2019         return 0;
2020 }
2021
2022 #ifndef PERF_COUNTER_INDEX_OFFSET
2023 # define PERF_COUNTER_INDEX_OFFSET 0
2024 #endif
2025
2026 static int perf_counter_index(struct perf_counter *counter)
2027 {
2028         if (counter->state != PERF_COUNTER_STATE_ACTIVE)
2029                 return 0;
2030
2031         return counter->hw.idx + 1 - PERF_COUNTER_INDEX_OFFSET;
2032 }
2033
2034 /*
2035  * Callers need to ensure there can be no nesting of this function, otherwise
2036  * the seqlock logic goes bad. We can not serialize this because the arch
2037  * code calls this from NMI context.
2038  */
2039 void perf_counter_update_userpage(struct perf_counter *counter)
2040 {
2041         struct perf_counter_mmap_page *userpg;
2042         struct perf_mmap_data *data;
2043
2044         rcu_read_lock();
2045         data = rcu_dereference(counter->data);
2046         if (!data)
2047                 goto unlock;
2048
2049         userpg = data->user_page;
2050
2051         /*
2052          * Disable preemption so as to not let the corresponding user-space
2053          * spin too long if we get preempted.
2054          */
2055         preempt_disable();
2056         ++userpg->lock;
2057         barrier();
2058         userpg->index = perf_counter_index(counter);
2059         userpg->offset = atomic64_read(&counter->count);
2060         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
2061                 userpg->offset -= atomic64_read(&counter->hw.prev_count);
2062
2063         userpg->time_enabled = counter->total_time_enabled +
2064                         atomic64_read(&counter->child_total_time_enabled);
2065
2066         userpg->time_running = counter->total_time_running +
2067                         atomic64_read(&counter->child_total_time_running);
2068
2069         barrier();
2070         ++userpg->lock;
2071         preempt_enable();
2072 unlock:
2073         rcu_read_unlock();
2074 }
2075
2076 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2077 {
2078         struct perf_counter *counter = vma->vm_file->private_data;
2079         struct perf_mmap_data *data;
2080         int ret = VM_FAULT_SIGBUS;
2081
2082         if (vmf->flags & FAULT_FLAG_MKWRITE) {
2083                 if (vmf->pgoff == 0)
2084                         ret = 0;
2085                 return ret;
2086         }
2087
2088         rcu_read_lock();
2089         data = rcu_dereference(counter->data);
2090         if (!data)
2091                 goto unlock;
2092
2093         if (vmf->pgoff == 0) {
2094                 vmf->page = virt_to_page(data->user_page);
2095         } else {
2096                 int nr = vmf->pgoff - 1;
2097
2098                 if ((unsigned)nr > data->nr_pages)
2099                         goto unlock;
2100
2101                 if (vmf->flags & FAULT_FLAG_WRITE)
2102                         goto unlock;
2103
2104                 vmf->page = virt_to_page(data->data_pages[nr]);
2105         }
2106
2107         get_page(vmf->page);
2108         vmf->page->mapping = vma->vm_file->f_mapping;
2109         vmf->page->index   = vmf->pgoff;
2110
2111         ret = 0;
2112 unlock:
2113         rcu_read_unlock();
2114
2115         return ret;
2116 }
2117
2118 static int perf_mmap_data_alloc(struct perf_counter *counter, int nr_pages)
2119 {
2120         struct perf_mmap_data *data;
2121         unsigned long size;
2122         int i;
2123
2124         WARN_ON(atomic_read(&counter->mmap_count));
2125
2126         size = sizeof(struct perf_mmap_data);
2127         size += nr_pages * sizeof(void *);
2128
2129         data = kzalloc(size, GFP_KERNEL);
2130         if (!data)
2131                 goto fail;
2132
2133         data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
2134         if (!data->user_page)
2135                 goto fail_user_page;
2136
2137         for (i = 0; i < nr_pages; i++) {
2138                 data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
2139                 if (!data->data_pages[i])
2140                         goto fail_data_pages;
2141         }
2142
2143         data->nr_pages = nr_pages;
2144         atomic_set(&data->lock, -1);
2145
2146         rcu_assign_pointer(counter->data, data);
2147
2148         return 0;
2149
2150 fail_data_pages:
2151         for (i--; i >= 0; i--)
2152                 free_page((unsigned long)data->data_pages[i]);
2153
2154         free_page((unsigned long)data->user_page);
2155
2156 fail_user_page:
2157         kfree(data);
2158
2159 fail:
2160         return -ENOMEM;
2161 }
2162
2163 static void perf_mmap_free_page(unsigned long addr)
2164 {
2165         struct page *page = virt_to_page((void *)addr);
2166
2167         page->mapping = NULL;
2168         __free_page(page);
2169 }
2170
2171 static void __perf_mmap_data_free(struct rcu_head *rcu_head)
2172 {
2173         struct perf_mmap_data *data;
2174         int i;
2175
2176         data = container_of(rcu_head, struct perf_mmap_data, rcu_head);
2177
2178         perf_mmap_free_page((unsigned long)data->user_page);
2179         for (i = 0; i < data->nr_pages; i++)
2180                 perf_mmap_free_page((unsigned long)data->data_pages[i]);
2181
2182         kfree(data);
2183 }
2184
2185 static void perf_mmap_data_free(struct perf_counter *counter)
2186 {
2187         struct perf_mmap_data *data = counter->data;
2188
2189         WARN_ON(atomic_read(&counter->mmap_count));
2190
2191         rcu_assign_pointer(counter->data, NULL);
2192         call_rcu(&data->rcu_head, __perf_mmap_data_free);
2193 }
2194
2195 static void perf_mmap_open(struct vm_area_struct *vma)
2196 {
2197         struct perf_counter *counter = vma->vm_file->private_data;
2198
2199         atomic_inc(&counter->mmap_count);
2200 }
2201
2202 static void perf_mmap_close(struct vm_area_struct *vma)
2203 {
2204         struct perf_counter *counter = vma->vm_file->private_data;
2205
2206         WARN_ON_ONCE(counter->ctx->parent_ctx);
2207         if (atomic_dec_and_mutex_lock(&counter->mmap_count, &counter->mmap_mutex)) {
2208                 struct user_struct *user = current_user();
2209
2210                 atomic_long_sub(counter->data->nr_pages + 1, &user->locked_vm);
2211                 vma->vm_mm->locked_vm -= counter->data->nr_locked;
2212                 perf_mmap_data_free(counter);
2213                 mutex_unlock(&counter->mmap_mutex);
2214         }
2215 }
2216
2217 static struct vm_operations_struct perf_mmap_vmops = {
2218         .open           = perf_mmap_open,
2219         .close          = perf_mmap_close,
2220         .fault          = perf_mmap_fault,
2221         .page_mkwrite   = perf_mmap_fault,
2222 };
2223
2224 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
2225 {
2226         struct perf_counter *counter = file->private_data;
2227         unsigned long user_locked, user_lock_limit;
2228         struct user_struct *user = current_user();
2229         unsigned long locked, lock_limit;
2230         unsigned long vma_size;
2231         unsigned long nr_pages;
2232         long user_extra, extra;
2233         int ret = 0;
2234
2235         if (!(vma->vm_flags & VM_SHARED))
2236                 return -EINVAL;
2237
2238         vma_size = vma->vm_end - vma->vm_start;
2239         nr_pages = (vma_size / PAGE_SIZE) - 1;
2240
2241         /*
2242          * If we have data pages ensure they're a power-of-two number, so we
2243          * can do bitmasks instead of modulo.
2244          */
2245         if (nr_pages != 0 && !is_power_of_2(nr_pages))
2246                 return -EINVAL;
2247
2248         if (vma_size != PAGE_SIZE * (1 + nr_pages))
2249                 return -EINVAL;
2250
2251         if (vma->vm_pgoff != 0)
2252                 return -EINVAL;
2253
2254         WARN_ON_ONCE(counter->ctx->parent_ctx);
2255         mutex_lock(&counter->mmap_mutex);
2256         if (atomic_inc_not_zero(&counter->mmap_count)) {
2257                 if (nr_pages != counter->data->nr_pages)
2258                         ret = -EINVAL;
2259                 goto unlock;
2260         }
2261
2262         user_extra = nr_pages + 1;
2263         user_lock_limit = sysctl_perf_counter_mlock >> (PAGE_SHIFT - 10);
2264
2265         /*
2266          * Increase the limit linearly with more CPUs:
2267          */
2268         user_lock_limit *= num_online_cpus();
2269
2270         user_locked = atomic_long_read(&user->locked_vm) + user_extra;
2271
2272         extra = 0;
2273         if (user_locked > user_lock_limit)
2274                 extra = user_locked - user_lock_limit;
2275
2276         lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
2277         lock_limit >>= PAGE_SHIFT;
2278         locked = vma->vm_mm->locked_vm + extra;
2279
2280         if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) {
2281                 ret = -EPERM;
2282                 goto unlock;
2283         }
2284
2285         WARN_ON(counter->data);
2286         ret = perf_mmap_data_alloc(counter, nr_pages);
2287         if (ret)
2288                 goto unlock;
2289
2290         atomic_set(&counter->mmap_count, 1);
2291         atomic_long_add(user_extra, &user->locked_vm);
2292         vma->vm_mm->locked_vm += extra;
2293         counter->data->nr_locked = extra;
2294         if (vma->vm_flags & VM_WRITE)
2295                 counter->data->writable = 1;
2296
2297 unlock:
2298         mutex_unlock(&counter->mmap_mutex);
2299
2300         vma->vm_flags |= VM_RESERVED;
2301         vma->vm_ops = &perf_mmap_vmops;
2302
2303         return ret;
2304 }
2305
2306 static int perf_fasync(int fd, struct file *filp, int on)
2307 {
2308         struct inode *inode = filp->f_path.dentry->d_inode;
2309         struct perf_counter *counter = filp->private_data;
2310         int retval;
2311
2312         mutex_lock(&inode->i_mutex);
2313         retval = fasync_helper(fd, filp, on, &counter->fasync);
2314         mutex_unlock(&inode->i_mutex);
2315
2316         if (retval < 0)
2317                 return retval;
2318
2319         return 0;
2320 }
2321
2322 static const struct file_operations perf_fops = {
2323         .release                = perf_release,
2324         .read                   = perf_read,
2325         .poll                   = perf_poll,
2326         .unlocked_ioctl         = perf_ioctl,
2327         .compat_ioctl           = perf_ioctl,
2328         .mmap                   = perf_mmap,
2329         .fasync                 = perf_fasync,
2330 };
2331
2332 /*
2333  * Perf counter wakeup
2334  *
2335  * If there's data, ensure we set the poll() state and publish everything
2336  * to user-space before waking everybody up.
2337  */
2338
2339 void perf_counter_wakeup(struct perf_counter *counter)
2340 {
2341         wake_up_all(&counter->waitq);
2342
2343         if (counter->pending_kill) {
2344                 kill_fasync(&counter->fasync, SIGIO, counter->pending_kill);
2345                 counter->pending_kill = 0;
2346         }
2347 }
2348
2349 /*
2350  * Pending wakeups
2351  *
2352  * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
2353  *
2354  * The NMI bit means we cannot possibly take locks. Therefore, maintain a
2355  * single linked list and use cmpxchg() to add entries lockless.
2356  */
2357
2358 static void perf_pending_counter(struct perf_pending_entry *entry)
2359 {
2360         struct perf_counter *counter = container_of(entry,
2361                         struct perf_counter, pending);
2362
2363         if (counter->pending_disable) {
2364                 counter->pending_disable = 0;
2365                 __perf_counter_disable(counter);
2366         }
2367
2368         if (counter->pending_wakeup) {
2369                 counter->pending_wakeup = 0;
2370                 perf_counter_wakeup(counter);
2371         }
2372 }
2373
2374 #define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
2375
2376 static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
2377         PENDING_TAIL,
2378 };
2379
2380 static void perf_pending_queue(struct perf_pending_entry *entry,
2381                                void (*func)(struct perf_pending_entry *))
2382 {
2383         struct perf_pending_entry **head;
2384
2385         if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
2386                 return;
2387
2388         entry->func = func;
2389
2390         head = &get_cpu_var(perf_pending_head);
2391
2392         do {
2393                 entry->next = *head;
2394         } while (cmpxchg(head, entry->next, entry) != entry->next);
2395
2396         set_perf_counter_pending();
2397
2398         put_cpu_var(perf_pending_head);
2399 }
2400
2401 static int __perf_pending_run(void)
2402 {
2403         struct perf_pending_entry *list;
2404         int nr = 0;
2405
2406         list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
2407         while (list != PENDING_TAIL) {
2408                 void (*func)(struct perf_pending_entry *);
2409                 struct perf_pending_entry *entry = list;
2410
2411                 list = list->next;
2412
2413                 func = entry->func;
2414                 entry->next = NULL;
2415                 /*
2416                  * Ensure we observe the unqueue before we issue the wakeup,
2417                  * so that we won't be waiting forever.
2418                  * -- see perf_not_pending().
2419                  */
2420                 smp_wmb();
2421
2422                 func(entry);
2423                 nr++;
2424         }
2425
2426         return nr;
2427 }
2428
2429 static inline int perf_not_pending(struct perf_counter *counter)
2430 {
2431         /*
2432          * If we flush on whatever cpu we run, there is a chance we don't
2433          * need to wait.
2434          */
2435         get_cpu();
2436         __perf_pending_run();
2437         put_cpu();
2438
2439         /*
2440          * Ensure we see the proper queue state before going to sleep
2441          * so that we do not miss the wakeup. -- see perf_pending_handle()
2442          */
2443         smp_rmb();
2444         return counter->pending.next == NULL;
2445 }
2446
2447 static void perf_pending_sync(struct perf_counter *counter)
2448 {
2449         wait_event(counter->waitq, perf_not_pending(counter));
2450 }
2451
2452 void perf_counter_do_pending(void)
2453 {
2454         __perf_pending_run();
2455 }
2456
2457 /*
2458  * Callchain support -- arch specific
2459  */
2460
2461 __weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2462 {
2463         return NULL;
2464 }
2465
2466 /*
2467  * Output
2468  */
2469
2470 struct perf_output_handle {
2471         struct perf_counter     *counter;
2472         struct perf_mmap_data   *data;
2473         unsigned long           head;
2474         unsigned long           offset;
2475         int                     nmi;
2476         int                     sample;
2477         int                     locked;
2478         unsigned long           flags;
2479 };
2480
2481 static bool perf_output_space(struct perf_mmap_data *data,
2482                               unsigned int offset, unsigned int head)
2483 {
2484         unsigned long tail;
2485         unsigned long mask;
2486
2487         if (!data->writable)
2488                 return true;
2489
2490         mask = (data->nr_pages << PAGE_SHIFT) - 1;
2491         /*
2492          * Userspace could choose to issue a mb() before updating the tail
2493          * pointer. So that all reads will be completed before the write is
2494          * issued.
2495          */
2496         tail = ACCESS_ONCE(data->user_page->data_tail);
2497         smp_rmb();
2498
2499         offset = (offset - tail) & mask;
2500         head   = (head   - tail) & mask;
2501
2502         if ((int)(head - offset) < 0)
2503                 return false;
2504
2505         return true;
2506 }
2507
2508 static void perf_output_wakeup(struct perf_output_handle *handle)
2509 {
2510         atomic_set(&handle->data->poll, POLL_IN);
2511
2512         if (handle->nmi) {
2513                 handle->counter->pending_wakeup = 1;
2514                 perf_pending_queue(&handle->counter->pending,
2515                                    perf_pending_counter);
2516         } else
2517                 perf_counter_wakeup(handle->counter);
2518 }
2519
2520 /*
2521  * Curious locking construct.
2522  *
2523  * We need to ensure a later event doesn't publish a head when a former
2524  * event isn't done writing. However since we need to deal with NMIs we
2525  * cannot fully serialize things.
2526  *
2527  * What we do is serialize between CPUs so we only have to deal with NMI
2528  * nesting on a single CPU.
2529  *
2530  * We only publish the head (and generate a wakeup) when the outer-most
2531  * event completes.
2532  */
2533 static void perf_output_lock(struct perf_output_handle *handle)
2534 {
2535         struct perf_mmap_data *data = handle->data;
2536         int cpu;
2537
2538         handle->locked = 0;
2539
2540         local_irq_save(handle->flags);
2541         cpu = smp_processor_id();
2542
2543         if (in_nmi() && atomic_read(&data->lock) == cpu)
2544                 return;
2545
2546         while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
2547                 cpu_relax();
2548
2549         handle->locked = 1;
2550 }
2551
2552 static void perf_output_unlock(struct perf_output_handle *handle)
2553 {
2554         struct perf_mmap_data *data = handle->data;
2555         unsigned long head;
2556         int cpu;
2557
2558         data->done_head = data->head;
2559
2560         if (!handle->locked)
2561                 goto out;
2562
2563 again:
2564         /*
2565          * The xchg implies a full barrier that ensures all writes are done
2566          * before we publish the new head, matched by a rmb() in userspace when
2567          * reading this position.
2568          */
2569         while ((head = atomic_long_xchg(&data->done_head, 0)))
2570                 data->user_page->data_head = head;
2571
2572         /*
2573          * NMI can happen here, which means we can miss a done_head update.
2574          */
2575
2576         cpu = atomic_xchg(&data->lock, -1);
2577         WARN_ON_ONCE(cpu != smp_processor_id());
2578
2579         /*
2580          * Therefore we have to validate we did not indeed do so.
2581          */
2582         if (unlikely(atomic_long_read(&data->done_head))) {
2583                 /*
2584                  * Since we had it locked, we can lock it again.
2585                  */
2586                 while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
2587                         cpu_relax();
2588
2589                 goto again;
2590         }
2591
2592         if (atomic_xchg(&data->wakeup, 0))
2593                 perf_output_wakeup(handle);
2594 out:
2595         local_irq_restore(handle->flags);
2596 }
2597
2598 static void perf_output_copy(struct perf_output_handle *handle,
2599                              const void *buf, unsigned int len)
2600 {
2601         unsigned int pages_mask;
2602         unsigned int offset;
2603         unsigned int size;
2604         void **pages;
2605
2606         offset          = handle->offset;
2607         pages_mask      = handle->data->nr_pages - 1;
2608         pages           = handle->data->data_pages;
2609
2610         do {
2611                 unsigned int page_offset;
2612                 int nr;
2613
2614                 nr          = (offset >> PAGE_SHIFT) & pages_mask;
2615                 page_offset = offset & (PAGE_SIZE - 1);
2616                 size        = min_t(unsigned int, PAGE_SIZE - page_offset, len);
2617
2618                 memcpy(pages[nr] + page_offset, buf, size);
2619
2620                 len         -= size;
2621                 buf         += size;
2622                 offset      += size;
2623         } while (len);
2624
2625         handle->offset = offset;
2626
2627         /*
2628          * Check we didn't copy past our reservation window, taking the
2629          * possible unsigned int wrap into account.
2630          */
2631         WARN_ON_ONCE(((long)(handle->head - handle->offset)) < 0);
2632 }
2633
2634 #define perf_output_put(handle, x) \
2635         perf_output_copy((handle), &(x), sizeof(x))
2636
2637 static int perf_output_begin(struct perf_output_handle *handle,
2638                              struct perf_counter *counter, unsigned int size,
2639                              int nmi, int sample)
2640 {
2641         struct perf_mmap_data *data;
2642         unsigned int offset, head;
2643         int have_lost;
2644         struct {
2645                 struct perf_event_header header;
2646                 u64                      id;
2647                 u64                      lost;
2648         } lost_event;
2649
2650         /*
2651          * For inherited counters we send all the output towards the parent.
2652          */
2653         if (counter->parent)
2654                 counter = counter->parent;
2655
2656         rcu_read_lock();
2657         data = rcu_dereference(counter->data);
2658         if (!data)
2659                 goto out;
2660
2661         handle->data    = data;
2662         handle->counter = counter;
2663         handle->nmi     = nmi;
2664         handle->sample  = sample;
2665
2666         if (!data->nr_pages)
2667                 goto fail;
2668
2669         have_lost = atomic_read(&data->lost);
2670         if (have_lost)
2671                 size += sizeof(lost_event);
2672
2673         perf_output_lock(handle);
2674
2675         do {
2676                 offset = head = atomic_long_read(&data->head);
2677                 head += size;
2678                 if (unlikely(!perf_output_space(data, offset, head)))
2679                         goto fail;
2680         } while (atomic_long_cmpxchg(&data->head, offset, head) != offset);
2681
2682         handle->offset  = offset;
2683         handle->head    = head;
2684
2685         if ((offset >> PAGE_SHIFT) != (head >> PAGE_SHIFT))
2686                 atomic_set(&data->wakeup, 1);
2687
2688         if (have_lost) {
2689                 lost_event.header.type = PERF_EVENT_LOST;
2690                 lost_event.header.misc = 0;
2691                 lost_event.header.size = sizeof(lost_event);
2692                 lost_event.id          = counter->id;
2693                 lost_event.lost        = atomic_xchg(&data->lost, 0);
2694
2695                 perf_output_put(handle, lost_event);
2696         }
2697
2698         return 0;
2699
2700 fail:
2701         atomic_inc(&data->lost);
2702         perf_output_unlock(handle);
2703 out:
2704         rcu_read_unlock();
2705
2706         return -ENOSPC;
2707 }
2708
2709 static void perf_output_end(struct perf_output_handle *handle)
2710 {
2711         struct perf_counter *counter = handle->counter;
2712         struct perf_mmap_data *data = handle->data;
2713
2714         int wakeup_events = counter->attr.wakeup_events;
2715
2716         if (handle->sample && wakeup_events) {
2717                 int events = atomic_inc_return(&data->events);
2718                 if (events >= wakeup_events) {
2719                         atomic_sub(wakeup_events, &data->events);
2720                         atomic_set(&data->wakeup, 1);
2721                 }
2722         }
2723
2724         perf_output_unlock(handle);
2725         rcu_read_unlock();
2726 }
2727
2728 static u32 perf_counter_pid(struct perf_counter *counter, struct task_struct *p)
2729 {
2730         /*
2731          * only top level counters have the pid namespace they were created in
2732          */
2733         if (counter->parent)
2734                 counter = counter->parent;
2735
2736         return task_tgid_nr_ns(p, counter->ns);
2737 }
2738
2739 static u32 perf_counter_tid(struct perf_counter *counter, struct task_struct *p)
2740 {
2741         /*
2742          * only top level counters have the pid namespace they were created in
2743          */
2744         if (counter->parent)
2745                 counter = counter->parent;
2746
2747         return task_pid_nr_ns(p, counter->ns);
2748 }
2749
2750 static void perf_output_read_one(struct perf_output_handle *handle,
2751                                  struct perf_counter *counter)
2752 {
2753         u64 read_format = counter->attr.read_format;
2754         u64 values[4];
2755         int n = 0;
2756
2757         values[n++] = atomic64_read(&counter->count);
2758         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
2759                 values[n++] = counter->total_time_enabled +
2760                         atomic64_read(&counter->child_total_time_enabled);
2761         }
2762         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
2763                 values[n++] = counter->total_time_running +
2764                         atomic64_read(&counter->child_total_time_running);
2765         }
2766         if (read_format & PERF_FORMAT_ID)
2767                 values[n++] = primary_counter_id(counter);
2768
2769         perf_output_copy(handle, values, n * sizeof(u64));
2770 }
2771
2772 /*
2773  * XXX PERF_FORMAT_GROUP vs inherited counters seems difficult.
2774  */
2775 static void perf_output_read_group(struct perf_output_handle *handle,
2776                             struct perf_counter *counter)
2777 {
2778         struct perf_counter *leader = counter->group_leader, *sub;
2779         u64 read_format = counter->attr.read_format;
2780         u64 values[5];
2781         int n = 0;
2782
2783         values[n++] = 1 + leader->nr_siblings;
2784
2785         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
2786                 values[n++] = leader->total_time_enabled;
2787
2788         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
2789                 values[n++] = leader->total_time_running;
2790
2791         if (leader != counter)
2792                 leader->pmu->read(leader);
2793
2794         values[n++] = atomic64_read(&leader->count);
2795         if (read_format & PERF_FORMAT_ID)
2796                 values[n++] = primary_counter_id(leader);
2797
2798         perf_output_copy(handle, values, n * sizeof(u64));
2799
2800         list_for_each_entry(sub, &leader->sibling_list, list_entry) {
2801                 n = 0;
2802
2803                 if (sub != counter)
2804                         sub->pmu->read(sub);
2805
2806                 values[n++] = atomic64_read(&sub->count);
2807                 if (read_format & PERF_FORMAT_ID)
2808                         values[n++] = primary_counter_id(sub);
2809
2810                 perf_output_copy(handle, values, n * sizeof(u64));
2811         }
2812 }
2813
2814 static void perf_output_read(struct perf_output_handle *handle,
2815                              struct perf_counter *counter)
2816 {
2817         if (counter->attr.read_format & PERF_FORMAT_GROUP)
2818                 perf_output_read_group(handle, counter);
2819         else
2820                 perf_output_read_one(handle, counter);
2821 }
2822
2823 void perf_counter_output(struct perf_counter *counter, int nmi,
2824                                 struct perf_sample_data *data)
2825 {
2826         int ret;
2827         u64 sample_type = counter->attr.sample_type;
2828         struct perf_output_handle handle;
2829         struct perf_event_header header;
2830         u64 ip;
2831         struct {
2832                 u32 pid, tid;
2833         } tid_entry;
2834         struct perf_callchain_entry *callchain = NULL;
2835         int callchain_size = 0;
2836         u64 time;
2837         struct {
2838                 u32 cpu, reserved;
2839         } cpu_entry;
2840
2841         header.type = PERF_EVENT_SAMPLE;
2842         header.size = sizeof(header);
2843
2844         header.misc = 0;
2845         header.misc |= perf_misc_flags(data->regs);
2846
2847         if (sample_type & PERF_SAMPLE_IP) {
2848                 ip = perf_instruction_pointer(data->regs);
2849                 header.size += sizeof(ip);
2850         }
2851
2852         if (sample_type & PERF_SAMPLE_TID) {
2853                 /* namespace issues */
2854                 tid_entry.pid = perf_counter_pid(counter, current);
2855                 tid_entry.tid = perf_counter_tid(counter, current);
2856
2857                 header.size += sizeof(tid_entry);
2858         }
2859
2860         if (sample_type & PERF_SAMPLE_TIME) {
2861                 /*
2862                  * Maybe do better on x86 and provide cpu_clock_nmi()
2863                  */
2864                 time = sched_clock();
2865
2866                 header.size += sizeof(u64);
2867         }
2868
2869         if (sample_type & PERF_SAMPLE_ADDR)
2870                 header.size += sizeof(u64);
2871
2872         if (sample_type & PERF_SAMPLE_ID)
2873                 header.size += sizeof(u64);
2874
2875         if (sample_type & PERF_SAMPLE_STREAM_ID)
2876                 header.size += sizeof(u64);
2877
2878         if (sample_type & PERF_SAMPLE_CPU) {
2879                 header.size += sizeof(cpu_entry);
2880
2881                 cpu_entry.cpu = raw_smp_processor_id();
2882                 cpu_entry.reserved = 0;
2883         }
2884
2885         if (sample_type & PERF_SAMPLE_PERIOD)
2886                 header.size += sizeof(u64);
2887
2888         if (sample_type & PERF_SAMPLE_READ)
2889                 header.size += perf_counter_read_size(counter);
2890
2891         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
2892                 callchain = perf_callchain(data->regs);
2893
2894                 if (callchain) {
2895                         callchain_size = (1 + callchain->nr) * sizeof(u64);
2896                         header.size += callchain_size;
2897                 } else
2898                         header.size += sizeof(u64);
2899         }
2900
2901         if (sample_type & PERF_SAMPLE_RAW) {
2902                 int size = sizeof(u32);
2903
2904                 if (data->raw)
2905                         size += data->raw->size;
2906                 else
2907                         size += sizeof(u32);
2908
2909                 WARN_ON_ONCE(size & (sizeof(u64)-1));
2910                 header.size += size;
2911         }
2912
2913         ret = perf_output_begin(&handle, counter, header.size, nmi, 1);
2914         if (ret)
2915                 return;
2916
2917         perf_output_put(&handle, header);
2918
2919         if (sample_type & PERF_SAMPLE_IP)
2920                 perf_output_put(&handle, ip);
2921
2922         if (sample_type & PERF_SAMPLE_TID)
2923                 perf_output_put(&handle, tid_entry);
2924
2925         if (sample_type & PERF_SAMPLE_TIME)
2926                 perf_output_put(&handle, time);
2927
2928         if (sample_type & PERF_SAMPLE_ADDR)
2929                 perf_output_put(&handle, data->addr);
2930
2931         if (sample_type & PERF_SAMPLE_ID) {
2932                 u64 id = primary_counter_id(counter);
2933
2934                 perf_output_put(&handle, id);
2935         }
2936
2937         if (sample_type & PERF_SAMPLE_STREAM_ID)
2938                 perf_output_put(&handle, counter->id);
2939
2940         if (sample_type & PERF_SAMPLE_CPU)
2941                 perf_output_put(&handle, cpu_entry);
2942
2943         if (sample_type & PERF_SAMPLE_PERIOD)
2944                 perf_output_put(&handle, data->period);
2945
2946         if (sample_type & PERF_SAMPLE_READ)
2947                 perf_output_read(&handle, counter);
2948
2949         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
2950                 if (callchain)
2951                         perf_output_copy(&handle, callchain, callchain_size);
2952                 else {
2953                         u64 nr = 0;
2954                         perf_output_put(&handle, nr);
2955                 }
2956         }
2957
2958         if (sample_type & PERF_SAMPLE_RAW) {
2959                 if (data->raw) {
2960                         perf_output_put(&handle, data->raw->size);
2961                         perf_output_copy(&handle, data->raw->data, data->raw->size);
2962                 } else {
2963                         struct {
2964                                 u32     size;
2965                                 u32     data;
2966                         } raw = {
2967                                 .size = sizeof(u32),
2968                                 .data = 0,
2969                         };
2970                         perf_output_put(&handle, raw);
2971                 }
2972         }
2973
2974         perf_output_end(&handle);
2975 }
2976
2977 /*
2978  * read event
2979  */
2980
2981 struct perf_read_event {
2982         struct perf_event_header        header;
2983
2984         u32                             pid;
2985         u32                             tid;
2986 };
2987
2988 static void
2989 perf_counter_read_event(struct perf_counter *counter,
2990                         struct task_struct *task)
2991 {
2992         struct perf_output_handle handle;
2993         struct perf_read_event event = {
2994                 .header = {
2995                         .type = PERF_EVENT_READ,
2996                         .misc = 0,
2997                         .size = sizeof(event) + perf_counter_read_size(counter),
2998                 },
2999                 .pid = perf_counter_pid(counter, task),
3000                 .tid = perf_counter_tid(counter, task),
3001         };
3002         int ret;
3003
3004         ret = perf_output_begin(&handle, counter, event.header.size, 0, 0);
3005         if (ret)
3006                 return;
3007
3008         perf_output_put(&handle, event);
3009         perf_output_read(&handle, counter);
3010
3011         perf_output_end(&handle);
3012 }
3013
3014 /*
3015  * task tracking -- fork/exit
3016  *
3017  * enabled by: attr.comm | attr.mmap | attr.task
3018  */
3019
3020 struct perf_task_event {
3021         struct task_struct              *task;
3022         struct perf_counter_context     *task_ctx;
3023
3024         struct {
3025                 struct perf_event_header        header;
3026
3027                 u32                             pid;
3028                 u32                             ppid;
3029                 u32                             tid;
3030                 u32                             ptid;
3031         } event;
3032 };
3033
3034 static void perf_counter_task_output(struct perf_counter *counter,
3035                                      struct perf_task_event *task_event)
3036 {
3037         struct perf_output_handle handle;
3038         int size = task_event->event.header.size;
3039         struct task_struct *task = task_event->task;
3040         int ret = perf_output_begin(&handle, counter, size, 0, 0);
3041
3042         if (ret)
3043                 return;
3044
3045         task_event->event.pid = perf_counter_pid(counter, task);
3046         task_event->event.ppid = perf_counter_pid(counter, current);
3047
3048         task_event->event.tid = perf_counter_tid(counter, task);
3049         task_event->event.ptid = perf_counter_tid(counter, current);
3050
3051         perf_output_put(&handle, task_event->event);
3052         perf_output_end(&handle);
3053 }
3054
3055 static int perf_counter_task_match(struct perf_counter *counter)
3056 {
3057         if (counter->attr.comm || counter->attr.mmap || counter->attr.task)
3058                 return 1;
3059
3060         return 0;
3061 }
3062
3063 static void perf_counter_task_ctx(struct perf_counter_context *ctx,
3064                                   struct perf_task_event *task_event)
3065 {
3066         struct perf_counter *counter;
3067
3068         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
3069                 return;
3070
3071         rcu_read_lock();
3072         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
3073                 if (perf_counter_task_match(counter))
3074                         perf_counter_task_output(counter, task_event);
3075         }
3076         rcu_read_unlock();
3077 }
3078
3079 static void perf_counter_task_event(struct perf_task_event *task_event)
3080 {
3081         struct perf_cpu_context *cpuctx;
3082         struct perf_counter_context *ctx = task_event->task_ctx;
3083
3084         cpuctx = &get_cpu_var(perf_cpu_context);
3085         perf_counter_task_ctx(&cpuctx->ctx, task_event);
3086         put_cpu_var(perf_cpu_context);
3087
3088         rcu_read_lock();
3089         if (!ctx)
3090                 ctx = rcu_dereference(task_event->task->perf_counter_ctxp);
3091         if (ctx)
3092                 perf_counter_task_ctx(ctx, task_event);
3093         rcu_read_unlock();
3094 }
3095
3096 static void perf_counter_task(struct task_struct *task,
3097                               struct perf_counter_context *task_ctx,
3098                               int new)
3099 {
3100         struct perf_task_event task_event;
3101
3102         if (!atomic_read(&nr_comm_counters) &&
3103             !atomic_read(&nr_mmap_counters) &&
3104             !atomic_read(&nr_task_counters))
3105                 return;
3106
3107         task_event = (struct perf_task_event){
3108                 .task     = task,
3109                 .task_ctx = task_ctx,
3110                 .event    = {
3111                         .header = {
3112                                 .type = new ? PERF_EVENT_FORK : PERF_EVENT_EXIT,
3113                                 .misc = 0,
3114                                 .size = sizeof(task_event.event),
3115                         },
3116                         /* .pid  */
3117                         /* .ppid */
3118                         /* .tid  */
3119                         /* .ptid */
3120                 },
3121         };
3122
3123         perf_counter_task_event(&task_event);
3124 }
3125
3126 void perf_counter_fork(struct task_struct *task)
3127 {
3128         perf_counter_task(task, NULL, 1);
3129 }
3130
3131 /*
3132  * comm tracking
3133  */
3134
3135 struct perf_comm_event {
3136         struct task_struct      *task;
3137         char                    *comm;
3138         int                     comm_size;
3139
3140         struct {
3141                 struct perf_event_header        header;
3142
3143                 u32                             pid;
3144                 u32                             tid;
3145         } event;
3146 };
3147
3148 static void perf_counter_comm_output(struct perf_counter *counter,
3149                                      struct perf_comm_event *comm_event)
3150 {
3151         struct perf_output_handle handle;
3152         int size = comm_event->event.header.size;
3153         int ret = perf_output_begin(&handle, counter, size, 0, 0);
3154
3155         if (ret)
3156                 return;
3157
3158         comm_event->event.pid = perf_counter_pid(counter, comm_event->task);
3159         comm_event->event.tid = perf_counter_tid(counter, comm_event->task);
3160
3161         perf_output_put(&handle, comm_event->event);
3162         perf_output_copy(&handle, comm_event->comm,
3163                                    comm_event->comm_size);
3164         perf_output_end(&handle);
3165 }
3166
3167 static int perf_counter_comm_match(struct perf_counter *counter)
3168 {
3169         if (counter->attr.comm)
3170                 return 1;
3171
3172         return 0;
3173 }
3174
3175 static void perf_counter_comm_ctx(struct perf_counter_context *ctx,
3176                                   struct perf_comm_event *comm_event)
3177 {
3178         struct perf_counter *counter;
3179
3180         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
3181                 return;
3182
3183         rcu_read_lock();
3184         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
3185                 if (perf_counter_comm_match(counter))
3186                         perf_counter_comm_output(counter, comm_event);
3187         }
3188         rcu_read_unlock();
3189 }
3190
3191 static void perf_counter_comm_event(struct perf_comm_event *comm_event)
3192 {
3193         struct perf_cpu_context *cpuctx;
3194         struct perf_counter_context *ctx;
3195         unsigned int size;
3196         char comm[TASK_COMM_LEN];
3197
3198         memset(comm, 0, sizeof(comm));
3199         strncpy(comm, comm_event->task->comm, sizeof(comm));
3200         size = ALIGN(strlen(comm)+1, sizeof(u64));
3201
3202         comm_event->comm = comm;
3203         comm_event->comm_size = size;
3204
3205         comm_event->event.header.size = sizeof(comm_event->event) + size;
3206
3207         cpuctx = &get_cpu_var(perf_cpu_context);
3208         perf_counter_comm_ctx(&cpuctx->ctx, comm_event);
3209         put_cpu_var(perf_cpu_context);
3210
3211         rcu_read_lock();
3212         /*
3213          * doesn't really matter which of the child contexts the
3214          * events ends up in.
3215          */
3216         ctx = rcu_dereference(current->perf_counter_ctxp);
3217         if (ctx)
3218                 perf_counter_comm_ctx(ctx, comm_event);
3219         rcu_read_unlock();
3220 }
3221
3222 void perf_counter_comm(struct task_struct *task)
3223 {
3224         struct perf_comm_event comm_event;
3225
3226         if (task->perf_counter_ctxp)
3227                 perf_counter_enable_on_exec(task);
3228
3229         if (!atomic_read(&nr_comm_counters))
3230                 return;
3231
3232         comm_event = (struct perf_comm_event){
3233                 .task   = task,
3234                 /* .comm      */
3235                 /* .comm_size */
3236                 .event  = {
3237                         .header = {
3238                                 .type = PERF_EVENT_COMM,
3239                                 .misc = 0,
3240                                 /* .size */
3241                         },
3242                         /* .pid */
3243                         /* .tid */
3244                 },
3245         };
3246
3247         perf_counter_comm_event(&comm_event);
3248 }
3249
3250 /*
3251  * mmap tracking
3252  */
3253
3254 struct perf_mmap_event {
3255         struct vm_area_struct   *vma;
3256
3257         const char              *file_name;
3258         int                     file_size;
3259
3260         struct {
3261                 struct perf_event_header        header;
3262
3263                 u32                             pid;
3264                 u32                             tid;
3265                 u64                             start;
3266                 u64                             len;
3267                 u64                             pgoff;
3268         } event;
3269 };
3270
3271 static void perf_counter_mmap_output(struct perf_counter *counter,
3272                                      struct perf_mmap_event *mmap_event)
3273 {
3274         struct perf_output_handle handle;
3275         int size = mmap_event->event.header.size;
3276         int ret = perf_output_begin(&handle, counter, size, 0, 0);
3277
3278         if (ret)
3279                 return;
3280
3281         mmap_event->event.pid = perf_counter_pid(counter, current);
3282         mmap_event->event.tid = perf_counter_tid(counter, current);
3283
3284         perf_output_put(&handle, mmap_event->event);
3285         perf_output_copy(&handle, mmap_event->file_name,
3286                                    mmap_event->file_size);
3287         perf_output_end(&handle);
3288 }
3289
3290 static int perf_counter_mmap_match(struct perf_counter *counter,
3291                                    struct perf_mmap_event *mmap_event)
3292 {
3293         if (counter->attr.mmap)
3294                 return 1;
3295
3296         return 0;
3297 }
3298
3299 static void perf_counter_mmap_ctx(struct perf_counter_context *ctx,
3300                                   struct perf_mmap_event *mmap_event)
3301 {
3302         struct perf_counter *counter;
3303
3304         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
3305                 return;
3306
3307         rcu_read_lock();
3308         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
3309                 if (perf_counter_mmap_match(counter, mmap_event))
3310                         perf_counter_mmap_output(counter, mmap_event);
3311         }
3312         rcu_read_unlock();
3313 }
3314
3315 static void perf_counter_mmap_event(struct perf_mmap_event *mmap_event)
3316 {
3317         struct perf_cpu_context *cpuctx;
3318         struct perf_counter_context *ctx;
3319         struct vm_area_struct *vma = mmap_event->vma;
3320         struct file *file = vma->vm_file;
3321         unsigned int size;
3322         char tmp[16];
3323         char *buf = NULL;
3324         const char *name;
3325
3326         memset(tmp, 0, sizeof(tmp));
3327
3328         if (file) {
3329                 /*
3330                  * d_path works from the end of the buffer backwards, so we
3331                  * need to add enough zero bytes after the string to handle
3332                  * the 64bit alignment we do later.
3333                  */
3334                 buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
3335                 if (!buf) {
3336                         name = strncpy(tmp, "//enomem", sizeof(tmp));
3337                         goto got_name;
3338                 }
3339                 name = d_path(&file->f_path, buf, PATH_MAX);
3340                 if (IS_ERR(name)) {
3341                         name = strncpy(tmp, "//toolong", sizeof(tmp));
3342                  &