Merge branch 'stable-3.2' into pandora-3.2
[pandora-kernel.git] / kernel / trace / trace.c
1 /*
2  * ring buffer based function tracer
3  *
4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6  *
7  * Originally taken from the RT patch by:
8  *    Arnaldo Carvalho de Melo <acme@redhat.com>
9  *
10  * Based on code from the latency_tracer, that is:
11  *  Copyright (C) 2004-2006 Ingo Molnar
12  *  Copyright (C) 2004 William Lee Irwin III
13  */
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/pagemap.h>
24 #include <linux/hardirq.h>
25 #include <linux/linkage.h>
26 #include <linux/uaccess.h>
27 #include <linux/kprobes.h>
28 #include <linux/ftrace.h>
29 #include <linux/module.h>
30 #include <linux/percpu.h>
31 #include <linux/splice.h>
32 #include <linux/kdebug.h>
33 #include <linux/string.h>
34 #include <linux/rwsem.h>
35 #include <linux/slab.h>
36 #include <linux/ctype.h>
37 #include <linux/init.h>
38 #include <linux/poll.h>
39 #include <linux/fs.h>
40
41 #include "trace.h"
42 #include "trace_output.h"
43
44 /*
45  * On boot up, the ring buffer is set to the minimum size, so that
46  * we do not waste memory on systems that are not using tracing.
47  */
48 int ring_buffer_expanded;
49
50 /*
51  * We need to change this state when a selftest is running.
52  * A selftest will lurk into the ring-buffer to count the
53  * entries inserted during the selftest although some concurrent
54  * insertions into the ring-buffer such as trace_printk could occurred
55  * at the same time, giving false positive or negative results.
56  */
57 static bool __read_mostly tracing_selftest_running;
58
59 /*
60  * If a tracer is running, we do not want to run SELFTEST.
61  */
62 bool __read_mostly tracing_selftest_disabled;
63
64 /* For tracers that don't implement custom flags */
65 static struct tracer_opt dummy_tracer_opt[] = {
66         { }
67 };
68
69 static struct tracer_flags dummy_tracer_flags = {
70         .val = 0,
71         .opts = dummy_tracer_opt
72 };
73
74 static int dummy_set_flag(u32 old_flags, u32 bit, int set)
75 {
76         return 0;
77 }
78
79 /*
80  * Kill all tracing for good (never come back).
81  * It is initialized to 1 but will turn to zero if the initialization
82  * of the tracer is successful. But that is the only place that sets
83  * this back to zero.
84  */
85 static int tracing_disabled = 1;
86
87 DEFINE_PER_CPU(int, ftrace_cpu_disabled);
88
89 static inline void ftrace_disable_cpu(void)
90 {
91         preempt_disable();
92         __this_cpu_inc(ftrace_cpu_disabled);
93 }
94
95 static inline void ftrace_enable_cpu(void)
96 {
97         __this_cpu_dec(ftrace_cpu_disabled);
98         preempt_enable();
99 }
100
101 cpumask_var_t __read_mostly     tracing_buffer_mask;
102
103 /*
104  * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
105  *
106  * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
107  * is set, then ftrace_dump is called. This will output the contents
108  * of the ftrace buffers to the console.  This is very useful for
109  * capturing traces that lead to crashes and outputing it to a
110  * serial console.
111  *
112  * It is default off, but you can enable it with either specifying
113  * "ftrace_dump_on_oops" in the kernel command line, or setting
114  * /proc/sys/kernel/ftrace_dump_on_oops
115  * Set 1 if you want to dump buffers of all CPUs
116  * Set 2 if you want to dump the buffer of the CPU that triggered oops
117  */
118
119 enum ftrace_dump_mode ftrace_dump_on_oops;
120
121 static int tracing_set_tracer(const char *buf);
122
123 #define MAX_TRACER_SIZE         100
124 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
125 static char *default_bootup_tracer;
126
127 static int __init set_cmdline_ftrace(char *str)
128 {
129         strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
130         default_bootup_tracer = bootup_tracer_buf;
131         /* We are using ftrace early, expand it */
132         ring_buffer_expanded = 1;
133         return 1;
134 }
135 __setup("ftrace=", set_cmdline_ftrace);
136
137 static int __init set_ftrace_dump_on_oops(char *str)
138 {
139         if (*str++ != '=' || !*str) {
140                 ftrace_dump_on_oops = DUMP_ALL;
141                 return 1;
142         }
143
144         if (!strcmp("orig_cpu", str)) {
145                 ftrace_dump_on_oops = DUMP_ORIG;
146                 return 1;
147         }
148
149         return 0;
150 }
151 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
152
153 unsigned long long ns2usecs(cycle_t nsec)
154 {
155         nsec += 500;
156         do_div(nsec, 1000);
157         return nsec;
158 }
159
160 /*
161  * The global_trace is the descriptor that holds the tracing
162  * buffers for the live tracing. For each CPU, it contains
163  * a link list of pages that will store trace entries. The
164  * page descriptor of the pages in the memory is used to hold
165  * the link list by linking the lru item in the page descriptor
166  * to each of the pages in the buffer per CPU.
167  *
168  * For each active CPU there is a data field that holds the
169  * pages for the buffer for that CPU. Each CPU has the same number
170  * of pages allocated for its buffer.
171  */
172 static struct trace_array       global_trace;
173
174 static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
175
176 int filter_current_check_discard(struct ring_buffer *buffer,
177                                  struct ftrace_event_call *call, void *rec,
178                                  struct ring_buffer_event *event)
179 {
180         return filter_check_discard(call, rec, buffer, event);
181 }
182 EXPORT_SYMBOL_GPL(filter_current_check_discard);
183
184 cycle_t ftrace_now(int cpu)
185 {
186         u64 ts;
187
188         /* Early boot up does not have a buffer yet */
189         if (!global_trace.buffer)
190                 return trace_clock_local();
191
192         ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
193         ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
194
195         return ts;
196 }
197
198 /*
199  * The max_tr is used to snapshot the global_trace when a maximum
200  * latency is reached. Some tracers will use this to store a maximum
201  * trace while it continues examining live traces.
202  *
203  * The buffers for the max_tr are set up the same as the global_trace.
204  * When a snapshot is taken, the link list of the max_tr is swapped
205  * with the link list of the global_trace and the buffers are reset for
206  * the global_trace so the tracing can continue.
207  */
208 static struct trace_array       max_tr;
209
210 static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
211
212 /* tracer_enabled is used to toggle activation of a tracer */
213 static int                      tracer_enabled = 1;
214
215 /**
216  * tracing_is_enabled - return tracer_enabled status
217  *
218  * This function is used by other tracers to know the status
219  * of the tracer_enabled flag.  Tracers may use this function
220  * to know if it should enable their features when starting
221  * up. See irqsoff tracer for an example (start_irqsoff_tracer).
222  */
223 int tracing_is_enabled(void)
224 {
225         return tracer_enabled;
226 }
227
228 /*
229  * trace_buf_size is the size in bytes that is allocated
230  * for a buffer. Note, the number of bytes is always rounded
231  * to page size.
232  *
233  * This number is purposely set to a low number of 16384.
234  * If the dump on oops happens, it will be much appreciated
235  * to not have to wait for all that output. Anyway this can be
236  * boot time and run time configurable.
237  */
238 #define TRACE_BUF_SIZE_DEFAULT  1441792UL /* 16384 * 88 (sizeof(entry)) */
239
240 static unsigned long            trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
241
242 /* trace_types holds a link list of available tracers. */
243 static struct tracer            *trace_types __read_mostly;
244
245 /* current_trace points to the tracer that is currently active */
246 static struct tracer            *current_trace __read_mostly;
247
248 /*
249  * trace_types_lock is used to protect the trace_types list.
250  */
251 static DEFINE_MUTEX(trace_types_lock);
252
253 /*
254  * serialize the access of the ring buffer
255  *
256  * ring buffer serializes readers, but it is low level protection.
257  * The validity of the events (which returns by ring_buffer_peek() ..etc)
258  * are not protected by ring buffer.
259  *
260  * The content of events may become garbage if we allow other process consumes
261  * these events concurrently:
262  *   A) the page of the consumed events may become a normal page
263  *      (not reader page) in ring buffer, and this page will be rewrited
264  *      by events producer.
265  *   B) The page of the consumed events may become a page for splice_read,
266  *      and this page will be returned to system.
267  *
268  * These primitives allow multi process access to different cpu ring buffer
269  * concurrently.
270  *
271  * These primitives don't distinguish read-only and read-consume access.
272  * Multi read-only access are also serialized.
273  */
274
275 #ifdef CONFIG_SMP
276 static DECLARE_RWSEM(all_cpu_access_lock);
277 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
278
279 static inline void trace_access_lock(int cpu)
280 {
281         if (cpu == TRACE_PIPE_ALL_CPU) {
282                 /* gain it for accessing the whole ring buffer. */
283                 down_write(&all_cpu_access_lock);
284         } else {
285                 /* gain it for accessing a cpu ring buffer. */
286
287                 /* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */
288                 down_read(&all_cpu_access_lock);
289
290                 /* Secondly block other access to this @cpu ring buffer. */
291                 mutex_lock(&per_cpu(cpu_access_lock, cpu));
292         }
293 }
294
295 static inline void trace_access_unlock(int cpu)
296 {
297         if (cpu == TRACE_PIPE_ALL_CPU) {
298                 up_write(&all_cpu_access_lock);
299         } else {
300                 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
301                 up_read(&all_cpu_access_lock);
302         }
303 }
304
305 static inline void trace_access_lock_init(void)
306 {
307         int cpu;
308
309         for_each_possible_cpu(cpu)
310                 mutex_init(&per_cpu(cpu_access_lock, cpu));
311 }
312
313 #else
314
315 static DEFINE_MUTEX(access_lock);
316
317 static inline void trace_access_lock(int cpu)
318 {
319         (void)cpu;
320         mutex_lock(&access_lock);
321 }
322
323 static inline void trace_access_unlock(int cpu)
324 {
325         (void)cpu;
326         mutex_unlock(&access_lock);
327 }
328
329 static inline void trace_access_lock_init(void)
330 {
331 }
332
333 #endif
334
335 /* trace_wait is a waitqueue for tasks blocked on trace_poll */
336 static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
337
338 /* trace_flags holds trace_options default values */
339 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
340         TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
341         TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE;
342
343 static int trace_stop_count;
344 static DEFINE_RAW_SPINLOCK(tracing_start_lock);
345
346 static void wakeup_work_handler(struct work_struct *work)
347 {
348         wake_up(&trace_wait);
349 }
350
351 static DECLARE_DELAYED_WORK(wakeup_work, wakeup_work_handler);
352
353 /**
354  * trace_wake_up - wake up tasks waiting for trace input
355  *
356  * Schedules a delayed work to wake up any task that is blocked on the
357  * trace_wait queue. These is used with trace_poll for tasks polling the
358  * trace.
359  */
360 void trace_wake_up(void)
361 {
362         const unsigned long delay = msecs_to_jiffies(2);
363
364         if (trace_flags & TRACE_ITER_BLOCK)
365                 return;
366         schedule_delayed_work(&wakeup_work, delay);
367 }
368
369 static int __init set_buf_size(char *str)
370 {
371         unsigned long buf_size;
372
373         if (!str)
374                 return 0;
375         buf_size = memparse(str, &str);
376         /* nr_entries can not be zero */
377         if (buf_size == 0)
378                 return 0;
379         trace_buf_size = buf_size;
380         return 1;
381 }
382 __setup("trace_buf_size=", set_buf_size);
383
384 static int __init set_tracing_thresh(char *str)
385 {
386         unsigned long threshhold;
387         int ret;
388
389         if (!str)
390                 return 0;
391         ret = strict_strtoul(str, 0, &threshhold);
392         if (ret < 0)
393                 return 0;
394         tracing_thresh = threshhold * 1000;
395         return 1;
396 }
397 __setup("tracing_thresh=", set_tracing_thresh);
398
399 unsigned long nsecs_to_usecs(unsigned long nsecs)
400 {
401         return nsecs / 1000;
402 }
403
404 /* These must match the bit postions in trace_iterator_flags */
405 static const char *trace_options[] = {
406         "print-parent",
407         "sym-offset",
408         "sym-addr",
409         "verbose",
410         "raw",
411         "hex",
412         "bin",
413         "block",
414         "stacktrace",
415         "trace_printk",
416         "ftrace_preempt",
417         "branch",
418         "annotate",
419         "userstacktrace",
420         "sym-userobj",
421         "printk-msg-only",
422         "context-info",
423         "latency-format",
424         "sleep-time",
425         "graph-time",
426         "record-cmd",
427         "overwrite",
428         "disable_on_free",
429         NULL
430 };
431
432 static struct {
433         u64 (*func)(void);
434         const char *name;
435 } trace_clocks[] = {
436         { trace_clock_local,    "local" },
437         { trace_clock_global,   "global" },
438         { trace_clock_counter,  "counter" },
439 };
440
441 int trace_clock_id;
442
443 /*
444  * trace_parser_get_init - gets the buffer for trace parser
445  */
446 int trace_parser_get_init(struct trace_parser *parser, int size)
447 {
448         memset(parser, 0, sizeof(*parser));
449
450         parser->buffer = kmalloc(size, GFP_KERNEL);
451         if (!parser->buffer)
452                 return 1;
453
454         parser->size = size;
455         return 0;
456 }
457
458 /*
459  * trace_parser_put - frees the buffer for trace parser
460  */
461 void trace_parser_put(struct trace_parser *parser)
462 {
463         kfree(parser->buffer);
464 }
465
466 /*
467  * trace_get_user - reads the user input string separated by  space
468  * (matched by isspace(ch))
469  *
470  * For each string found the 'struct trace_parser' is updated,
471  * and the function returns.
472  *
473  * Returns number of bytes read.
474  *
475  * See kernel/trace/trace.h for 'struct trace_parser' details.
476  */
477 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
478         size_t cnt, loff_t *ppos)
479 {
480         char ch;
481         size_t read = 0;
482         ssize_t ret;
483
484         if (!*ppos)
485                 trace_parser_clear(parser);
486
487         ret = get_user(ch, ubuf++);
488         if (ret)
489                 goto out;
490
491         read++;
492         cnt--;
493
494         /*
495          * The parser is not finished with the last write,
496          * continue reading the user input without skipping spaces.
497          */
498         if (!parser->cont) {
499                 /* skip white space */
500                 while (cnt && isspace(ch)) {
501                         ret = get_user(ch, ubuf++);
502                         if (ret)
503                                 goto out;
504                         read++;
505                         cnt--;
506                 }
507
508                 /* only spaces were written */
509                 if (isspace(ch)) {
510                         *ppos += read;
511                         ret = read;
512                         goto out;
513                 }
514
515                 parser->idx = 0;
516         }
517
518         /* read the non-space input */
519         while (cnt && !isspace(ch)) {
520                 if (parser->idx < parser->size - 1)
521                         parser->buffer[parser->idx++] = ch;
522                 else {
523                         ret = -EINVAL;
524                         goto out;
525                 }
526                 ret = get_user(ch, ubuf++);
527                 if (ret)
528                         goto out;
529                 read++;
530                 cnt--;
531         }
532
533         /* We either got finished input or we have to wait for another call. */
534         if (isspace(ch)) {
535                 parser->buffer[parser->idx] = 0;
536                 parser->cont = false;
537         } else if (parser->idx < parser->size - 1) {
538                 parser->cont = true;
539                 parser->buffer[parser->idx++] = ch;
540         } else {
541                 ret = -EINVAL;
542                 goto out;
543         }
544
545         *ppos += read;
546         ret = read;
547
548 out:
549         return ret;
550 }
551
552 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
553 {
554         int len;
555         int ret;
556
557         if (!cnt)
558                 return 0;
559
560         if (s->len <= s->readpos)
561                 return -EBUSY;
562
563         len = s->len - s->readpos;
564         if (cnt > len)
565                 cnt = len;
566         ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
567         if (ret == cnt)
568                 return -EFAULT;
569
570         cnt -= ret;
571
572         s->readpos += cnt;
573         return cnt;
574 }
575
576 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
577 {
578         int len;
579         void *ret;
580
581         if (s->len <= s->readpos)
582                 return -EBUSY;
583
584         len = s->len - s->readpos;
585         if (cnt > len)
586                 cnt = len;
587         ret = memcpy(buf, s->buffer + s->readpos, cnt);
588         if (!ret)
589                 return -EFAULT;
590
591         s->readpos += cnt;
592         return cnt;
593 }
594
595 /*
596  * ftrace_max_lock is used to protect the swapping of buffers
597  * when taking a max snapshot. The buffers themselves are
598  * protected by per_cpu spinlocks. But the action of the swap
599  * needs its own lock.
600  *
601  * This is defined as a arch_spinlock_t in order to help
602  * with performance when lockdep debugging is enabled.
603  *
604  * It is also used in other places outside the update_max_tr
605  * so it needs to be defined outside of the
606  * CONFIG_TRACER_MAX_TRACE.
607  */
608 static arch_spinlock_t ftrace_max_lock =
609         (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
610
611 unsigned long __read_mostly     tracing_thresh;
612
613 #ifdef CONFIG_TRACER_MAX_TRACE
614 unsigned long __read_mostly     tracing_max_latency;
615
616 /*
617  * Copy the new maximum trace into the separate maximum-trace
618  * structure. (this way the maximum trace is permanently saved,
619  * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
620  */
621 static void
622 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
623 {
624         struct trace_array_cpu *data = tr->data[cpu];
625         struct trace_array_cpu *max_data;
626
627         max_tr.cpu = cpu;
628         max_tr.time_start = data->preempt_timestamp;
629
630         max_data = max_tr.data[cpu];
631         max_data->saved_latency = tracing_max_latency;
632         max_data->critical_start = data->critical_start;
633         max_data->critical_end = data->critical_end;
634
635         memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
636         max_data->pid = tsk->pid;
637         /*
638          * If tsk == current, then use current_uid(), as that does not use
639          * RCU. The irq tracer can be called out of RCU scope.
640          */
641         if (tsk == current)
642                 max_data->uid = current_uid();
643         else
644                 max_data->uid = task_uid(tsk);
645
646         max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
647         max_data->policy = tsk->policy;
648         max_data->rt_priority = tsk->rt_priority;
649
650         /* record this tasks comm */
651         tracing_record_cmdline(tsk);
652 }
653
654 /**
655  * update_max_tr - snapshot all trace buffers from global_trace to max_tr
656  * @tr: tracer
657  * @tsk: the task with the latency
658  * @cpu: The cpu that initiated the trace.
659  *
660  * Flip the buffers between the @tr and the max_tr and record information
661  * about which task was the cause of this latency.
662  */
663 void
664 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
665 {
666         struct ring_buffer *buf;
667
668         if (trace_stop_count)
669                 return;
670
671         WARN_ON_ONCE(!irqs_disabled());
672         if (!current_trace->use_max_tr) {
673                 WARN_ON_ONCE(1);
674                 return;
675         }
676         arch_spin_lock(&ftrace_max_lock);
677
678         buf = tr->buffer;
679         tr->buffer = max_tr.buffer;
680         max_tr.buffer = buf;
681
682         __update_max_tr(tr, tsk, cpu);
683         arch_spin_unlock(&ftrace_max_lock);
684 }
685
686 /**
687  * update_max_tr_single - only copy one trace over, and reset the rest
688  * @tr - tracer
689  * @tsk - task with the latency
690  * @cpu - the cpu of the buffer to copy.
691  *
692  * Flip the trace of a single CPU buffer between the @tr and the max_tr.
693  */
694 void
695 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
696 {
697         int ret;
698
699         if (trace_stop_count)
700                 return;
701
702         WARN_ON_ONCE(!irqs_disabled());
703         if (!current_trace->use_max_tr) {
704                 WARN_ON_ONCE(1);
705                 return;
706         }
707
708         arch_spin_lock(&ftrace_max_lock);
709
710         ftrace_disable_cpu();
711
712         ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
713
714         if (ret == -EBUSY) {
715                 /*
716                  * We failed to swap the buffer due to a commit taking
717                  * place on this CPU. We fail to record, but we reset
718                  * the max trace buffer (no one writes directly to it)
719                  * and flag that it failed.
720                  */
721                 trace_array_printk(&max_tr, _THIS_IP_,
722                         "Failed to swap buffers due to commit in progress\n");
723         }
724
725         ftrace_enable_cpu();
726
727         WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
728
729         __update_max_tr(tr, tsk, cpu);
730         arch_spin_unlock(&ftrace_max_lock);
731 }
732 #endif /* CONFIG_TRACER_MAX_TRACE */
733
734 /**
735  * register_tracer - register a tracer with the ftrace system.
736  * @type - the plugin for the tracer
737  *
738  * Register a new plugin tracer.
739  */
740 int register_tracer(struct tracer *type)
741 __releases(kernel_lock)
742 __acquires(kernel_lock)
743 {
744         struct tracer *t;
745         int ret = 0;
746
747         if (!type->name) {
748                 pr_info("Tracer must have a name\n");
749                 return -1;
750         }
751
752         if (strlen(type->name) >= MAX_TRACER_SIZE) {
753                 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
754                 return -1;
755         }
756
757         mutex_lock(&trace_types_lock);
758
759         tracing_selftest_running = true;
760
761         for (t = trace_types; t; t = t->next) {
762                 if (strcmp(type->name, t->name) == 0) {
763                         /* already found */
764                         pr_info("Tracer %s already registered\n",
765                                 type->name);
766                         ret = -1;
767                         goto out;
768                 }
769         }
770
771         if (!type->set_flag)
772                 type->set_flag = &dummy_set_flag;
773         if (!type->flags)
774                 type->flags = &dummy_tracer_flags;
775         else
776                 if (!type->flags->opts)
777                         type->flags->opts = dummy_tracer_opt;
778         if (!type->wait_pipe)
779                 type->wait_pipe = default_wait_pipe;
780
781
782 #ifdef CONFIG_FTRACE_STARTUP_TEST
783         if (type->selftest && !tracing_selftest_disabled) {
784                 struct tracer *saved_tracer = current_trace;
785                 struct trace_array *tr = &global_trace;
786
787                 /*
788                  * Run a selftest on this tracer.
789                  * Here we reset the trace buffer, and set the current
790                  * tracer to be this tracer. The tracer can then run some
791                  * internal tracing to verify that everything is in order.
792                  * If we fail, we do not register this tracer.
793                  */
794                 tracing_reset_online_cpus(tr);
795
796                 current_trace = type;
797
798                 /* If we expanded the buffers, make sure the max is expanded too */
799                 if (ring_buffer_expanded && type->use_max_tr)
800                         ring_buffer_resize(max_tr.buffer, trace_buf_size);
801
802                 /* the test is responsible for initializing and enabling */
803                 pr_info("Testing tracer %s: ", type->name);
804                 ret = type->selftest(type, tr);
805                 /* the test is responsible for resetting too */
806                 current_trace = saved_tracer;
807                 if (ret) {
808                         printk(KERN_CONT "FAILED!\n");
809                         goto out;
810                 }
811                 /* Only reset on passing, to avoid touching corrupted buffers */
812                 tracing_reset_online_cpus(tr);
813
814                 /* Shrink the max buffer again */
815                 if (ring_buffer_expanded && type->use_max_tr)
816                         ring_buffer_resize(max_tr.buffer, 1);
817
818                 printk(KERN_CONT "PASSED\n");
819         }
820 #endif
821
822         type->next = trace_types;
823         trace_types = type;
824
825  out:
826         tracing_selftest_running = false;
827         mutex_unlock(&trace_types_lock);
828
829         if (ret || !default_bootup_tracer)
830                 goto out_unlock;
831
832         if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
833                 goto out_unlock;
834
835         printk(KERN_INFO "Starting tracer '%s'\n", type->name);
836         /* Do we want this tracer to start on bootup? */
837         tracing_set_tracer(type->name);
838         default_bootup_tracer = NULL;
839         /* disable other selftests, since this will break it. */
840         tracing_selftest_disabled = 1;
841 #ifdef CONFIG_FTRACE_STARTUP_TEST
842         printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
843                type->name);
844 #endif
845
846  out_unlock:
847         return ret;
848 }
849
850 void unregister_tracer(struct tracer *type)
851 {
852         struct tracer **t;
853
854         mutex_lock(&trace_types_lock);
855         for (t = &trace_types; *t; t = &(*t)->next) {
856                 if (*t == type)
857                         goto found;
858         }
859         pr_info("Tracer %s not registered\n", type->name);
860         goto out;
861
862  found:
863         *t = (*t)->next;
864
865         if (type == current_trace && tracer_enabled) {
866                 tracer_enabled = 0;
867                 tracing_stop();
868                 if (current_trace->stop)
869                         current_trace->stop(&global_trace);
870                 current_trace = &nop_trace;
871         }
872 out:
873         mutex_unlock(&trace_types_lock);
874 }
875
876 static void __tracing_reset(struct ring_buffer *buffer, int cpu)
877 {
878         ftrace_disable_cpu();
879         ring_buffer_reset_cpu(buffer, cpu);
880         ftrace_enable_cpu();
881 }
882
883 void tracing_reset(struct trace_array *tr, int cpu)
884 {
885         struct ring_buffer *buffer = tr->buffer;
886
887         ring_buffer_record_disable(buffer);
888
889         /* Make sure all commits have finished */
890         synchronize_sched();
891         __tracing_reset(buffer, cpu);
892
893         ring_buffer_record_enable(buffer);
894 }
895
896 void tracing_reset_online_cpus(struct trace_array *tr)
897 {
898         struct ring_buffer *buffer = tr->buffer;
899         int cpu;
900
901         ring_buffer_record_disable(buffer);
902
903         /* Make sure all commits have finished */
904         synchronize_sched();
905
906         tr->time_start = ftrace_now(tr->cpu);
907
908         for_each_online_cpu(cpu)
909                 __tracing_reset(buffer, cpu);
910
911         ring_buffer_record_enable(buffer);
912 }
913
914 void tracing_reset_current(int cpu)
915 {
916         tracing_reset(&global_trace, cpu);
917 }
918
919 void tracing_reset_current_online_cpus(void)
920 {
921         tracing_reset_online_cpus(&global_trace);
922 }
923
924 #define SAVED_CMDLINES 128
925 #define NO_CMDLINE_MAP UINT_MAX
926 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
927 static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
928 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
929 static int cmdline_idx;
930 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
931
932 /* temporary disable recording */
933 static atomic_t trace_record_cmdline_disabled __read_mostly;
934
935 static void trace_init_cmdlines(void)
936 {
937         memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
938         memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
939         cmdline_idx = 0;
940 }
941
942 int is_tracing_stopped(void)
943 {
944         return trace_stop_count;
945 }
946
947 /**
948  * ftrace_off_permanent - disable all ftrace code permanently
949  *
950  * This should only be called when a serious anomally has
951  * been detected.  This will turn off the function tracing,
952  * ring buffers, and other tracing utilites. It takes no
953  * locks and can be called from any context.
954  */
955 void ftrace_off_permanent(void)
956 {
957         tracing_disabled = 1;
958         ftrace_stop();
959         tracing_off_permanent();
960 }
961
962 /**
963  * tracing_start - quick start of the tracer
964  *
965  * If tracing is enabled but was stopped by tracing_stop,
966  * this will start the tracer back up.
967  */
968 void tracing_start(void)
969 {
970         struct ring_buffer *buffer;
971         unsigned long flags;
972
973         if (tracing_disabled)
974                 return;
975
976         raw_spin_lock_irqsave(&tracing_start_lock, flags);
977         if (--trace_stop_count) {
978                 if (trace_stop_count < 0) {
979                         /* Someone screwed up their debugging */
980                         WARN_ON_ONCE(1);
981                         trace_stop_count = 0;
982                 }
983                 goto out;
984         }
985
986         /* Prevent the buffers from switching */
987         arch_spin_lock(&ftrace_max_lock);
988
989         buffer = global_trace.buffer;
990         if (buffer)
991                 ring_buffer_record_enable(buffer);
992
993         buffer = max_tr.buffer;
994         if (buffer)
995                 ring_buffer_record_enable(buffer);
996
997         arch_spin_unlock(&ftrace_max_lock);
998
999         ftrace_start();
1000  out:
1001         raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
1002 }
1003
1004 /**
1005  * tracing_stop - quick stop of the tracer
1006  *
1007  * Light weight way to stop tracing. Use in conjunction with
1008  * tracing_start.
1009  */
1010 void tracing_stop(void)
1011 {
1012         struct ring_buffer *buffer;
1013         unsigned long flags;
1014
1015         ftrace_stop();
1016         raw_spin_lock_irqsave(&tracing_start_lock, flags);
1017         if (trace_stop_count++)
1018                 goto out;
1019
1020         /* Prevent the buffers from switching */
1021         arch_spin_lock(&ftrace_max_lock);
1022
1023         buffer = global_trace.buffer;
1024         if (buffer)
1025                 ring_buffer_record_disable(buffer);
1026
1027         buffer = max_tr.buffer;
1028         if (buffer)
1029                 ring_buffer_record_disable(buffer);
1030
1031         arch_spin_unlock(&ftrace_max_lock);
1032
1033  out:
1034         raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
1035 }
1036
1037 void trace_stop_cmdline_recording(void);
1038
1039 static void trace_save_cmdline(struct task_struct *tsk)
1040 {
1041         unsigned pid, idx;
1042
1043         if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1044                 return;
1045
1046         /*
1047          * It's not the end of the world if we don't get
1048          * the lock, but we also don't want to spin
1049          * nor do we want to disable interrupts,
1050          * so if we miss here, then better luck next time.
1051          */
1052         if (!arch_spin_trylock(&trace_cmdline_lock))
1053                 return;
1054
1055         idx = map_pid_to_cmdline[tsk->pid];
1056         if (idx == NO_CMDLINE_MAP) {
1057                 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1058
1059                 /*
1060                  * Check whether the cmdline buffer at idx has a pid
1061                  * mapped. We are going to overwrite that entry so we
1062                  * need to clear the map_pid_to_cmdline. Otherwise we
1063                  * would read the new comm for the old pid.
1064                  */
1065                 pid = map_cmdline_to_pid[idx];
1066                 if (pid != NO_CMDLINE_MAP)
1067                         map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1068
1069                 map_cmdline_to_pid[idx] = tsk->pid;
1070                 map_pid_to_cmdline[tsk->pid] = idx;
1071
1072                 cmdline_idx = idx;
1073         }
1074
1075         memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1076
1077         arch_spin_unlock(&trace_cmdline_lock);
1078 }
1079
1080 void trace_find_cmdline(int pid, char comm[])
1081 {
1082         unsigned map;
1083
1084         if (!pid) {
1085                 strcpy(comm, "<idle>");
1086                 return;
1087         }
1088
1089         if (WARN_ON_ONCE(pid < 0)) {
1090                 strcpy(comm, "<XXX>");
1091                 return;
1092         }
1093
1094         if (pid > PID_MAX_DEFAULT) {
1095                 strcpy(comm, "<...>");
1096                 return;
1097         }
1098
1099         preempt_disable();
1100         arch_spin_lock(&trace_cmdline_lock);
1101         map = map_pid_to_cmdline[pid];
1102         if (map != NO_CMDLINE_MAP)
1103                 strcpy(comm, saved_cmdlines[map]);
1104         else
1105                 strcpy(comm, "<...>");
1106
1107         arch_spin_unlock(&trace_cmdline_lock);
1108         preempt_enable();
1109 }
1110
1111 void tracing_record_cmdline(struct task_struct *tsk)
1112 {
1113         if (atomic_read(&trace_record_cmdline_disabled) || !tracer_enabled ||
1114             !tracing_is_on())
1115                 return;
1116
1117         trace_save_cmdline(tsk);
1118 }
1119
1120 void
1121 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1122                              int pc)
1123 {
1124         struct task_struct *tsk = current;
1125
1126         entry->preempt_count            = pc & 0xff;
1127         entry->pid                      = (tsk) ? tsk->pid : 0;
1128         entry->padding                  = 0;
1129         entry->flags =
1130 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1131                 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1132 #else
1133                 TRACE_FLAG_IRQS_NOSUPPORT |
1134 #endif
1135                 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1136                 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1137                 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1138 }
1139 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1140
1141 struct ring_buffer_event *
1142 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1143                           int type,
1144                           unsigned long len,
1145                           unsigned long flags, int pc)
1146 {
1147         struct ring_buffer_event *event;
1148
1149         event = ring_buffer_lock_reserve(buffer, len);
1150         if (event != NULL) {
1151                 struct trace_entry *ent = ring_buffer_event_data(event);
1152
1153                 tracing_generic_entry_update(ent, flags, pc);
1154                 ent->type = type;
1155         }
1156
1157         return event;
1158 }
1159
1160 static inline void
1161 __trace_buffer_unlock_commit(struct ring_buffer *buffer,
1162                              struct ring_buffer_event *event,
1163                              unsigned long flags, int pc,
1164                              int wake)
1165 {
1166         ring_buffer_unlock_commit(buffer, event);
1167
1168         ftrace_trace_stack(buffer, flags, 6, pc);
1169         ftrace_trace_userstack(buffer, flags, pc);
1170
1171         if (wake)
1172                 trace_wake_up();
1173 }
1174
1175 void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1176                                 struct ring_buffer_event *event,
1177                                 unsigned long flags, int pc)
1178 {
1179         __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1180 }
1181
1182 struct ring_buffer_event *
1183 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1184                                   int type, unsigned long len,
1185                                   unsigned long flags, int pc)
1186 {
1187         *current_rb = global_trace.buffer;
1188         return trace_buffer_lock_reserve(*current_rb,
1189                                          type, len, flags, pc);
1190 }
1191 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1192
1193 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1194                                         struct ring_buffer_event *event,
1195                                         unsigned long flags, int pc)
1196 {
1197         __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1198 }
1199 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1200
1201 void trace_nowake_buffer_unlock_commit(struct ring_buffer *buffer,
1202                                        struct ring_buffer_event *event,
1203                                        unsigned long flags, int pc)
1204 {
1205         __trace_buffer_unlock_commit(buffer, event, flags, pc, 0);
1206 }
1207 EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
1208
1209 void trace_nowake_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1210                                             struct ring_buffer_event *event,
1211                                             unsigned long flags, int pc,
1212                                             struct pt_regs *regs)
1213 {
1214         ring_buffer_unlock_commit(buffer, event);
1215
1216         ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1217         ftrace_trace_userstack(buffer, flags, pc);
1218 }
1219 EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit_regs);
1220
1221 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1222                                          struct ring_buffer_event *event)
1223 {
1224         ring_buffer_discard_commit(buffer, event);
1225 }
1226 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1227
1228 void
1229 trace_function(struct trace_array *tr,
1230                unsigned long ip, unsigned long parent_ip, unsigned long flags,
1231                int pc)
1232 {
1233         struct ftrace_event_call *call = &event_function;
1234         struct ring_buffer *buffer = tr->buffer;
1235         struct ring_buffer_event *event;
1236         struct ftrace_entry *entry;
1237
1238         /* If we are reading the ring buffer, don't trace */
1239         if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1240                 return;
1241
1242         event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1243                                           flags, pc);
1244         if (!event)
1245                 return;
1246         entry   = ring_buffer_event_data(event);
1247         entry->ip                       = ip;
1248         entry->parent_ip                = parent_ip;
1249
1250         if (!filter_check_discard(call, entry, buffer, event))
1251                 ring_buffer_unlock_commit(buffer, event);
1252 }
1253
1254 void
1255 ftrace(struct trace_array *tr, struct trace_array_cpu *data,
1256        unsigned long ip, unsigned long parent_ip, unsigned long flags,
1257        int pc)
1258 {
1259         if (likely(!atomic_read(&data->disabled)))
1260                 trace_function(tr, ip, parent_ip, flags, pc);
1261 }
1262
1263 #ifdef CONFIG_STACKTRACE
1264
1265 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1266 struct ftrace_stack {
1267         unsigned long           calls[FTRACE_STACK_MAX_ENTRIES];
1268 };
1269
1270 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1271 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1272
1273 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1274                                  unsigned long flags,
1275                                  int skip, int pc, struct pt_regs *regs)
1276 {
1277         struct ftrace_event_call *call = &event_kernel_stack;
1278         struct ring_buffer_event *event;
1279         struct stack_entry *entry;
1280         struct stack_trace trace;
1281         int use_stack;
1282         int size = FTRACE_STACK_ENTRIES;
1283
1284         trace.nr_entries        = 0;
1285         trace.skip              = skip;
1286
1287         /*
1288          * Since events can happen in NMIs there's no safe way to
1289          * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1290          * or NMI comes in, it will just have to use the default
1291          * FTRACE_STACK_SIZE.
1292          */
1293         preempt_disable_notrace();
1294
1295         use_stack = ++__get_cpu_var(ftrace_stack_reserve);
1296         /*
1297          * We don't need any atomic variables, just a barrier.
1298          * If an interrupt comes in, we don't care, because it would
1299          * have exited and put the counter back to what we want.
1300          * We just need a barrier to keep gcc from moving things
1301          * around.
1302          */
1303         barrier();
1304         if (use_stack == 1) {
1305                 trace.entries           = &__get_cpu_var(ftrace_stack).calls[0];
1306                 trace.max_entries       = FTRACE_STACK_MAX_ENTRIES;
1307
1308                 if (regs)
1309                         save_stack_trace_regs(regs, &trace);
1310                 else
1311                         save_stack_trace(&trace);
1312
1313                 if (trace.nr_entries > size)
1314                         size = trace.nr_entries;
1315         } else
1316                 /* From now on, use_stack is a boolean */
1317                 use_stack = 0;
1318
1319         size *= sizeof(unsigned long);
1320
1321         event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1322                                           sizeof(*entry) + size, flags, pc);
1323         if (!event)
1324                 goto out;
1325         entry = ring_buffer_event_data(event);
1326
1327         memset(&entry->caller, 0, size);
1328
1329         if (use_stack)
1330                 memcpy(&entry->caller, trace.entries,
1331                        trace.nr_entries * sizeof(unsigned long));
1332         else {
1333                 trace.max_entries       = FTRACE_STACK_ENTRIES;
1334                 trace.entries           = entry->caller;
1335                 if (regs)
1336                         save_stack_trace_regs(regs, &trace);
1337                 else
1338                         save_stack_trace(&trace);
1339         }
1340
1341         entry->size = trace.nr_entries;
1342
1343         if (!filter_check_discard(call, entry, buffer, event))
1344                 ring_buffer_unlock_commit(buffer, event);
1345
1346  out:
1347         /* Again, don't let gcc optimize things here */
1348         barrier();
1349         __get_cpu_var(ftrace_stack_reserve)--;
1350         preempt_enable_notrace();
1351
1352 }
1353
1354 void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1355                              int skip, int pc, struct pt_regs *regs)
1356 {
1357         if (!(trace_flags & TRACE_ITER_STACKTRACE))
1358                 return;
1359
1360         __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1361 }
1362
1363 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1364                         int skip, int pc)
1365 {
1366         if (!(trace_flags & TRACE_ITER_STACKTRACE))
1367                 return;
1368
1369         __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1370 }
1371
1372 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1373                    int pc)
1374 {
1375         __ftrace_trace_stack(tr->buffer, flags, skip, pc, NULL);
1376 }
1377
1378 /**
1379  * trace_dump_stack - record a stack back trace in the trace buffer
1380  */
1381 void trace_dump_stack(void)
1382 {
1383         unsigned long flags;
1384
1385         if (tracing_disabled || tracing_selftest_running)
1386                 return;
1387
1388         local_save_flags(flags);
1389
1390         /* skipping 3 traces, seems to get us at the caller of this function */
1391         __ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count(), NULL);
1392 }
1393
1394 static DEFINE_PER_CPU(int, user_stack_count);
1395
1396 void
1397 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1398 {
1399         struct ftrace_event_call *call = &event_user_stack;
1400         struct ring_buffer_event *event;
1401         struct userstack_entry *entry;
1402         struct stack_trace trace;
1403
1404         if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1405                 return;
1406
1407         /*
1408          * NMIs can not handle page faults, even with fix ups.
1409          * The save user stack can (and often does) fault.
1410          */
1411         if (unlikely(in_nmi()))
1412                 return;
1413
1414         /*
1415          * prevent recursion, since the user stack tracing may
1416          * trigger other kernel events.
1417          */
1418         preempt_disable();
1419         if (__this_cpu_read(user_stack_count))
1420                 goto out;
1421
1422         __this_cpu_inc(user_stack_count);
1423
1424         event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1425                                           sizeof(*entry), flags, pc);
1426         if (!event)
1427                 goto out_drop_count;
1428         entry   = ring_buffer_event_data(event);
1429
1430         entry->tgid             = current->tgid;
1431         memset(&entry->caller, 0, sizeof(entry->caller));
1432
1433         trace.nr_entries        = 0;
1434         trace.max_entries       = FTRACE_STACK_ENTRIES;
1435         trace.skip              = 0;
1436         trace.entries           = entry->caller;
1437
1438         save_stack_trace_user(&trace);
1439         if (!filter_check_discard(call, entry, buffer, event))
1440                 ring_buffer_unlock_commit(buffer, event);
1441
1442  out_drop_count:
1443         __this_cpu_dec(user_stack_count);
1444  out:
1445         preempt_enable();
1446 }
1447
1448 #ifdef UNUSED
1449 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1450 {
1451         ftrace_trace_userstack(tr, flags, preempt_count());
1452 }
1453 #endif /* UNUSED */
1454
1455 #endif /* CONFIG_STACKTRACE */
1456
1457 /**
1458  * trace_vbprintk - write binary msg to tracing buffer
1459  *
1460  */
1461 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1462 {
1463         static arch_spinlock_t trace_buf_lock =
1464                 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
1465         static u32 trace_buf[TRACE_BUF_SIZE];
1466
1467         struct ftrace_event_call *call = &event_bprint;
1468         struct ring_buffer_event *event;
1469         struct ring_buffer *buffer;
1470         struct trace_array *tr = &global_trace;
1471         struct trace_array_cpu *data;
1472         struct bprint_entry *entry;
1473         unsigned long flags;
1474         int disable;
1475         int cpu, len = 0, size, pc;
1476
1477         if (unlikely(tracing_selftest_running || tracing_disabled))
1478                 return 0;
1479
1480         /* Don't pollute graph traces with trace_vprintk internals */
1481         pause_graph_tracing();
1482
1483         pc = preempt_count();
1484         preempt_disable_notrace();
1485         cpu = raw_smp_processor_id();
1486         data = tr->data[cpu];
1487
1488         disable = atomic_inc_return(&data->disabled);
1489         if (unlikely(disable != 1))
1490                 goto out;
1491
1492         /* Lockdep uses trace_printk for lock tracing */
1493         local_irq_save(flags);
1494         arch_spin_lock(&trace_buf_lock);
1495         len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1496
1497         if (len > TRACE_BUF_SIZE || len < 0)
1498                 goto out_unlock;
1499
1500         size = sizeof(*entry) + sizeof(u32) * len;
1501         buffer = tr->buffer;
1502         event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1503                                           flags, pc);
1504         if (!event)
1505                 goto out_unlock;
1506         entry = ring_buffer_event_data(event);
1507         entry->ip                       = ip;
1508         entry->fmt                      = fmt;
1509
1510         memcpy(entry->buf, trace_buf, sizeof(u32) * len);
1511         if (!filter_check_discard(call, entry, buffer, event)) {
1512                 ring_buffer_unlock_commit(buffer, event);
1513                 ftrace_trace_stack(buffer, flags, 6, pc);
1514         }
1515
1516 out_unlock:
1517         arch_spin_unlock(&trace_buf_lock);
1518         local_irq_restore(flags);
1519
1520 out:
1521         atomic_dec_return(&data->disabled);
1522         preempt_enable_notrace();
1523         unpause_graph_tracing();
1524
1525         return len;
1526 }
1527 EXPORT_SYMBOL_GPL(trace_vbprintk);
1528
1529 int trace_array_printk(struct trace_array *tr,
1530                        unsigned long ip, const char *fmt, ...)
1531 {
1532         int ret;
1533         va_list ap;
1534
1535         if (!(trace_flags & TRACE_ITER_PRINTK))
1536                 return 0;
1537
1538         va_start(ap, fmt);
1539         ret = trace_array_vprintk(tr, ip, fmt, ap);
1540         va_end(ap);
1541         return ret;
1542 }
1543
1544 int trace_array_vprintk(struct trace_array *tr,
1545                         unsigned long ip, const char *fmt, va_list args)
1546 {
1547         static arch_spinlock_t trace_buf_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1548         static char trace_buf[TRACE_BUF_SIZE];
1549
1550         struct ftrace_event_call *call = &event_print;
1551         struct ring_buffer_event *event;
1552         struct ring_buffer *buffer;
1553         struct trace_array_cpu *data;
1554         int cpu, len = 0, size, pc;
1555         struct print_entry *entry;
1556         unsigned long irq_flags;
1557         int disable;
1558
1559         if (tracing_disabled || tracing_selftest_running)
1560                 return 0;
1561
1562         pc = preempt_count();
1563         preempt_disable_notrace();
1564         cpu = raw_smp_processor_id();
1565         data = tr->data[cpu];
1566
1567         disable = atomic_inc_return(&data->disabled);
1568         if (unlikely(disable != 1))
1569                 goto out;
1570
1571         pause_graph_tracing();
1572         raw_local_irq_save(irq_flags);
1573         arch_spin_lock(&trace_buf_lock);
1574         len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1575
1576         size = sizeof(*entry) + len + 1;
1577         buffer = tr->buffer;
1578         event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1579                                           irq_flags, pc);
1580         if (!event)
1581                 goto out_unlock;
1582         entry = ring_buffer_event_data(event);
1583         entry->ip = ip;
1584
1585         memcpy(&entry->buf, trace_buf, len);
1586         entry->buf[len] = '\0';
1587         if (!filter_check_discard(call, entry, buffer, event)) {
1588                 ring_buffer_unlock_commit(buffer, event);
1589                 ftrace_trace_stack(buffer, irq_flags, 6, pc);
1590         }
1591
1592  out_unlock:
1593         arch_spin_unlock(&trace_buf_lock);
1594         raw_local_irq_restore(irq_flags);
1595         unpause_graph_tracing();
1596  out:
1597         atomic_dec_return(&data->disabled);
1598         preempt_enable_notrace();
1599
1600         return len;
1601 }
1602
1603 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1604 {
1605         return trace_array_vprintk(&global_trace, ip, fmt, args);
1606 }
1607 EXPORT_SYMBOL_GPL(trace_vprintk);
1608
1609 static void trace_iterator_increment(struct trace_iterator *iter)
1610 {
1611         /* Don't allow ftrace to trace into the ring buffers */
1612         ftrace_disable_cpu();
1613
1614         iter->idx++;
1615         if (iter->buffer_iter[iter->cpu])
1616                 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1617
1618         ftrace_enable_cpu();
1619 }
1620
1621 static struct trace_entry *
1622 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
1623                 unsigned long *lost_events)
1624 {
1625         struct ring_buffer_event *event;
1626         struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
1627
1628         /* Don't allow ftrace to trace into the ring buffers */
1629         ftrace_disable_cpu();
1630
1631         if (buf_iter)
1632                 event = ring_buffer_iter_peek(buf_iter, ts);
1633         else
1634                 event = ring_buffer_peek(iter->tr->buffer, cpu, ts,
1635                                          lost_events);
1636
1637         ftrace_enable_cpu();
1638
1639         if (event) {
1640                 iter->ent_size = ring_buffer_event_length(event);
1641                 return ring_buffer_event_data(event);
1642         }
1643         iter->ent_size = 0;
1644         return NULL;
1645 }
1646
1647 static struct trace_entry *
1648 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
1649                   unsigned long *missing_events, u64 *ent_ts)
1650 {
1651         struct ring_buffer *buffer = iter->tr->buffer;
1652         struct trace_entry *ent, *next = NULL;
1653         unsigned long lost_events = 0, next_lost = 0;
1654         int cpu_file = iter->cpu_file;
1655         u64 next_ts = 0, ts;
1656         int next_cpu = -1;
1657         int next_size = 0;
1658         int cpu;
1659
1660         /*
1661          * If we are in a per_cpu trace file, don't bother by iterating over
1662          * all cpu and peek directly.
1663          */
1664         if (cpu_file > TRACE_PIPE_ALL_CPU) {
1665                 if (ring_buffer_empty_cpu(buffer, cpu_file))
1666                         return NULL;
1667                 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
1668                 if (ent_cpu)
1669                         *ent_cpu = cpu_file;
1670
1671                 return ent;
1672         }
1673
1674         for_each_tracing_cpu(cpu) {
1675
1676                 if (ring_buffer_empty_cpu(buffer, cpu))
1677                         continue;
1678
1679                 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
1680
1681                 /*
1682                  * Pick the entry with the smallest timestamp:
1683                  */
1684                 if (ent && (!next || ts < next_ts)) {
1685                         next = ent;
1686                         next_cpu = cpu;
1687                         next_ts = ts;
1688                         next_lost = lost_events;
1689                         next_size = iter->ent_size;
1690                 }
1691         }
1692
1693         iter->ent_size = next_size;
1694
1695         if (ent_cpu)
1696                 *ent_cpu = next_cpu;
1697
1698         if (ent_ts)
1699                 *ent_ts = next_ts;
1700
1701         if (missing_events)
1702                 *missing_events = next_lost;
1703
1704         return next;
1705 }
1706
1707 /* Find the next real entry, without updating the iterator itself */
1708 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1709                                           int *ent_cpu, u64 *ent_ts)
1710 {
1711         return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
1712 }
1713
1714 /* Find the next real entry, and increment the iterator to the next entry */
1715 void *trace_find_next_entry_inc(struct trace_iterator *iter)
1716 {
1717         iter->ent = __find_next_entry(iter, &iter->cpu,
1718                                       &iter->lost_events, &iter->ts);
1719
1720         if (iter->ent)
1721                 trace_iterator_increment(iter);
1722
1723         return iter->ent ? iter : NULL;
1724 }
1725
1726 static void trace_consume(struct trace_iterator *iter)
1727 {
1728         /* Don't allow ftrace to trace into the ring buffers */
1729         ftrace_disable_cpu();
1730         ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts,
1731                             &iter->lost_events);
1732         ftrace_enable_cpu();
1733 }
1734
1735 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1736 {
1737         struct trace_iterator *iter = m->private;
1738         int i = (int)*pos;
1739         void *ent;
1740
1741         WARN_ON_ONCE(iter->leftover);
1742
1743         (*pos)++;
1744
1745         /* can't go backwards */
1746         if (iter->idx > i)
1747                 return NULL;
1748
1749         if (iter->idx < 0)
1750                 ent = trace_find_next_entry_inc(iter);
1751         else
1752                 ent = iter;
1753
1754         while (ent && iter->idx < i)
1755                 ent = trace_find_next_entry_inc(iter);
1756
1757         iter->pos = *pos;
1758
1759         return ent;
1760 }
1761
1762 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
1763 {
1764         struct trace_array *tr = iter->tr;
1765         struct ring_buffer_event *event;
1766         struct ring_buffer_iter *buf_iter;
1767         unsigned long entries = 0;
1768         u64 ts;
1769
1770         tr->data[cpu]->skipped_entries = 0;
1771
1772         if (!iter->buffer_iter[cpu])
1773                 return;
1774
1775         buf_iter = iter->buffer_iter[cpu];
1776         ring_buffer_iter_reset(buf_iter);
1777
1778         /*
1779          * We could have the case with the max latency tracers
1780          * that a reset never took place on a cpu. This is evident
1781          * by the timestamp being before the start of the buffer.
1782          */
1783         while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
1784                 if (ts >= iter->tr->time_start)
1785                         break;
1786                 entries++;
1787                 ring_buffer_read(buf_iter, NULL);
1788         }
1789
1790         tr->data[cpu]->skipped_entries = entries;
1791 }
1792
1793 /*
1794  * The current tracer is copied to avoid a global locking
1795  * all around.
1796  */
1797 static void *s_start(struct seq_file *m, loff_t *pos)
1798 {
1799         struct trace_iterator *iter = m->private;
1800         static struct tracer *old_tracer;
1801         int cpu_file = iter->cpu_file;
1802         void *p = NULL;
1803         loff_t l = 0;
1804         int cpu;
1805
1806         /* copy the tracer to avoid using a global lock all around */
1807         mutex_lock(&trace_types_lock);
1808         if (unlikely(old_tracer != current_trace && current_trace)) {
1809                 old_tracer = current_trace;
1810                 *iter->trace = *current_trace;
1811         }
1812         mutex_unlock(&trace_types_lock);
1813
1814         atomic_inc(&trace_record_cmdline_disabled);
1815
1816         if (*pos != iter->pos) {
1817                 iter->ent = NULL;
1818                 iter->cpu = 0;
1819                 iter->idx = -1;
1820
1821                 ftrace_disable_cpu();
1822
1823                 if (cpu_file == TRACE_PIPE_ALL_CPU) {
1824                         for_each_tracing_cpu(cpu)
1825                                 tracing_iter_reset(iter, cpu);
1826                 } else
1827                         tracing_iter_reset(iter, cpu_file);
1828
1829                 ftrace_enable_cpu();
1830
1831                 iter->leftover = 0;
1832                 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1833                         ;
1834
1835         } else {
1836                 /*
1837                  * If we overflowed the seq_file before, then we want
1838                  * to just reuse the trace_seq buffer again.
1839                  */
1840                 if (iter->leftover)
1841                         p = iter;
1842                 else {
1843                         l = *pos - 1;
1844                         p = s_next(m, p, &l);
1845                 }
1846         }
1847
1848         trace_event_read_lock();
1849         trace_access_lock(cpu_file);
1850         return p;
1851 }
1852
1853 static void s_stop(struct seq_file *m, void *p)
1854 {
1855         struct trace_iterator *iter = m->private;
1856
1857         atomic_dec(&trace_record_cmdline_disabled);
1858         trace_access_unlock(iter->cpu_file);
1859         trace_event_read_unlock();
1860 }
1861
1862 static void print_lat_help_header(struct seq_file *m)
1863 {
1864         seq_puts(m, "#                  _------=> CPU#            \n");
1865         seq_puts(m, "#                 / _-----=> irqs-off        \n");
1866         seq_puts(m, "#                | / _----=> need-resched    \n");
1867         seq_puts(m, "#                || / _---=> hardirq/softirq \n");
1868         seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
1869         seq_puts(m, "#                |||| /     delay             \n");
1870         seq_puts(m, "#  cmd     pid   ||||| time  |   caller      \n");
1871         seq_puts(m, "#     \\   /      |||||  \\    |   /           \n");
1872 }
1873
1874 static void print_func_help_header(struct seq_file *m)
1875 {
1876         seq_puts(m, "#           TASK-PID    CPU#    TIMESTAMP  FUNCTION\n");
1877         seq_puts(m, "#              | |       |          |         |\n");
1878 }
1879
1880
1881 void
1882 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1883 {
1884         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1885         struct trace_array *tr = iter->tr;
1886         struct trace_array_cpu *data = tr->data[tr->cpu];
1887         struct tracer *type = current_trace;
1888         unsigned long entries = 0;
1889         unsigned long total = 0;
1890         unsigned long count;
1891         const char *name = "preemption";
1892         int cpu;
1893
1894         if (type)
1895                 name = type->name;
1896
1897
1898         for_each_tracing_cpu(cpu) {
1899                 count = ring_buffer_entries_cpu(tr->buffer, cpu);
1900                 /*
1901                  * If this buffer has skipped entries, then we hold all
1902                  * entries for the trace and we need to ignore the
1903                  * ones before the time stamp.
1904                  */
1905                 if (tr->data[cpu]->skipped_entries) {
1906                         count -= tr->data[cpu]->skipped_entries;
1907                         /* total is the same as the entries */
1908                         total += count;
1909                 } else
1910                         total += count +
1911                                 ring_buffer_overrun_cpu(tr->buffer, cpu);
1912                 entries += count;
1913         }
1914
1915         seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
1916                    name, UTS_RELEASE);
1917         seq_puts(m, "# -----------------------------------"
1918                  "---------------------------------\n");
1919         seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
1920                    " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
1921                    nsecs_to_usecs(data->saved_latency),
1922                    entries,
1923                    total,
1924                    tr->cpu,
1925 #if defined(CONFIG_PREEMPT_NONE)
1926                    "server",
1927 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
1928                    "desktop",
1929 #elif defined(CONFIG_PREEMPT)
1930                    "preempt",
1931 #else
1932                    "unknown",
1933 #endif
1934                    /* These are reserved for later use */
1935                    0, 0, 0, 0);
1936 #ifdef CONFIG_SMP
1937         seq_printf(m, " #P:%d)\n", num_online_cpus());
1938 #else
1939         seq_puts(m, ")\n");
1940 #endif
1941         seq_puts(m, "#    -----------------\n");
1942         seq_printf(m, "#    | task: %.16s-%d "
1943                    "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1944                    data->comm, data->pid, data->uid, data->nice,
1945                    data->policy, data->rt_priority);
1946         seq_puts(m, "#    -----------------\n");
1947
1948         if (data->critical_start) {
1949                 seq_puts(m, "#  => started at: ");
1950                 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1951                 trace_print_seq(m, &iter->seq);
1952                 seq_puts(m, "\n#  => ended at:   ");
1953                 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1954                 trace_print_seq(m, &iter->seq);
1955                 seq_puts(m, "\n#\n");
1956         }
1957
1958         seq_puts(m, "#\n");
1959 }
1960
1961 static void test_cpu_buff_start(struct trace_iterator *iter)
1962 {
1963         struct trace_seq *s = &iter->seq;
1964
1965         if (!(trace_flags & TRACE_ITER_ANNOTATE))
1966                 return;
1967
1968         if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
1969                 return;
1970
1971         if (cpumask_test_cpu(iter->cpu, iter->started))
1972                 return;
1973
1974         if (iter->tr->data[iter->cpu]->skipped_entries)
1975                 return;
1976
1977         cpumask_set_cpu(iter->cpu, iter->started);
1978
1979         /* Don't print started cpu buffer for the first entry of the trace */
1980         if (iter->idx > 1)
1981                 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
1982                                 iter->cpu);
1983 }
1984
1985 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1986 {
1987         struct trace_seq *s = &iter->seq;
1988         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1989         struct trace_entry *entry;
1990         struct trace_event *event;
1991
1992         entry = iter->ent;
1993
1994         test_cpu_buff_start(iter);
1995
1996         event = ftrace_find_event(entry->type);
1997
1998         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1999                 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2000                         if (!trace_print_lat_context(iter))
2001                                 goto partial;
2002                 } else {
2003                         if (!trace_print_context(iter))
2004                                 goto partial;
2005                 }
2006         }
2007
2008         if (event)
2009                 return event->funcs->trace(iter, sym_flags, event);
2010
2011         if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2012                 goto partial;
2013
2014         return TRACE_TYPE_HANDLED;
2015 partial:
2016         return TRACE_TYPE_PARTIAL_LINE;
2017 }
2018
2019 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2020 {
2021         struct trace_seq *s = &iter->seq;
2022         struct trace_entry *entry;
2023         struct trace_event *event;
2024
2025         entry = iter->ent;
2026
2027         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2028                 if (!trace_seq_printf(s, "%d %d %llu ",
2029                                       entry->pid, iter->cpu, iter->ts))
2030                         goto partial;
2031         }
2032
2033         event = ftrace_find_event(entry->type);
2034         if (event)
2035                 return event->funcs->raw(iter, 0, event);
2036
2037         if (!trace_seq_printf(s, "%d ?\n", entry->type))
2038                 goto partial;
2039
2040         return TRACE_TYPE_HANDLED;
2041 partial:
2042         return TRACE_TYPE_PARTIAL_LINE;
2043 }
2044
2045 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2046 {
2047         struct trace_seq *s = &iter->seq;
2048         unsigned char newline = '\n';
2049         struct trace_entry *entry;
2050         struct trace_event *event;
2051
2052         entry = iter->ent;
2053
2054         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2055                 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2056                 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2057                 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2058         }
2059
2060         event = ftrace_find_event(entry->type);
2061         if (event) {
2062                 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2063                 if (ret != TRACE_TYPE_HANDLED)
2064                         return ret;
2065         }
2066
2067         SEQ_PUT_FIELD_RET(s, newline);
2068
2069         return TRACE_TYPE_HANDLED;
2070 }
2071
2072 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2073 {
2074         struct trace_seq *s = &iter->seq;
2075         struct trace_entry *entry;
2076         struct trace_event *event;
2077
2078         entry = iter->ent;
2079
2080         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2081                 SEQ_PUT_FIELD_RET(s, entry->pid);
2082                 SEQ_PUT_FIELD_RET(s, iter->cpu);
2083                 SEQ_PUT_FIELD_RET(s, iter->ts);
2084         }
2085
2086         event = ftrace_find_event(entry->type);
2087         return event ? event->funcs->binary(iter, 0, event) :
2088                 TRACE_TYPE_HANDLED;
2089 }
2090
2091 int trace_empty(struct trace_iterator *iter)
2092 {
2093         int cpu;
2094
2095         /* If we are looking at one CPU buffer, only check that one */
2096         if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
2097                 cpu = iter->cpu_file;
2098                 if (iter->buffer_iter[cpu]) {
2099                         if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
2100                                 return 0;
2101                 } else {
2102                         if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2103                                 return 0;
2104                 }
2105                 return 1;
2106         }
2107
2108         for_each_tracing_cpu(cpu) {
2109                 if (iter->buffer_iter[cpu]) {
2110                         if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
2111                                 return 0;
2112                 } else {
2113                         if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2114                                 return 0;
2115                 }
2116         }
2117
2118         return 1;
2119 }
2120
2121 /*  Called with trace_event_read_lock() held. */
2122 enum print_line_t print_trace_line(struct trace_iterator *iter)
2123 {
2124         enum print_line_t ret;
2125
2126         if (iter->lost_events &&
2127             !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2128                                  iter->cpu, iter->lost_events))
2129                 return TRACE_TYPE_PARTIAL_LINE;
2130
2131         if (iter->trace && iter->trace->print_line) {
2132                 ret = iter->trace->print_line(iter);
2133                 if (ret != TRACE_TYPE_UNHANDLED)
2134                         return ret;
2135         }
2136
2137         if (iter->ent->type == TRACE_BPRINT &&
2138                         trace_flags & TRACE_ITER_PRINTK &&
2139                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2140                 return trace_print_bprintk_msg_only(iter);
2141
2142         if (iter->ent->type == TRACE_PRINT &&
2143                         trace_flags & TRACE_ITER_PRINTK &&
2144                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2145                 return trace_print_printk_msg_only(iter);
2146
2147         if (trace_flags & TRACE_ITER_BIN)
2148                 return print_bin_fmt(iter);
2149
2150         if (trace_flags & TRACE_ITER_HEX)
2151                 return print_hex_fmt(iter);
2152
2153         if (trace_flags & TRACE_ITER_RAW)
2154                 return print_raw_fmt(iter);
2155
2156         return print_trace_fmt(iter);
2157 }
2158
2159 void trace_default_header(struct seq_file *m)
2160 {
2161         struct trace_iterator *iter = m->private;
2162
2163         if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2164                 return;
2165
2166         if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2167                 /* print nothing if the buffers are empty */
2168                 if (trace_empty(iter))
2169                         return;
2170                 print_trace_header(m, iter);
2171                 if (!(trace_flags & TRACE_ITER_VERBOSE))
2172                         print_lat_help_header(m);
2173         } else {
2174                 if (!(trace_flags & TRACE_ITER_VERBOSE))
2175                         print_func_help_header(m);
2176         }
2177 }
2178
2179 static void test_ftrace_alive(struct seq_file *m)
2180 {
2181         if (!ftrace_is_dead())
2182                 return;
2183         seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2184         seq_printf(m, "#          MAY BE MISSING FUNCTION EVENTS\n");
2185 }
2186
2187 static int s_show(struct seq_file *m, void *v)
2188 {
2189         struct trace_iterator *iter = v;
2190         int ret;
2191
2192         if (iter->ent == NULL) {
2193                 if (iter->tr) {
2194                         seq_printf(m, "# tracer: %s\n", iter->trace->name);
2195                         seq_puts(m, "#\n");
2196                         test_ftrace_alive(m);
2197                 }
2198                 if (iter->trace && iter->trace->print_header)
2199                         iter->trace->print_header(m);
2200                 else
2201                         trace_default_header(m);
2202
2203         } else if (iter->leftover) {
2204                 /*
2205                  * If we filled the seq_file buffer earlier, we
2206                  * want to just show it now.
2207                  */
2208                 ret = trace_print_seq(m, &iter->seq);
2209
2210                 /* ret should this time be zero, but you never know */
2211                 iter->leftover = ret;
2212
2213         } else {
2214                 print_trace_line(iter);
2215                 ret = trace_print_seq(m, &iter->seq);
2216                 /*
2217                  * If we overflow the seq_file buffer, then it will
2218                  * ask us for this data again at start up.
2219                  * Use that instead.
2220                  *  ret is 0 if seq_file write succeeded.
2221                  *        -1 otherwise.
2222                  */
2223                 iter->leftover = ret;
2224         }
2225
2226         return 0;
2227 }
2228
2229 static const struct seq_operations tracer_seq_ops = {
2230         .start          = s_start,
2231         .next           = s_next,
2232         .stop           = s_stop,
2233         .show           = s_show,
2234 };
2235
2236 static struct trace_iterator *
2237 __tracing_open(struct inode *inode, struct file *file)
2238 {
2239         long cpu_file = (long) inode->i_private;
2240         void *fail_ret = ERR_PTR(-ENOMEM);
2241         struct trace_iterator *iter;
2242         struct seq_file *m;
2243         int cpu, ret;
2244
2245         if (tracing_disabled)
2246                 return ERR_PTR(-ENODEV);
2247
2248         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2249         if (!iter)
2250                 return ERR_PTR(-ENOMEM);
2251
2252         /*
2253          * We make a copy of the current tracer to avoid concurrent
2254          * changes on it while we are reading.
2255          */
2256         mutex_lock(&trace_types_lock);
2257         iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2258         if (!iter->trace)
2259                 goto fail;
2260
2261         if (current_trace)
2262                 *iter->trace = *current_trace;
2263
2264         if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2265                 goto fail;
2266
2267         if (current_trace && current_trace->print_max)
2268                 iter->tr = &max_tr;
2269         else
2270                 iter->tr = &global_trace;
2271         iter->pos = -1;
2272         mutex_init(&iter->mutex);
2273         iter->cpu_file = cpu_file;
2274
2275         /* Notify the tracer early; before we stop tracing. */
2276         if (iter->trace && iter->trace->open)
2277                 iter->trace->open(iter);
2278
2279         /* Annotate start of buffers if we had overruns */
2280         if (ring_buffer_overruns(iter->tr->buffer))
2281                 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2282
2283         /* stop the trace while dumping */
2284         tracing_stop();
2285
2286         if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
2287                 for_each_tracing_cpu(cpu) {
2288                         iter->buffer_iter[cpu] =
2289                                 ring_buffer_read_prepare(iter->tr->buffer, cpu);
2290                 }
2291                 ring_buffer_read_prepare_sync();
2292                 for_each_tracing_cpu(cpu) {
2293                         ring_buffer_read_start(iter->buffer_iter[cpu]);
2294                         tracing_iter_reset(iter, cpu);
2295                 }
2296         } else {
2297                 cpu = iter->cpu_file;
2298                 iter->buffer_iter[cpu] =
2299                         ring_buffer_read_prepare(iter->tr->buffer, cpu);
2300                 ring_buffer_read_prepare_sync();
2301                 ring_buffer_read_start(iter->buffer_iter[cpu]);
2302                 tracing_iter_reset(iter, cpu);
2303         }
2304
2305         ret = seq_open(file, &tracer_seq_ops);
2306         if (ret < 0) {
2307                 fail_ret = ERR_PTR(ret);
2308                 goto fail_buffer;
2309         }
2310
2311         m = file->private_data;
2312         m->private = iter;
2313
2314         mutex_unlock(&trace_types_lock);
2315
2316         return iter;
2317
2318  fail_buffer:
2319         for_each_tracing_cpu(cpu) {
2320                 if (iter->buffer_iter[cpu])
2321                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
2322         }
2323         free_cpumask_var(iter->started);
2324         tracing_start();
2325  fail:
2326         mutex_unlock(&trace_types_lock);
2327         kfree(iter->trace);
2328         kfree(iter);
2329
2330         return fail_ret;
2331 }
2332
2333 int tracing_open_generic(struct inode *inode, struct file *filp)
2334 {
2335         if (tracing_disabled)
2336                 return -ENODEV;
2337
2338         filp->private_data = inode->i_private;
2339         return 0;
2340 }
2341
2342 static int tracing_release(struct inode *inode, struct file *file)
2343 {
2344         struct seq_file *m = file->private_data;
2345         struct trace_iterator *iter;
2346         int cpu;
2347
2348         if (!(file->f_mode & FMODE_READ))
2349                 return 0;
2350
2351         iter = m->private;
2352
2353         mutex_lock(&trace_types_lock);
2354         for_each_tracing_cpu(cpu) {
2355                 if (iter->buffer_iter[cpu])
2356                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
2357         }
2358
2359         if (iter->trace && iter->trace->close)
2360                 iter->trace->close(iter);
2361
2362         /* reenable tracing if it was previously enabled */
2363         tracing_start();
2364         mutex_unlock(&trace_types_lock);
2365
2366         seq_release(inode, file);
2367         mutex_destroy(&iter->mutex);
2368         free_cpumask_var(iter->started);
2369         kfree(iter->trace);
2370         kfree(iter);
2371         return 0;
2372 }
2373
2374 static int tracing_open(struct inode *inode, struct file *file)
2375 {
2376         struct trace_iterator *iter;
2377         int ret = 0;
2378
2379         /* If this file was open for write, then erase contents */
2380         if ((file->f_mode & FMODE_WRITE) &&
2381             (file->f_flags & O_TRUNC)) {
2382                 long cpu = (long) inode->i_private;
2383
2384                 if (cpu == TRACE_PIPE_ALL_CPU)
2385                         tracing_reset_online_cpus(&global_trace);
2386                 else
2387                         tracing_reset(&global_trace, cpu);
2388         }
2389
2390         if (file->f_mode & FMODE_READ) {
2391                 iter = __tracing_open(inode, file);
2392                 if (IS_ERR(iter))
2393                         ret = PTR_ERR(iter);
2394                 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2395                         iter->iter_flags |= TRACE_FILE_LAT_FMT;
2396         }
2397         return ret;
2398 }
2399
2400 static void *
2401 t_next(struct seq_file *m, void *v, loff_t *pos)
2402 {
2403         struct tracer *t = v;
2404
2405         (*pos)++;
2406
2407         if (t)
2408                 t = t->next;
2409
2410         return t;
2411 }
2412
2413 static void *t_start(struct seq_file *m, loff_t *pos)
2414 {
2415         struct tracer *t;
2416         loff_t l = 0;
2417
2418         mutex_lock(&trace_types_lock);
2419         for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
2420                 ;
2421
2422         return t;
2423 }
2424
2425 static void t_stop(struct seq_file *m, void *p)
2426 {
2427         mutex_unlock(&trace_types_lock);
2428 }
2429
2430 static int t_show(struct seq_file *m, void *v)
2431 {
2432         struct tracer *t = v;
2433
2434         if (!t)
2435                 return 0;
2436
2437         seq_printf(m, "%s", t->name);
2438         if (t->next)
2439                 seq_putc(m, ' ');
2440         else
2441                 seq_putc(m, '\n');
2442
2443         return 0;
2444 }
2445
2446 static const struct seq_operations show_traces_seq_ops = {
2447         .start          = t_start,
2448         .next           = t_next,
2449         .stop           = t_stop,
2450         .show           = t_show,
2451 };
2452
2453 static int show_traces_open(struct inode *inode, struct file *file)
2454 {
2455         if (tracing_disabled)
2456                 return -ENODEV;
2457
2458         return seq_open(file, &show_traces_seq_ops);
2459 }
2460
2461 static ssize_t
2462 tracing_write_stub(struct file *filp, const char __user *ubuf,
2463                    size_t count, loff_t *ppos)
2464 {
2465         return count;
2466 }
2467
2468 static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
2469 {
2470         if (file->f_mode & FMODE_READ)
2471                 return seq_lseek(file, offset, origin);
2472         else
2473                 return 0;
2474 }
2475
2476 static const struct file_operations tracing_fops = {
2477         .open           = tracing_open,
2478         .read           = seq_read,
2479         .write          = tracing_write_stub,
2480         .llseek         = tracing_seek,
2481         .release        = tracing_release,
2482 };
2483
2484 static const struct file_operations show_traces_fops = {
2485         .open           = show_traces_open,
2486         .read           = seq_read,
2487         .release        = seq_release,
2488         .llseek         = seq_lseek,
2489 };
2490
2491 /*
2492  * Only trace on a CPU if the bitmask is set:
2493  */
2494 static cpumask_var_t tracing_cpumask;
2495
2496 /*
2497  * The tracer itself will not take this lock, but still we want
2498  * to provide a consistent cpumask to user-space:
2499  */
2500 static DEFINE_MUTEX(tracing_cpumask_update_lock);
2501
2502 /*
2503  * Temporary storage for the character representation of the
2504  * CPU bitmask (and one more byte for the newline):
2505  */
2506 static char mask_str[NR_CPUS + 1];
2507
2508 static ssize_t
2509 tracing_cpumask_read(struct file *filp, char __user *ubuf,
2510                      size_t count, loff_t *ppos)
2511 {
2512         int len;
2513
2514         mutex_lock(&tracing_cpumask_update_lock);
2515
2516         len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2517         if (count - len < 2) {
2518                 count = -EINVAL;
2519                 goto out_err;
2520         }
2521         len += sprintf(mask_str + len, "\n");
2522         count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2523
2524 out_err:
2525         mutex_unlock(&tracing_cpumask_update_lock);
2526
2527         return count;
2528 }
2529
2530 static ssize_t
2531 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2532                       size_t count, loff_t *ppos)
2533 {
2534         int err, cpu;
2535         cpumask_var_t tracing_cpumask_new;
2536
2537         if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2538                 return -ENOMEM;
2539
2540         err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2541         if (err)
2542                 goto err_unlock;
2543
2544         mutex_lock(&tracing_cpumask_update_lock);
2545
2546         local_irq_disable();
2547         arch_spin_lock(&ftrace_max_lock);
2548         for_each_tracing_cpu(cpu) {
2549                 /*
2550                  * Increase/decrease the disabled counter if we are
2551                  * about to flip a bit in the cpumask:
2552                  */
2553                 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2554                                 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2555                         atomic_inc(&global_trace.data[cpu]->disabled);
2556                         ring_buffer_record_disable_cpu(global_trace.buffer, cpu);
2557                 }
2558                 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2559                                 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2560                         atomic_dec(&global_trace.data[cpu]->disabled);
2561                         ring_buffer_record_enable_cpu(global_trace.buffer, cpu);
2562                 }
2563         }
2564         arch_spin_unlock(&ftrace_max_lock);
2565         local_irq_enable();
2566
2567         cpumask_copy(tracing_cpumask, tracing_cpumask_new);
2568
2569         mutex_unlock(&tracing_cpumask_update_lock);
2570         free_cpumask_var(tracing_cpumask_new);
2571
2572         return count;
2573
2574 err_unlock:
2575         free_cpumask_var(tracing_cpumask_new);
2576
2577         return err;
2578 }
2579
2580 static const struct file_operations tracing_cpumask_fops = {
2581         .open           = tracing_open_generic,
2582         .read           = tracing_cpumask_read,
2583         .write          = tracing_cpumask_write,
2584         .llseek         = generic_file_llseek,
2585 };
2586
2587 static int tracing_trace_options_show(struct seq_file *m, void *v)
2588 {
2589         struct tracer_opt *trace_opts;
2590         u32 tracer_flags;
2591         int i;
2592
2593         mutex_lock(&trace_types_lock);
2594         tracer_flags = current_trace->flags->val;
2595         trace_opts = current_trace->flags->opts;
2596
2597         for (i = 0; trace_options[i]; i++) {
2598                 if (trace_flags & (1 << i))
2599                         seq_printf(m, "%s\n", trace_options[i]);
2600                 else
2601                         seq_printf(m, "no%s\n", trace_options[i]);
2602         }
2603
2604         for (i = 0; trace_opts[i].name; i++) {
2605                 if (tracer_flags & trace_opts[i].bit)
2606                         seq_printf(m, "%s\n", trace_opts[i].name);
2607                 else
2608                         seq_printf(m, "no%s\n", trace_opts[i].name);
2609         }
2610         mutex_unlock(&trace_types_lock);
2611
2612         return 0;
2613 }
2614
2615 static int __set_tracer_option(struct tracer *trace,
2616                                struct tracer_flags *tracer_flags,
2617                                struct tracer_opt *opts, int neg)
2618 {
2619         int ret;
2620
2621         ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
2622         if (ret)
2623                 return ret;
2624
2625         if (neg)
2626                 tracer_flags->val &= ~opts->bit;
2627         else
2628                 tracer_flags->val |= opts->bit;
2629         return 0;
2630 }
2631
2632 /* Try to assign a tracer specific option */
2633 static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
2634 {
2635         struct tracer_flags *tracer_flags = trace->flags;
2636         struct tracer_opt *opts = NULL;
2637         int i;
2638
2639         for (i = 0; tracer_flags->opts[i].name; i++) {
2640                 opts = &tracer_flags->opts[i];
2641
2642                 if (strcmp(cmp, opts->name) == 0)
2643                         return __set_tracer_option(trace, trace->flags,
2644                                                    opts, neg);
2645         }
2646
2647         return -EINVAL;
2648 }
2649
2650 /* Some tracers require overwrite to stay enabled */
2651 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
2652 {
2653         if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
2654                 return -1;
2655
2656         return 0;
2657 }
2658
2659 int set_tracer_flag(unsigned int mask, int enabled)
2660 {
2661         /* do nothing if flag is already set */
2662         if (!!(trace_flags & mask) == !!enabled)
2663                 return 0;
2664
2665         /* Give the tracer a chance to approve the change */
2666         if (current_trace->flag_changed)
2667                 if (current_trace->flag_changed(current_trace, mask, !!enabled))
2668                         return -EINVAL;
2669
2670         if (enabled)
2671                 trace_flags |= mask;
2672         else
2673                 trace_flags &= ~mask;
2674
2675         if (mask == TRACE_ITER_RECORD_CMD)
2676                 trace_event_enable_cmd_record(enabled);
2677
2678         if (mask == TRACE_ITER_OVERWRITE) {
2679                 ring_buffer_change_overwrite(global_trace.buffer, enabled);
2680 #ifdef CONFIG_TRACER_MAX_TRACE
2681                 ring_buffer_change_overwrite(max_tr.buffer, enabled);
2682 #endif
2683         }
2684
2685         return 0;
2686 }
2687
2688 static ssize_t
2689 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
2690                         size_t cnt, loff_t *ppos)
2691 {
2692         char buf[64];
2693         char *cmp;
2694         int neg = 0;
2695         int ret = 0;
2696         int i;
2697
2698         if (cnt >= sizeof(buf))
2699                 return -EINVAL;
2700
2701         if (copy_from_user(&buf, ubuf, cnt))
2702                 return -EFAULT;
2703
2704         buf[cnt] = 0;
2705         cmp = strstrip(buf);
2706
2707         if (strncmp(cmp, "no", 2) == 0) {
2708                 neg = 1;
2709                 cmp += 2;
2710         }
2711
2712         mutex_lock(&trace_types_lock);
2713
2714         for (i = 0; trace_options[i]; i++) {
2715                 if (strcmp(cmp, trace_options[i]) == 0) {
2716                         ret = set_tracer_flag(1 << i, !neg);
2717                         break;
2718                 }
2719         }
2720
2721         /* If no option could be set, test the specific tracer options */
2722         if (!trace_options[i])
2723                 ret = set_tracer_option(current_trace, cmp, neg);
2724
2725         mutex_unlock(&trace_types_lock);
2726
2727         if (ret)
2728                 return ret;
2729
2730         *ppos += cnt;
2731
2732         return cnt;
2733 }
2734
2735 static int tracing_trace_options_open(struct inode *inode, struct file *file)
2736 {
2737         if (tracing_disabled)
2738                 return -ENODEV;
2739         return single_open(file, tracing_trace_options_show, NULL);
2740 }
2741
2742 static const struct file_operations tracing_iter_fops = {
2743         .open           = tracing_trace_options_open,
2744         .read           = seq_read,
2745         .llseek         = seq_lseek,
2746         .release        = single_release,
2747         .write          = tracing_trace_options_write,
2748 };
2749
2750 static const char readme_msg[] =
2751         "tracing mini-HOWTO:\n\n"
2752         "# mount -t debugfs nodev /sys/kernel/debug\n\n"
2753         "# cat /sys/kernel/debug/tracing/available_tracers\n"
2754         "wakeup preemptirqsoff preemptoff irqsoff function sched_switch nop\n\n"
2755         "# cat /sys/kernel/debug/tracing/current_tracer\n"
2756         "nop\n"
2757         "# echo sched_switch > /sys/kernel/debug/tracing/current_tracer\n"
2758         "# cat /sys/kernel/debug/tracing/current_tracer\n"
2759         "sched_switch\n"
2760         "# cat /sys/kernel/debug/tracing/trace_options\n"
2761         "noprint-parent nosym-offset nosym-addr noverbose\n"
2762         "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
2763         "# echo 1 > /sys/kernel/debug/tracing/tracing_on\n"
2764         "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
2765         "# echo 0 > /sys/kernel/debug/tracing/tracing_on\n"
2766 ;
2767
2768 static ssize_t
2769 tracing_readme_read(struct file *filp, char __user *ubuf,
2770                        size_t cnt, loff_t *ppos)
2771 {
2772         return simple_read_from_buffer(ubuf, cnt, ppos,
2773                                         readme_msg, strlen(readme_msg));
2774 }
2775
2776 static const struct file_operations tracing_readme_fops = {
2777         .open           = tracing_open_generic,
2778         .read           = tracing_readme_read,
2779         .llseek         = generic_file_llseek,
2780 };
2781
2782 static ssize_t
2783 tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
2784                                 size_t cnt, loff_t *ppos)
2785 {
2786         char *buf_comm;
2787         char *file_buf;
2788         char *buf;
2789         int len = 0;
2790         int pid;
2791         int i;
2792
2793         file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
2794         if (!file_buf)
2795                 return -ENOMEM;
2796
2797         buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
2798         if (!buf_comm) {
2799                 kfree(file_buf);
2800                 return -ENOMEM;
2801         }
2802
2803         buf = file_buf;
2804
2805         for (i = 0; i < SAVED_CMDLINES; i++) {
2806                 int r;
2807
2808                 pid = map_cmdline_to_pid[i];
2809                 if (pid == -1 || pid == NO_CMDLINE_MAP)
2810                         continue;
2811
2812                 trace_find_cmdline(pid, buf_comm);
2813                 r = sprintf(buf, "%d %s\n", pid, buf_comm);
2814                 buf += r;
2815                 len += r;
2816         }
2817
2818         len = simple_read_from_buffer(ubuf, cnt, ppos,
2819                                       file_buf, len);
2820
2821         kfree(file_buf);
2822         kfree(buf_comm);
2823
2824         return len;
2825 }
2826
2827 static const struct file_operations tracing_saved_cmdlines_fops = {
2828     .open       = tracing_open_generic,
2829     .read       = tracing_saved_cmdlines_read,
2830     .llseek     = generic_file_llseek,
2831 };
2832
2833 static ssize_t
2834 tracing_ctrl_read(struct file *filp, char __user *ubuf,
2835                   size_t cnt, loff_t *ppos)
2836 {
2837         char buf[64];
2838         int r;
2839
2840         r = sprintf(buf, "%u\n", tracer_enabled);
2841         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2842 }
2843
2844 static ssize_t
2845 tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2846                    size_t cnt, loff_t *ppos)
2847 {
2848         struct trace_array *tr = filp->private_data;
2849         unsigned long val;
2850         int ret;
2851
2852         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
2853         if (ret)
2854                 return ret;
2855
2856         val = !!val;
2857
2858         mutex_lock(&trace_types_lock);
2859         if (tracer_enabled ^ val) {
2860
2861                 /* Only need to warn if this is used to change the state */
2862                 WARN_ONCE(1, "tracing_enabled is deprecated. Use tracing_on");
2863
2864                 if (val) {
2865                         tracer_enabled = 1;
2866                         if (current_trace->start)
2867                                 current_trace->start(tr);
2868                         tracing_start();
2869                 } else {
2870                         tracer_enabled = 0;
2871                         tracing_stop();
2872                         if (current_trace->stop)
2873                                 current_trace->stop(tr);
2874                 }
2875         }
2876         mutex_unlock(&trace_types_lock);
2877
2878         *ppos += cnt;
2879
2880         return cnt;
2881 }
2882
2883 static ssize_t
2884 tracing_set_trace_read(struct file *filp, char __user *ubuf,
2885                        size_t cnt, loff_t *ppos)
2886 {
2887         char buf[MAX_TRACER_SIZE+2];
2888         int r;
2889
2890         mutex_lock(&trace_types_lock);
2891         if (current_trace)
2892                 r = sprintf(buf, "%s\n", current_trace->name);
2893         else
2894                 r = sprintf(buf, "\n");
2895         mutex_unlock(&trace_types_lock);
2896
2897         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2898 }
2899
2900 int tracer_init(struct tracer *t, struct trace_array *tr)
2901 {
2902         tracing_reset_online_cpus(tr);
2903         return t->init(tr);
2904 }
2905
2906 static int __tracing_resize_ring_buffer(unsigned long size)
2907 {
2908         int ret;
2909
2910         /*
2911          * If kernel or user changes the size of the ring buffer
2912          * we use the size that was given, and we can forget about
2913          * expanding it later.
2914          */
2915         ring_buffer_expanded = 1;
2916
2917         ret = ring_buffer_resize(global_trace.buffer, size);
2918         if (ret < 0)
2919                 return ret;
2920
2921         if (!current_trace->use_max_tr)
2922                 goto out;
2923
2924         ret = ring_buffer_resize(max_tr.buffer, size);
2925         if (ret < 0) {
2926                 int r;
2927
2928                 r = ring_buffer_resize(global_trace.buffer,
2929                                        global_trace.entries);
2930                 if (r < 0) {
2931                         /*
2932                          * AARGH! We are left with different
2933                          * size max buffer!!!!
2934                          * The max buffer is our "snapshot" buffer.
2935                          * When a tracer needs a snapshot (one of the
2936                          * latency tracers), it swaps the max buffer
2937                          * with the saved snap shot. We succeeded to
2938                          * update the size of the main buffer, but failed to
2939                          * update the size of the max buffer. But when we tried
2940                          * to reset the main buffer to the original size, we
2941                          * failed there too. This is very unlikely to
2942                          * happen, but if it does, warn and kill all
2943                          * tracing.
2944                          */
2945                         WARN_ON(1);
2946                         tracing_disabled = 1;
2947                 }
2948                 return ret;
2949         }
2950
2951         max_tr.entries = size;
2952  out:
2953         global_trace.entries = size;
2954
2955         return ret;
2956 }
2957
2958 static ssize_t tracing_resize_ring_buffer(unsigned long size)
2959 {
2960         int cpu, ret = size;
2961
2962         mutex_lock(&trace_types_lock);
2963
2964         tracing_stop();
2965
2966         /* disable all cpu buffers */
2967         for_each_tracing_cpu(cpu) {
2968                 if (global_trace.data[cpu])
2969                         atomic_inc(&global_trace.data[cpu]->disabled);
2970                 if (max_tr.data[cpu])
2971                         atomic_inc(&max_tr.data[cpu]->disabled);
2972         }
2973
2974         if (size != global_trace.entries)
2975                 ret = __tracing_resize_ring_buffer(size);
2976
2977         if (ret < 0)
2978                 ret = -ENOMEM;
2979
2980         for_each_tracing_cpu(cpu) {
2981                 if (global_trace.data[cpu])
2982                         atomic_dec(&global_trace.data[cpu]->disabled);
2983                 if (max_tr.data[cpu])
2984                         atomic_dec(&max_tr.data[cpu]->disabled);
2985         }
2986
2987         tracing_start();
2988         mutex_unlock(&trace_types_lock);
2989
2990         return ret;
2991 }
2992
2993
2994 /**
2995  * tracing_update_buffers - used by tracing facility to expand ring buffers
2996  *
2997  * To save on memory when the tracing is never used on a system with it
2998  * configured in. The ring buffers are set to a minimum size. But once
2999  * a user starts to use the tracing facility, then they need to grow
3000  * to their default size.
3001  *
3002  * This function is to be called when a tracer is about to be used.
3003  */
3004 int tracing_update_buffers(void)
3005 {
3006         int ret = 0;
3007
3008         mutex_lock(&trace_types_lock);
3009         if (!ring_buffer_expanded)
3010                 ret = __tracing_resize_ring_buffer(trace_buf_size);
3011         mutex_unlock(&trace_types_lock);
3012
3013         return ret;
3014 }
3015
3016 struct trace_option_dentry;
3017
3018 static struct trace_option_dentry *
3019 create_trace_option_files(struct tracer *tracer);
3020
3021 static void
3022 destroy_trace_option_files(struct trace_option_dentry *topts);
3023
3024 static int tracing_set_tracer(const char *buf)
3025 {
3026         static struct trace_option_dentry *topts;
3027         struct trace_array *tr = &global_trace;
3028         struct tracer *t;
3029         int ret = 0;
3030
3031         mutex_lock(&trace_types_lock);
3032
3033         if (!ring_buffer_expanded) {
3034                 ret = __tracing_resize_ring_buffer(trace_buf_size);
3035                 if (ret < 0)
3036                         goto out;
3037                 ret = 0;
3038         }
3039
3040         for (t = trace_types; t; t = t->next) {
3041                 if (strcmp(t->name, buf) == 0)
3042                         break;
3043         }
3044         if (!t) {
3045                 ret = -EINVAL;
3046                 goto out;
3047         }
3048         if (t == current_trace)
3049                 goto out;
3050
3051         trace_branch_disable();
3052
3053         current_trace->enabled = false;
3054
3055         if (current_trace && current_trace->reset)
3056                 current_trace->reset(tr);
3057         if (current_trace && current_trace->use_max_tr) {
3058                 /*
3059                  * We don't free the ring buffer. instead, resize it because
3060                  * The max_tr ring buffer has some state (e.g. ring->clock) and
3061                  * we want preserve it.
3062                  */
3063                 ring_buffer_resize(max_tr.buffer, 1);
3064                 max_tr.entries = 1;
3065         }
3066         destroy_trace_option_files(topts);
3067
3068         current_trace = t;
3069
3070         topts = create_trace_option_files(current_trace);
3071         if (current_trace->use_max_tr) {
3072                 ret = ring_buffer_resize(max_tr.buffer, global_trace.entries);
3073                 if (ret < 0)
3074                         goto out;
3075                 max_tr.entries = global_trace.entries;
3076         }
3077
3078         if (t->init) {
3079                 ret = tracer_init(t, tr);
3080                 if (ret)
3081                         goto out;
3082         }
3083
3084         current_trace->enabled = true;
3085         trace_branch_enable(tr);
3086  out:
3087         mutex_unlock(&trace_types_lock);
3088
3089         return ret;
3090 }
3091
3092 static ssize_t
3093 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3094                         size_t cnt, loff_t *ppos)
3095 {
3096         char buf[MAX_TRACER_SIZE+1];
3097         int i;
3098         size_t ret;
3099         int err;
3100
3101         ret = cnt;
3102
3103         if (cnt > MAX_TRACER_SIZE)
3104                 cnt = MAX_TRACER_SIZE;
3105
3106         if (copy_from_user(&buf, ubuf, cnt))
3107                 return -EFAULT;
3108
3109         buf[cnt] = 0;
3110
3111         /* strip ending whitespace. */
3112         for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3113                 buf[i] = 0;
3114
3115         err = tracing_set_tracer(buf);
3116         if (err)
3117                 return err;
3118
3119         *ppos += ret;
3120
3121         return ret;
3122 }
3123
3124 static ssize_t
3125 tracing_max_lat_read(struct file *filp, char __user *ubuf,
3126                      size_t cnt, loff_t *ppos)
3127 {
3128         unsigned long *ptr = filp->private_data;
3129         char buf[64];
3130         int r;
3131
3132         r = snprintf(buf, sizeof(buf), "%ld\n",
3133                      *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
3134         if (r > sizeof(buf))
3135                 r = sizeof(buf);
3136         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3137 }
3138
3139 static ssize_t
3140 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3141                       size_t cnt, loff_t *ppos)
3142 {
3143         unsigned long *ptr = filp->private_data;
3144         unsigned long val;
3145         int ret;
3146
3147         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3148         if (ret)
3149                 return ret;
3150
3151         *ptr = val * 1000;
3152
3153         return cnt;
3154 }
3155
3156 static int tracing_open_pipe(struct inode *inode, struct file *filp)
3157 {
3158         long cpu_file = (long) inode->i_private;
3159         struct trace_iterator *iter;
3160         int ret = 0;
3161
3162         if (tracing_disabled)
3163                 return -ENODEV;
3164
3165         mutex_lock(&trace_types_lock);
3166
3167         /* create a buffer to store the information to pass to userspace */
3168         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3169         if (!iter) {
3170                 ret = -ENOMEM;
3171                 goto out;
3172         }
3173
3174         /*
3175          * We make a copy of the current tracer to avoid concurrent
3176          * changes on it while we are reading.
3177          */
3178         iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3179         if (!iter->trace) {
3180                 ret = -ENOMEM;
3181                 goto fail;
3182         }
3183         if (current_trace)
3184                 *iter->trace = *current_trace;
3185
3186         if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3187                 ret = -ENOMEM;
3188                 goto fail;
3189         }
3190
3191         /* trace pipe does not show start of buffer */
3192         cpumask_setall(iter->started);
3193
3194         if (trace_flags & TRACE_ITER_LATENCY_FMT)
3195                 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3196
3197         iter->cpu_file = cpu_file;
3198         iter->tr = &global_trace;
3199         mutex_init(&iter->mutex);
3200         filp->private_data = iter;
3201
3202         if (iter->trace->pipe_open)
3203                 iter->trace->pipe_open(iter);
3204
3205         nonseekable_open(inode, filp);
3206 out:
3207         mutex_unlock(&trace_types_lock);
3208         return ret;
3209
3210 fail:
3211         kfree(iter->trace);
3212         kfree(iter);
3213         mutex_unlock(&trace_types_lock);
3214         return ret;
3215 }
3216
3217 static int tracing_release_pipe(struct inode *inode, struct file *file)
3218 {
3219         struct trace_iterator *iter = file->private_data;
3220
3221         mutex_lock(&trace_types_lock);
3222
3223         if (iter->trace->pipe_close)
3224                 iter->trace->pipe_close(iter);
3225
3226         mutex_unlock(&trace_types_lock);
3227
3228         free_cpumask_var(iter->started);
3229         mutex_destroy(&iter->mutex);
3230         kfree(iter->trace);
3231         kfree(iter);
3232
3233         return 0;
3234 }
3235
3236 static unsigned int
3237 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3238 {
3239         struct trace_iterator *iter = filp->private_data;
3240
3241         if (trace_flags & TRACE_ITER_BLOCK) {
3242                 /*
3243                  * Always select as readable when in blocking mode
3244                  */
3245                 return POLLIN | POLLRDNORM;
3246         } else {
3247                 if (!trace_empty(iter))
3248                         return POLLIN | POLLRDNORM;
3249                 poll_wait(filp, &trace_wait, poll_table);
3250                 if (!trace_empty(iter))
3251                         return POLLIN | POLLRDNORM;
3252
3253                 return 0;
3254         }
3255 }
3256
3257
3258 void default_wait_pipe(struct trace_iterator *iter)
3259 {
3260         DEFINE_WAIT(wait);
3261
3262         prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
3263
3264         if (trace_empty(iter))
3265                 schedule();
3266
3267         finish_wait(&trace_wait, &wait);
3268 }
3269
3270 /*
3271  * This is a make-shift waitqueue.
3272  * A tracer might use this callback on some rare cases:
3273  *
3274  *  1) the current tracer might hold the runqueue lock when it wakes up
3275  *     a reader, hence a deadlock (sched, function, and function graph tracers)
3276  *  2) the function tracers, trace all functions, we don't want
3277  *     the overhead of calling wake_up and friends
3278  *     (and tracing them too)
3279  *
3280  *     Anyway, this is really very primitive wakeup.
3281  */
3282 void poll_wait_pipe(struct trace_iterator *iter)
3283 {
3284         set_current_state(TASK_INTERRUPTIBLE);
3285         /* sleep for 100 msecs, and try again. */
3286         schedule_timeout(HZ / 10);
3287 }
3288
3289 /* Must be called with trace_types_lock mutex held. */
3290 static int tracing_wait_pipe(struct file *filp)
3291 {
3292         struct trace_iterator *iter = filp->private_data;
3293
3294         while (trace_empty(iter)) {
3295
3296                 if ((filp->f_flags & O_NONBLOCK)) {
3297                         return -EAGAIN;
3298                 }
3299
3300                 mutex_unlock(&iter->mutex);
3301
3302                 iter->trace->wait_pipe(iter);
3303
3304                 mutex_lock(&iter->mutex);
3305
3306                 if (signal_pending(current))
3307                         return -EINTR;
3308
3309                 /*
3310                  * We block until we read something and tracing is disabled.
3311                  * We still block if tracing is disabled, but we have never
3312                  * read anything. This allows a user to cat this file, and
3313                  * then enable tracing. But after we have read something,
3314                  * we give an EOF when tracing is again disabled.
3315                  *
3316                  * iter->pos will be 0 if we haven't read anything.
3317                  */
3318                 if (!tracer_enabled && iter->pos)
3319                         break;
3320         }
3321
3322         return 1;
3323 }
3324
3325 /*
3326  * Consumer reader.
3327  */
3328 static ssize_t
3329 tracing_read_pipe(struct file *filp, char __user *ubuf,
3330                   size_t cnt, loff_t *ppos)
3331 {
3332         struct trace_iterator *iter = filp->private_data;
3333         static struct tracer *old_tracer;
3334         ssize_t sret;
3335
3336         /* return any leftover data */
3337         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3338         if (sret != -EBUSY)
3339                 return sret;
3340
3341         trace_seq_init(&iter->seq);
3342
3343         /* copy the tracer to avoid using a global lock all around */
3344         mutex_lock(&trace_types_lock);
3345         if (unlikely(old_tracer != current_trace && current_trace)) {
3346                 old_tracer = current_trace;
3347                 *iter->trace = *current_trace;
3348         }
3349         mutex_unlock(&trace_types_lock);
3350
3351         /*
3352          * Avoid more than one consumer on a single file descriptor
3353          * This is just a matter of traces coherency, the ring buffer itself
3354          * is protected.
3355          */
3356         mutex_lock(&iter->mutex);
3357         if (iter->trace->read) {
3358                 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3359                 if (sret)
3360                         goto out;
3361         }
3362
3363 waitagain:
3364         sret = tracing_wait_pipe(filp);
3365         if (sret <= 0)
3366                 goto out;
3367
3368         /* stop when tracing is finished */
3369         if (trace_empty(iter)) {
3370                 sret = 0;
3371                 goto out;
3372         }
3373
3374         if (cnt >= PAGE_SIZE)
3375                 cnt = PAGE_SIZE - 1;
3376
3377         /* reset all but tr, trace, and overruns */
3378         memset(&iter->seq, 0,
3379                sizeof(struct trace_iterator) -
3380                offsetof(struct trace_iterator, seq));
3381         cpumask_clear(iter->started);
3382         iter->pos = -1;
3383
3384         trace_event_read_lock();