kernel/trace/ftrace.c

   1 /*
   2  * Infrastructure for profiling code inserted by 'gcc -pg'.
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Originally ported from the -rt patch by:
   8  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
   9  *
  10  * Based on code in the latency_tracer, that is:
  11  *
  12  *  Copyright (C) 2004-2006 Ingo Molnar
  13  *  Copyright (C) 2004 William Lee Irwin III
  14  */
  15
  16 #include <linux/stop_machine.h>
  17 #include <linux/clocksource.h>
  18 #include <linux/kallsyms.h>
  19 #include <linux/seq_file.h>
  20 #include <linux/suspend.h>
  21 #include <linux/debugfs.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/kthread.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/module.h>
  26 #include <linux/ftrace.h>
  27 #include <linux/sysctl.h>
  28 #include <linux/slab.h>
  29 #include <linux/ctype.h>
  30 #include <linux/list.h>
  31 #include <linux/hash.h>
  32 #include <linux/rcupdate.h>
  33
  34 #include <trace/events/sched.h>
  35
  36 #include <asm/setup.h>
  37
  38 #include "trace_output.h"
  39 #include "trace_stat.h"
  40
  41 #define FTRACE_WARN_ON(cond)                    \
  42         ({                                      \
  43                 int ___r = cond;                \
  44                 if (WARN_ON(___r))              \
  45                         ftrace_kill();          \
  46                 ___r;                           \
  47         })
  48
  49 #define FTRACE_WARN_ON_ONCE(cond)               \
  50         ({                                      \
  51                 int ___r = cond;                \
  52                 if (WARN_ON_ONCE(___r))         \
  53                         ftrace_kill();          \
  54                 ___r;                           \
  55         })
  56
  57 /* hash bits for specific function selection */
  58 #define FTRACE_HASH_BITS 7
  59 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
  60 #define FTRACE_HASH_DEFAULT_BITS 10
  61 #define FTRACE_HASH_MAX_BITS 12
  62
  63 /* ftrace_enabled is a method to turn ftrace on or off */
  64 int ftrace_enabled __read_mostly;
  65 static int last_ftrace_enabled;
  66
  67 /* Quick disabling of function tracer. */
  68 int function_trace_stop;
  69
  70 /* List for set_ftrace_pid's pids. */
  71 LIST_HEAD(ftrace_pids);
  72 struct ftrace_pid {
  73         struct list_head list;
  74         struct pid *pid;
  75 };
  76
  77 /*
  78  * ftrace_disabled is set when an anomaly is discovered.
  79  * ftrace_disabled is much stronger than ftrace_enabled.
  80  */
  81 static int ftrace_disabled __read_mostly;
  82
  83 static DEFINE_MUTEX(ftrace_lock);
  84
  85 static struct ftrace_ops ftrace_list_end __read_mostly = {
  86         .func           = ftrace_stub,
  87 };
  88
  89 static struct ftrace_ops *ftrace_global_list __read_mostly = &ftrace_list_end;
  90 static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;
  91 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
  92 static ftrace_func_t __ftrace_trace_function_delay __read_mostly = ftrace_stub;
  93 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
  94 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
  95 static struct ftrace_ops global_ops;
  96
  97 static void
  98 ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip);
  99
 100 /*
 101  * Traverse the ftrace_global_list, invoking all entries.  The reason that we
 102  * can use rcu_dereference_raw() is that elements removed from this list
 103  * are simply leaked, so there is no need to interact with a grace-period
 104  * mechanism.  The rcu_dereference_raw() calls are needed to handle
 105  * concurrent insertions into the ftrace_global_list.
 106  *
 107  * Silly Alpha and silly pointer-speculation compiler optimizations!
 108  */
 109 static void ftrace_global_list_func(unsigned long ip,
 110                                     unsigned long parent_ip)
 111 {
 112         struct ftrace_ops *op;
 113
 114         if (unlikely(trace_recursion_test(TRACE_GLOBAL_BIT)))
 115                 return;
 116
 117         trace_recursion_set(TRACE_GLOBAL_BIT);
 118         op = rcu_dereference_raw(ftrace_global_list); /*see above*/
 119         while (op != &ftrace_list_end) {
 120                 op->func(ip, parent_ip);
 121                 op = rcu_dereference_raw(op->next); /*see above*/
 122         };
 123         trace_recursion_clear(TRACE_GLOBAL_BIT);
 124 }
 125
 126 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
 127 {
 128         if (!test_tsk_trace_trace(current))
 129                 return;
 130
 131         ftrace_pid_function(ip, parent_ip);
 132 }
 133
 134 static void set_ftrace_pid_function(ftrace_func_t func)
 135 {
 136         /* do not set ftrace_pid_function to itself! */
 137         if (func != ftrace_pid_func)
 138                 ftrace_pid_function = func;
 139 }
 140
 141 /**
 142  * clear_ftrace_function - reset the ftrace function
 143  *
 144  * This NULLs the ftrace function and in essence stops
 145  * tracing.  There may be lag
 146  */
 147 void clear_ftrace_function(void)
 148 {
 149         ftrace_trace_function = ftrace_stub;
 150         __ftrace_trace_function = ftrace_stub;
 151         __ftrace_trace_function_delay = ftrace_stub;
 152         ftrace_pid_function = ftrace_stub;
 153 }
 154
 155 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 156 /*
 157  * For those archs that do not test ftrace_trace_stop in their
 158  * mcount call site, we need to do it from C.
 159  */
 160 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
 161 {
 162         if (function_trace_stop)
 163                 return;
 164
 165         __ftrace_trace_function(ip, parent_ip);
 166 }
 167 #endif
 168
 169 static void update_global_ops(void)
 170 {
 171         ftrace_func_t func;
 172
 173         /*
 174          * If there's only one function registered, then call that
 175          * function directly. Otherwise, we need to iterate over the
 176          * registered callers.
 177          */
 178         if (ftrace_global_list == &ftrace_list_end ||
 179             ftrace_global_list->next == &ftrace_list_end)
 180                 func = ftrace_global_list->func;
 181         else
 182                 func = ftrace_global_list_func;
 183
 184         /* If we filter on pids, update to use the pid function */
 185         if (!list_empty(&ftrace_pids)) {
 186                 set_ftrace_pid_function(func);
 187                 func = ftrace_pid_func;
 188         }
 189
 190         global_ops.func = func;
 191 }
 192
 193 static void update_ftrace_function(void)
 194 {
 195         ftrace_func_t func;
 196
 197         update_global_ops();
 198
 199         /*
 200          * If we are at the end of the list and this ops is
 201          * not dynamic, then have the mcount trampoline call
 202          * the function directly
 203          */
 204         if (ftrace_ops_list == &ftrace_list_end ||
 205             (ftrace_ops_list->next == &ftrace_list_end &&
 206              !(ftrace_ops_list->flags & FTRACE_OPS_FL_DYNAMIC)))
 207                 func = ftrace_ops_list->func;
 208         else
 209                 func = ftrace_ops_list_func;
 210
 211 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 212         ftrace_trace_function = func;
 213 #else
 214 #ifdef CONFIG_DYNAMIC_FTRACE
 215         /* do not update till all functions have been modified */
 216         __ftrace_trace_function_delay = func;
 217 #else
 218         __ftrace_trace_function = func;
 219 #endif
 220         ftrace_trace_function = ftrace_test_stop_func;
 221 #endif
 222 }
 223
 224 static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
 225 {
 226         ops->next = *list;
 227         /*
 228          * We are entering ops into the list but another
 229          * CPU might be walking that list. We need to make sure
 230          * the ops->next pointer is valid before another CPU sees
 231          * the ops pointer included into the list.
 232          */
 233         rcu_assign_pointer(*list, ops);
 234 }
 235
 236 static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
 237 {
 238         struct ftrace_ops **p;
 239
 240         /*
 241          * If we are removing the last function, then simply point
 242          * to the ftrace_stub.
 243          */
 244         if (*list == ops && ops->next == &ftrace_list_end) {
 245                 *list = &ftrace_list_end;
 246                 return 0;
 247         }
 248
 249         for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
 250                 if (*p == ops)
 251                         break;
 252
 253         if (*p != ops)
 254                 return -1;
 255
 256         *p = (*p)->next;
 257         return 0;
 258 }
 259
 260 static int __register_ftrace_function(struct ftrace_ops *ops)
 261 {
 262         if (FTRACE_WARN_ON(ops == &global_ops))
 263                 return -EINVAL;
 264
 265         if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
 266                 return -EBUSY;
 267
 268         if (!core_kernel_data((unsigned long)ops))
 269                 ops->flags |= FTRACE_OPS_FL_DYNAMIC;
 270
 271         if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
 272                 int first = ftrace_global_list == &ftrace_list_end;
 273                 add_ftrace_ops(&ftrace_global_list, ops);
 274                 ops->flags |= FTRACE_OPS_FL_ENABLED;
 275                 if (first)
 276                         add_ftrace_ops(&ftrace_ops_list, &global_ops);
 277         } else
 278                 add_ftrace_ops(&ftrace_ops_list, ops);
 279
 280         if (ftrace_enabled)
 281                 update_ftrace_function();
 282
 283         return 0;
 284 }
 285
 286 static int __unregister_ftrace_function(struct ftrace_ops *ops)
 287 {
 288         int ret;
 289
 290         if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
 291                 return -EBUSY;
 292
 293         if (FTRACE_WARN_ON(ops == &global_ops))
 294                 return -EINVAL;
 295
 296         if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
 297                 ret = remove_ftrace_ops(&ftrace_global_list, ops);
 298                 if (!ret && ftrace_global_list == &ftrace_list_end)
 299                         ret = remove_ftrace_ops(&ftrace_ops_list, &global_ops);
 300                 if (!ret)
 301                         ops->flags &= ~FTRACE_OPS_FL_ENABLED;
 302         } else
 303                 ret = remove_ftrace_ops(&ftrace_ops_list, ops);
 304
 305         if (ret < 0)
 306                 return ret;
 307
 308         if (ftrace_enabled)
 309                 update_ftrace_function();
 310
 311         /*
 312          * Dynamic ops may be freed, we must make sure that all
 313          * callers are done before leaving this function.
 314          */
 315         if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
 316                 synchronize_sched();
 317
 318         return 0;
 319 }
 320
 321 static void ftrace_update_pid_func(void)
 322 {
 323         /* Only do something if we are tracing something */
 324         if (ftrace_trace_function == ftrace_stub)
 325                 return;
 326
 327         update_ftrace_function();
 328 }
 329
 330 #ifdef CONFIG_FUNCTION_PROFILER
 331 struct ftrace_profile {
 332         struct hlist_node               node;
 333         unsigned long                   ip;
 334         unsigned long                   counter;
 335 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 336         unsigned long long              time;
 337         unsigned long long              time_squared;
 338 #endif
 339 };
 340
 341 struct ftrace_profile_page {
 342         struct ftrace_profile_page      *next;
 343         unsigned long                   index;
 344         struct ftrace_profile           records[];
 345 };
 346
 347 struct ftrace_profile_stat {
 348         atomic_t                        disabled;
 349         struct hlist_head               *hash;
 350         struct ftrace_profile_page      *pages;
 351         struct ftrace_profile_page      *start;
 352         struct tracer_stat              stat;
 353 };
 354
 355 #define PROFILE_RECORDS_SIZE                                            \
 356         (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
 357
 358 #define PROFILES_PER_PAGE                                       \
 359         (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
 360
 361 static int ftrace_profile_bits __read_mostly;
 362 static int ftrace_profile_enabled __read_mostly;
 363
 364 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
 365 static DEFINE_MUTEX(ftrace_profile_lock);
 366
 367 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
 368
 369 #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */
 370
 371 static void *
 372 function_stat_next(void *v, int idx)
 373 {
 374         struct ftrace_profile *rec = v;
 375         struct ftrace_profile_page *pg;
 376
 377         pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
 378
 379  again:
 380         if (idx != 0)
 381                 rec++;
 382
 383         if ((void *)rec >= (void *)&pg->records[pg->index]) {
 384                 pg = pg->next;
 385                 if (!pg)
 386                         return NULL;
 387                 rec = &pg->records[0];
 388                 if (!rec->counter)
 389                         goto again;
 390         }
 391
 392         return rec;
 393 }
 394
 395 static void *function_stat_start(struct tracer_stat *trace)
 396 {
 397         struct ftrace_profile_stat *stat =
 398                 container_of(trace, struct ftrace_profile_stat, stat);
 399
 400         if (!stat || !stat->start)
 401                 return NULL;
 402
 403         return function_stat_next(&stat->start->records[0], 0);
 404 }
 405
 406 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 407 /* function graph compares on total time */
 408 static int function_stat_cmp(void *p1, void *p2)
 409 {
 410         struct ftrace_profile *a = p1;
 411         struct ftrace_profile *b = p2;
 412
 413         if (a->time < b->time)
 414                 return -1;
 415         if (a->time > b->time)
 416                 return 1;
 417         else
 418                 return 0;
 419 }
 420 #else
 421 /* not function graph compares against hits */
 422 static int function_stat_cmp(void *p1, void *p2)
 423 {
 424         struct ftrace_profile *a = p1;
 425         struct ftrace_profile *b = p2;
 426
 427         if (a->counter < b->counter)
 428                 return -1;
 429         if (a->counter > b->counter)
 430                 return 1;
 431         else
 432                 return 0;
 433 }
 434 #endif
 435
 436 static int function_stat_headers(struct seq_file *m)
 437 {
 438 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 439         seq_printf(m, "  Function                               "
 440                    "Hit    Time            Avg             s^2\n"
 441                       "  --------                               "
 442                    "---    ----            ---             ---\n");
 443 #else
 444         seq_printf(m, "  Function                               Hit\n"
 445                       "  --------                               ---\n");
 446 #endif
 447         return 0;
 448 }
 449
 450 static int function_stat_show(struct seq_file *m, void *v)
 451 {
 452         struct ftrace_profile *rec = v;
 453         char str[KSYM_SYMBOL_LEN];
 454         int ret = 0;
 455 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 456         static struct trace_seq s;
 457         unsigned long long avg;
 458         unsigned long long stddev;
 459 #endif
 460         mutex_lock(&ftrace_profile_lock);
 461
 462         /* we raced with function_profile_reset() */
 463         if (unlikely(rec->counter == 0)) {
 464                 ret = -EBUSY;
 465                 goto out;
 466         }
 467
 468         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
 469         seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);
 470
 471 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 472         seq_printf(m, "    ");
 473         avg = rec->time;
 474         do_div(avg, rec->counter);
 475
 476         /* Sample standard deviation (s^2) */
 477         if (rec->counter <= 1)
 478                 stddev = 0;
 479         else {
 480                 stddev = rec->time_squared - rec->counter * avg * avg;
 481                 /*
 482                  * Divide only 1000 for ns^2 -> us^2 conversion.
 483                  * trace_print_graph_duration will divide 1000 again.
 484                  */
 485                 do_div(stddev, (rec->counter - 1) * 1000);
 486         }
 487
 488         trace_seq_init(&s);
 489         trace_print_graph_duration(rec->time, &s);
 490         trace_seq_puts(&s, "    ");
 491         trace_print_graph_duration(avg, &s);
 492         trace_seq_puts(&s, "    ");
 493         trace_print_graph_duration(stddev, &s);
 494         trace_print_seq(m, &s);
 495 #endif
 496         seq_putc(m, '\n');
 497 out:
 498         mutex_unlock(&ftrace_profile_lock);
 499
 500         return ret;
 501 }
 502
 503 static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
 504 {
 505         struct ftrace_profile_page *pg;
 506
 507         pg = stat->pages = stat->start;
 508
 509         while (pg) {
 510                 memset(pg->records, 0, PROFILE_RECORDS_SIZE);
 511                 pg->index = 0;
 512                 pg = pg->next;
 513         }
 514
 515         memset(stat->hash, 0,
 516                FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
 517 }
 518
 519 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
 520 {
 521         struct ftrace_profile_page *pg;
 522         int functions;
 523         int pages;
 524         int i;
 525
 526         /* If we already allocated, do nothing */
 527         if (stat->pages)
 528                 return 0;
 529
 530         stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
 531         if (!stat->pages)
 532                 return -ENOMEM;
 533
 534 #ifdef CONFIG_DYNAMIC_FTRACE
 535         functions = ftrace_update_tot_cnt;
 536 #else
 537         /*
 538          * We do not know the number of functions that exist because
 539          * dynamic tracing is what counts them. With past experience
 540          * we have around 20K functions. That should be more than enough.
 541          * It is highly unlikely we will execute every function in
 542          * the kernel.
 543          */
 544         functions = 20000;
 545 #endif
 546
 547         pg = stat->start = stat->pages;
 548
 549         pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
 550
 551         for (i = 1; i < pages; i++) {
 552                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
 553                 if (!pg->next)
 554                         goto out_free;
 555                 pg = pg->next;
 556         }
 557
 558         return 0;
 559
 560  out_free:
 561         pg = stat->start;
 562         while (pg) {
 563                 unsigned long tmp = (unsigned long)pg;
 564
 565                 pg = pg->next;
 566                 free_page(tmp);
 567         }
 568
 569         stat->pages = NULL;
 570         stat->start = NULL;
 571
 572         return -ENOMEM;
 573 }
 574
 575 static int ftrace_profile_init_cpu(int cpu)
 576 {
 577         struct ftrace_profile_stat *stat;
 578         int size;
 579
 580         stat = &per_cpu(ftrace_profile_stats, cpu);
 581
 582         if (stat->hash) {
 583                 /* If the profile is already created, simply reset it */
 584                 ftrace_profile_reset(stat);
 585                 return 0;
 586         }
 587
 588         /*
 589          * We are profiling all functions, but usually only a few thousand
 590          * functions are hit. We'll make a hash of 1024 items.
 591          */
 592         size = FTRACE_PROFILE_HASH_SIZE;
 593
 594         stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
 595
 596         if (!stat->hash)
 597                 return -ENOMEM;
 598
 599         if (!ftrace_profile_bits) {
 600                 size--;
 601
 602                 for (; size; size >>= 1)
 603                         ftrace_profile_bits++;
 604         }
 605
 606         /* Preallocate the function profiling pages */
 607         if (ftrace_profile_pages_init(stat) < 0) {
 608                 kfree(stat->hash);
 609                 stat->hash = NULL;
 610                 return -ENOMEM;
 611         }
 612
 613         return 0;
 614 }
 615
 616 static int ftrace_profile_init(void)
 617 {
 618         int cpu;
 619         int ret = 0;
 620
 621         for_each_possible_cpu(cpu) {
 622                 ret = ftrace_profile_init_cpu(cpu);
 623                 if (ret)
 624                         break;
 625         }
 626
 627         return ret;
 628 }
 629
 630 /* interrupts must be disabled */
 631 static struct ftrace_profile *
 632 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
 633 {
 634         struct ftrace_profile *rec;
 635         struct hlist_head *hhd;
 636         struct hlist_node *n;
 637         unsigned long key;
 638
 639         key = hash_long(ip, ftrace_profile_bits);
 640         hhd = &stat->hash[key];
 641
 642         if (hlist_empty(hhd))
 643                 return NULL;
 644
 645         hlist_for_each_entry_rcu(rec, n, hhd, node) {
 646                 if (rec->ip == ip)
 647                         return rec;
 648         }
 649
 650         return NULL;
 651 }
 652
 653 static void ftrace_add_profile(struct ftrace_profile_stat *stat,
 654                                struct ftrace_profile *rec)
 655 {
 656         unsigned long key;
 657
 658         key = hash_long(rec->ip, ftrace_profile_bits);
 659         hlist_add_head_rcu(&rec->node, &stat->hash[key]);
 660 }
 661
 662 /*
 663  * The memory is already allocated, this simply finds a new record to use.
 664  */
 665 static struct ftrace_profile *
 666 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
 667 {
 668         struct ftrace_profile *rec = NULL;
 669
 670         /* prevent recursion (from NMIs) */
 671         if (atomic_inc_return(&stat->disabled) != 1)
 672                 goto out;
 673
 674         /*
 675          * Try to find the function again since an NMI
 676          * could have added it
 677          */
 678         rec = ftrace_find_profiled_func(stat, ip);
 679         if (rec)
 680                 goto out;
 681
 682         if (stat->pages->index == PROFILES_PER_PAGE) {
 683                 if (!stat->pages->next)
 684                         goto out;
 685                 stat->pages = stat->pages->next;
 686         }
 687
 688         rec = &stat->pages->records[stat->pages->index++];
 689         rec->ip = ip;
 690         ftrace_add_profile(stat, rec);
 691
 692  out:
 693         atomic_dec(&stat->disabled);
 694
 695         return rec;
 696 }
 697
 698 static void
 699 function_profile_call(unsigned long ip, unsigned long parent_ip)
 700 {
 701         struct ftrace_profile_stat *stat;
 702         struct ftrace_profile *rec;
 703         unsigned long flags;
 704
 705         if (!ftrace_profile_enabled)
 706                 return;
 707
 708         local_irq_save(flags);
 709
 710         stat = &__get_cpu_var(ftrace_profile_stats);
 711         if (!stat->hash || !ftrace_profile_enabled)
 712                 goto out;
 713
 714         rec = ftrace_find_profiled_func(stat, ip);
 715         if (!rec) {
 716                 rec = ftrace_profile_alloc(stat, ip);
 717                 if (!rec)
 718                         goto out;
 719         }
 720
 721         rec->counter++;
 722  out:
 723         local_irq_restore(flags);
 724 }
 725
 726 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 727 static int profile_graph_entry(struct ftrace_graph_ent *trace)
 728 {
 729         function_profile_call(trace->func, 0);
 730         return 1;
 731 }
 732
 733 static void profile_graph_return(struct ftrace_graph_ret *trace)
 734 {
 735         struct ftrace_profile_stat *stat;
 736         unsigned long long calltime;
 737         struct ftrace_profile *rec;
 738         unsigned long flags;
 739
 740         local_irq_save(flags);
 741         stat = &__get_cpu_var(ftrace_profile_stats);
 742         if (!stat->hash || !ftrace_profile_enabled)
 743                 goto out;
 744
 745         /* If the calltime was zero'd ignore it */
 746         if (!trace->calltime)
 747                 goto out;
 748
 749         calltime = trace->rettime - trace->calltime;
 750
 751         if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
 752                 int index;
 753
 754                 index = trace->depth;
 755
 756                 /* Append this call time to the parent time to subtract */
 757                 if (index)
 758                         current->ret_stack[index - 1].subtime += calltime;
 759
 760                 if (current->ret_stack[index].subtime < calltime)
 761                         calltime -= current->ret_stack[index].subtime;
 762                 else
 763                         calltime = 0;
 764         }
 765
 766         rec = ftrace_find_profiled_func(stat, trace->func);
 767         if (rec) {
 768                 rec->time += calltime;
 769                 rec->time_squared += calltime * calltime;
 770         }
 771
 772  out:
 773         local_irq_restore(flags);
 774 }
 775
 776 static int register_ftrace_profiler(void)
 777 {
 778         return register_ftrace_graph(&profile_graph_return,
 779                                      &profile_graph_entry);
 780 }
 781
 782 static void unregister_ftrace_profiler(void)
 783 {
 784         unregister_ftrace_graph();
 785 }
 786 #else
 787 static struct ftrace_ops ftrace_profile_ops __read_mostly = {
 788         .func           = function_profile_call,
 789 };
 790
 791 static int register_ftrace_profiler(void)
 792 {
 793         return register_ftrace_function(&ftrace_profile_ops);
 794 }
 795
 796 static void unregister_ftrace_profiler(void)
 797 {
 798         unregister_ftrace_function(&ftrace_profile_ops);
 799 }
 800 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 801
 802 static ssize_t
 803 ftrace_profile_write(struct file *filp, const char __user *ubuf,
 804                      size_t cnt, loff_t *ppos)
 805 {
 806         unsigned long val;
 807         int ret;
 808
 809         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
 810         if (ret)
 811                 return ret;
 812
 813         val = !!val;
 814
 815         mutex_lock(&ftrace_profile_lock);
 816         if (ftrace_profile_enabled ^ val) {
 817                 if (val) {
 818                         ret = ftrace_profile_init();
 819                         if (ret < 0) {
 820                                 cnt = ret;
 821                                 goto out;
 822                         }
 823
 824                         ret = register_ftrace_profiler();
 825                         if (ret < 0) {
 826                                 cnt = ret;
 827                                 goto out;
 828                         }
 829                         ftrace_profile_enabled = 1;
 830                 } else {
 831                         ftrace_profile_enabled = 0;
 832                         /*
 833                          * unregister_ftrace_profiler calls stop_machine
 834                          * so this acts like an synchronize_sched.
 835                          */
 836                         unregister_ftrace_profiler();
 837                 }
 838         }
 839  out:
 840         mutex_unlock(&ftrace_profile_lock);
 841
 842         *ppos += cnt;
 843
 844         return cnt;
 845 }
 846
 847 static ssize_t
 848 ftrace_profile_read(struct file *filp, char __user *ubuf,
 849                      size_t cnt, loff_t *ppos)
 850 {
 851         char buf[64];           /* big enough to hold a number */
 852         int r;
 853
 854         r = sprintf(buf, "%u\n", ftrace_profile_enabled);
 855         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
 856 }
 857
 858 static const struct file_operations ftrace_profile_fops = {
 859         .open           = tracing_open_generic,
 860         .read           = ftrace_profile_read,
 861         .write          = ftrace_profile_write,
 862         .llseek         = default_llseek,
 863 };
 864
 865 /* used to initialize the real stat files */
 866 static struct tracer_stat function_stats __initdata = {
 867         .name           = "functions",
 868         .stat_start     = function_stat_start,
 869         .stat_next      = function_stat_next,
 870         .stat_cmp       = function_stat_cmp,
 871         .stat_headers   = function_stat_headers,
 872         .stat_show      = function_stat_show
 873 };
 874
 875 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
 876 {
 877         struct ftrace_profile_stat *stat;
 878         struct dentry *entry;
 879         char *name;
 880         int ret;
 881         int cpu;
 882
 883         for_each_possible_cpu(cpu) {
 884                 stat = &per_cpu(ftrace_profile_stats, cpu);
 885
 886                 /* allocate enough for function name + cpu number */
 887                 name = kmalloc(32, GFP_KERNEL);
 888                 if (!name) {
 889                         /*
 890                          * The files created are permanent, if something happens
 891                          * we still do not free memory.
 892                          */
 893                         WARN(1,
 894                              "Could not allocate stat file for cpu %d\n",
 895                              cpu);
 896                         return;
 897                 }
 898                 stat->stat = function_stats;
 899                 snprintf(name, 32, "function%d", cpu);
 900                 stat->stat.name = name;
 901                 ret = register_stat_tracer(&stat->stat);
 902                 if (ret) {
 903                         WARN(1,
 904                              "Could not register function stat for cpu %d\n",
 905                              cpu);
 906                         kfree(name);
 907                         return;
 908                 }
 909         }
 910
 911         entry = debugfs_create_file("function_profile_enabled", 0644,
 912                                     d_tracer, NULL, &ftrace_profile_fops);
 913         if (!entry)
 914                 pr_warning("Could not create debugfs "
 915                            "'function_profile_enabled' entry\n");
 916 }
 917
 918 #else /* CONFIG_FUNCTION_PROFILER */
 919 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
 920 {
 921 }
 922 #endif /* CONFIG_FUNCTION_PROFILER */
 923
 924 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
 925
 926 static loff_t
 927 ftrace_filter_lseek(struct file *file, loff_t offset, int whence)
 928 {
 929         loff_t ret;
 930
 931         if (file->f_mode & FMODE_READ)
 932                 ret = seq_lseek(file, offset, whence);
 933         else
 934                 file->f_pos = ret = 1;
 935
 936         return ret;
 937 }
 938
 939 #ifdef CONFIG_DYNAMIC_FTRACE
 940
 941 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
 942 # error Dynamic ftrace depends on MCOUNT_RECORD
 943 #endif
 944
 945 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
 946
 947 struct ftrace_func_probe {
 948         struct hlist_node       node;
 949         struct ftrace_probe_ops *ops;
 950         unsigned long           flags;
 951         unsigned long           ip;
 952         void                    *data;
 953         struct rcu_head         rcu;
 954 };
 955
 956 enum {
 957         FTRACE_UPDATE_CALLS             = (1 << 0),
 958         FTRACE_DISABLE_CALLS            = (1 << 1),
 959         FTRACE_UPDATE_TRACE_FUNC        = (1 << 2),
 960         FTRACE_START_FUNC_RET           = (1 << 3),
 961         FTRACE_STOP_FUNC_RET            = (1 << 4),
 962 };
 963 struct ftrace_func_entry {
 964         struct hlist_node hlist;
 965         unsigned long ip;
 966 };
 967
 968 struct ftrace_hash {
 969         unsigned long           size_bits;
 970         struct hlist_head       *buckets;
 971         unsigned long           count;
 972         struct rcu_head         rcu;
 973 };
 974
 975 /*
 976  * We make these constant because no one should touch them,
 977  * but they are used as the default "empty hash", to avoid allocating
 978  * it all the time. These are in a read only section such that if
 979  * anyone does try to modify it, it will cause an exception.
 980  */
 981 static const struct hlist_head empty_buckets[1];
 982 static const struct ftrace_hash empty_hash = {
 983         .buckets = (struct hlist_head *)empty_buckets,
 984 };
 985 #define EMPTY_HASH      ((struct ftrace_hash *)&empty_hash)
 986
 987 static struct ftrace_ops global_ops = {
 988         .func                   = ftrace_stub,
 989         .notrace_hash           = EMPTY_HASH,
 990         .filter_hash            = EMPTY_HASH,
 991 };
 992
 993 static struct dyn_ftrace *ftrace_new_addrs;
 994
 995 static DEFINE_MUTEX(ftrace_regex_lock);
 996
 997 struct ftrace_page {
 998         struct ftrace_page      *next;
 999         int                     index;
1000         struct dyn_ftrace       records[];
1001 };
1002
1003 #define ENTRIES_PER_PAGE \
1004   ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
1005
1006 /* estimate from running different kernels */
1007 #define NR_TO_INIT              10000
1008
1009 static struct ftrace_page       *ftrace_pages_start;
1010 static struct ftrace_page       *ftrace_pages;
1011
1012 static struct dyn_ftrace *ftrace_free_records;
1013
1014 static bool ftrace_hash_empty(struct ftrace_hash *hash)
1015 {
1016         return !hash || !hash->count;
1017 }
1018
1019 static struct ftrace_func_entry *
1020 ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1021 {
1022         unsigned long key;
1023         struct ftrace_func_entry *entry;
1024         struct hlist_head *hhd;
1025         struct hlist_node *n;
1026
1027         if (ftrace_hash_empty(hash))
1028                 return NULL;
1029
1030         if (hash->size_bits > 0)
1031                 key = hash_long(ip, hash->size_bits);
1032         else
1033                 key = 0;
1034
1035         hhd = &hash->buckets[key];
1036
1037         hlist_for_each_entry_rcu(entry, n, hhd, hlist) {
1038                 if (entry->ip == ip)
1039                         return entry;
1040         }
1041         return NULL;
1042 }
1043
1044 static void __add_hash_entry(struct ftrace_hash *hash,
1045                              struct ftrace_func_entry *entry)
1046 {
1047         struct hlist_head *hhd;
1048         unsigned long key;
1049
1050         if (hash->size_bits)
1051                 key = hash_long(entry->ip, hash->size_bits);
1052         else
1053                 key = 0;
1054
1055         hhd = &hash->buckets[key];
1056         hlist_add_head(&entry->hlist, hhd);
1057         hash->count++;
1058 }
1059
1060 static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
1061 {
1062         struct ftrace_func_entry *entry;
1063
1064         entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1065         if (!entry)
1066                 return -ENOMEM;
1067
1068         entry->ip = ip;
1069         __add_hash_entry(hash, entry);
1070
1071         return 0;
1072 }
1073
1074 static void
1075 free_hash_entry(struct ftrace_hash *hash,
1076                   struct ftrace_func_entry *entry)
1077 {
1078         hlist_del(&entry->hlist);
1079         kfree(entry);
1080         hash->count--;
1081 }
1082
1083 static void
1084 remove_hash_entry(struct ftrace_hash *hash,
1085                   struct ftrace_func_entry *entry)
1086 {
1087         hlist_del(&entry->hlist);
1088         hash->count--;
1089 }
1090
1091 static void ftrace_hash_clear(struct ftrace_hash *hash)
1092 {
1093         struct hlist_head *hhd;
1094         struct hlist_node *tp, *tn;
1095         struct ftrace_func_entry *entry;
1096         int size = 1 << hash->size_bits;
1097         int i;
1098
1099         if (!hash->count)
1100                 return;
1101
1102         for (i = 0; i < size; i++) {
1103                 hhd = &hash->buckets[i];
1104                 hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist)
1105                         free_hash_entry(hash, entry);
1106         }
1107         FTRACE_WARN_ON(hash->count);
1108 }
1109
1110 static void free_ftrace_hash(struct ftrace_hash *hash)
1111 {
1112         if (!hash || hash == EMPTY_HASH)
1113                 return;
1114         ftrace_hash_clear(hash);
1115         kfree(hash->buckets);
1116         kfree(hash);
1117 }
1118
1119 static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
1120 {
1121         struct ftrace_hash *hash;
1122
1123         hash = container_of(rcu, struct ftrace_hash, rcu);
1124         free_ftrace_hash(hash);
1125 }
1126
1127 static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
1128 {
1129         if (!hash || hash == EMPTY_HASH)
1130                 return;
1131         call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
1132 }
1133
1134 static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
1135 {
1136         struct ftrace_hash *hash;
1137         int size;
1138
1139         hash = kzalloc(sizeof(*hash), GFP_KERNEL);
1140         if (!hash)
1141                 return NULL;
1142
1143         size = 1 << size_bits;
1144         hash->buckets = kzalloc(sizeof(*hash->buckets) * size, GFP_KERNEL);
1145
1146         if (!hash->buckets) {
1147                 kfree(hash);
1148                 return NULL;
1149         }
1150
1151         hash->size_bits = size_bits;
1152
1153         return hash;
1154 }
1155
1156 static struct ftrace_hash *
1157 alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
1158 {
1159         struct ftrace_func_entry *entry;
1160         struct ftrace_hash *new_hash;
1161         struct hlist_node *tp;
1162         int size;
1163         int ret;
1164         int i;
1165
1166         new_hash = alloc_ftrace_hash(size_bits);
1167         if (!new_hash)
1168                 return NULL;
1169
1170         /* Empty hash? */
1171         if (ftrace_hash_empty(hash))
1172                 return new_hash;
1173
1174         size = 1 << hash->size_bits;
1175         for (i = 0; i < size; i++) {
1176                 hlist_for_each_entry(entry, tp, &hash->buckets[i], hlist) {
1177                         ret = add_hash_entry(new_hash, entry->ip);
1178                         if (ret < 0)
1179                                 goto free_hash;
1180                 }
1181         }
1182
1183         FTRACE_WARN_ON(new_hash->count != hash->count);
1184
1185         return new_hash;
1186
1187  free_hash:
1188         free_ftrace_hash(new_hash);
1189         return NULL;
1190 }
1191
1192 static void
1193 ftrace_hash_rec_disable(struct ftrace_ops *ops, int filter_hash);
1194 static void
1195 ftrace_hash_rec_enable(struct ftrace_ops *ops, int filter_hash);
1196
1197 static int
1198 ftrace_hash_move(struct ftrace_ops *ops, int enable,
1199                  struct ftrace_hash **dst, struct ftrace_hash *src)
1200 {
1201         struct ftrace_func_entry *entry;
1202         struct hlist_node *tp, *tn;
1203         struct hlist_head *hhd;
1204         struct ftrace_hash *old_hash;
1205         struct ftrace_hash *new_hash;
1206         unsigned long key;
1207         int size = src->count;
1208         int bits = 0;
1209         int ret;
1210         int i;
1211
1212         /*
1213          * Remove the current set, update the hash and add
1214          * them back.
1215          */
1216         ftrace_hash_rec_disable(ops, enable);
1217
1218         /*
1219          * If the new source is empty, just free dst and assign it
1220          * the empty_hash.
1221          */
1222         if (!src->count) {
1223                 free_ftrace_hash_rcu(*dst);
1224                 rcu_assign_pointer(*dst, EMPTY_HASH);
1225                 /* still need to update the function records */
1226                 ret = 0;
1227                 goto out;
1228         }
1229
1230         /*
1231          * Make the hash size about 1/2 the # found
1232          */
1233         for (size /= 2; size; size >>= 1)
1234                 bits++;
1235
1236         /* Don't allocate too much */
1237         if (bits > FTRACE_HASH_MAX_BITS)
1238                 bits = FTRACE_HASH_MAX_BITS;
1239
1240         ret = -ENOMEM;
1241         new_hash = alloc_ftrace_hash(bits);
1242         if (!new_hash)
1243                 goto out;
1244
1245         size = 1 << src->size_bits;
1246         for (i = 0; i < size; i++) {
1247                 hhd = &src->buckets[i];
1248                 hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist) {
1249                         if (bits > 0)
1250                                 key = hash_long(entry->ip, bits);
1251                         else
1252                                 key = 0;
1253                         remove_hash_entry(src, entry);
1254                         __add_hash_entry(new_hash, entry);
1255                 }
1256         }
1257
1258         old_hash = *dst;
1259         rcu_assign_pointer(*dst, new_hash);
1260         free_ftrace_hash_rcu(old_hash);
1261
1262         ret = 0;
1263  out:
1264         /*
1265          * Enable regardless of ret:
1266          *  On success, we enable the new hash.
1267          *  On failure, we re-enable the original hash.
1268          */
1269         ftrace_hash_rec_enable(ops, enable);
1270
1271         return ret;
1272 }
1273
1274 /*
1275  * Test the hashes for this ops to see if we want to call
1276  * the ops->func or not.
1277  *
1278  * It's a match if the ip is in the ops->filter_hash or
1279  * the filter_hash does not exist or is empty,
1280  *  AND
1281  * the ip is not in the ops->notrace_hash.
1282  *
1283  * This needs to be called with preemption disabled as
1284  * the hashes are freed with call_rcu_sched().
1285  */
1286 static int
1287 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
1288 {
1289         struct ftrace_hash *filter_hash;
1290         struct ftrace_hash *notrace_hash;
1291         int ret;
1292
1293         filter_hash = rcu_dereference_raw(ops->filter_hash);
1294         notrace_hash = rcu_dereference_raw(ops->notrace_hash);
1295
1296         if ((ftrace_hash_empty(filter_hash) ||
1297              ftrace_lookup_ip(filter_hash, ip)) &&
1298             (ftrace_hash_empty(notrace_hash) ||
1299              !ftrace_lookup_ip(notrace_hash, ip)))
1300                 ret = 1;
1301         else
1302                 ret = 0;
1303
1304         return ret;
1305 }
1306
1307 /*
1308  * This is a double for. Do not use 'break' to break out of the loop,
1309  * you must use a goto.
1310  */
1311 #define do_for_each_ftrace_rec(pg, rec)                                 \
1312         for (pg = ftrace_pages_start; pg; pg = pg->next) {              \
1313                 int _____i;                                             \
1314                 for (_____i = 0; _____i < pg->index; _____i++) {        \
1315                         rec = &pg->records[_____i];
1316
1317 #define while_for_each_ftrace_rec()             \
1318                 }                               \
1319         }
1320
1321 static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
1322                                      int filter_hash,
1323                                      bool inc)
1324 {
1325         struct ftrace_hash *hash;
1326         struct ftrace_hash *other_hash;
1327         struct ftrace_page *pg;
1328         struct dyn_ftrace *rec;
1329         int count = 0;
1330         int all = 0;
1331
1332         /* Only update if the ops has been registered */
1333         if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1334                 return;
1335
1336         /*
1337          * In the filter_hash case:
1338          *   If the count is zero, we update all records.
1339          *   Otherwise we just update the items in the hash.
1340          *
1341          * In the notrace_hash case:
1342          *   We enable the update in the hash.
1343          *   As disabling notrace means enabling the tracing,
1344          *   and enabling notrace means disabling, the inc variable
1345          *   gets inversed.
1346          */
1347         if (filter_hash) {
1348                 hash = ops->filter_hash;
1349                 other_hash = ops->notrace_hash;
1350                 if (ftrace_hash_empty(hash))
1351                         all = 1;
1352         } else {
1353                 inc = !inc;
1354                 hash = ops->notrace_hash;
1355                 other_hash = ops->filter_hash;
1356                 /*
1357                  * If the notrace hash has no items,
1358                  * then there's nothing to do.
1359                  */
1360                 if (ftrace_hash_empty(hash))
1361                         return;
1362         }
1363
1364         do_for_each_ftrace_rec(pg, rec) {
1365                 int in_other_hash = 0;
1366                 int in_hash = 0;
1367                 int match = 0;
1368
1369                 if (all) {
1370                         /*
1371                          * Only the filter_hash affects all records.
1372                          * Update if the record is not in the notrace hash.
1373                          */
1374                         if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
1375                                 match = 1;
1376                 } else {
1377                         in_hash = !!ftrace_lookup_ip(hash, rec->ip);
1378                         in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
1379
1380                         /*
1381                          *
1382                          */
1383                         if (filter_hash && in_hash && !in_other_hash)
1384                                 match = 1;
1385                         else if (!filter_hash && in_hash &&
1386                                  (in_other_hash || ftrace_hash_empty(other_hash)))
1387                                 match = 1;
1388                 }
1389                 if (!match)
1390                         continue;
1391
1392                 if (inc) {
1393                         rec->flags++;
1394                         if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == FTRACE_REF_MAX))
1395                                 return;
1396                 } else {
1397                         if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == 0))
1398                                 return;
1399                         rec->flags--;
1400                 }
1401                 count++;
1402                 /* Shortcut, if we handled all records, we are done. */
1403                 if (!all && count == hash->count)
1404                         return;
1405         } while_for_each_ftrace_rec();
1406 }
1407
1408 static void ftrace_hash_rec_disable(struct ftrace_ops *ops,
1409                                     int filter_hash)
1410 {
1411         __ftrace_hash_rec_update(ops, filter_hash, 0);
1412 }
1413
1414 static void ftrace_hash_rec_enable(struct ftrace_ops *ops,
1415                                    int filter_hash)
1416 {
1417         __ftrace_hash_rec_update(ops, filter_hash, 1);
1418 }
1419
1420 static void ftrace_free_rec(struct dyn_ftrace *rec)
1421 {
1422         rec->freelist = ftrace_free_records;
1423         ftrace_free_records = rec;
1424         rec->flags |= FTRACE_FL_FREE;
1425 }
1426
1427 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
1428 {
1429         struct dyn_ftrace *rec;
1430
1431         /* First check for freed records */
1432         if (ftrace_free_records) {
1433                 rec = ftrace_free_records;
1434
1435                 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
1436                         FTRACE_WARN_ON_ONCE(1);
1437                         ftrace_free_records = NULL;
1438                         return NULL;
1439                 }
1440
1441                 ftrace_free_records = rec->freelist;
1442                 memset(rec, 0, sizeof(*rec));
1443                 return rec;
1444         }
1445
1446         if (ftrace_pages->index == ENTRIES_PER_PAGE) {
1447                 if (!ftrace_pages->next) {
1448                         /* allocate another page */
1449                         ftrace_pages->next =
1450                                 (void *)get_zeroed_page(GFP_KERNEL);
1451                         if (!ftrace_pages->next)
1452                                 return NULL;
1453                 }
1454                 ftrace_pages = ftrace_pages->next;
1455         }
1456
1457         return &ftrace_pages->records[ftrace_pages->index++];
1458 }
1459
1460 static struct dyn_ftrace *
1461 ftrace_record_ip(unsigned long ip)
1462 {
1463         struct dyn_ftrace *rec;
1464
1465         if (ftrace_disabled)
1466                 return NULL;
1467
1468         rec = ftrace_alloc_dyn_node(ip);
1469         if (!rec)
1470                 return NULL;
1471
1472         rec->ip = ip;
1473         rec->newlist = ftrace_new_addrs;
1474         ftrace_new_addrs = rec;
1475
1476         return rec;
1477 }
1478
1479 static void print_ip_ins(const char *fmt, unsigned char *p)
1480 {
1481         int i;
1482
1483         printk(KERN_CONT "%s", fmt);
1484
1485         for (i = 0; i < MCOUNT_INSN_SIZE; i++)
1486                 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
1487 }
1488
1489 static void ftrace_bug(int failed, unsigned long ip)
1490 {
1491         switch (failed) {
1492         case -EFAULT:
1493                 FTRACE_WARN_ON_ONCE(1);
1494                 pr_info("ftrace faulted on modifying ");
1495                 print_ip_sym(ip);
1496                 break;
1497         case -EINVAL:
1498                 FTRACE_WARN_ON_ONCE(1);
1499                 pr_info("ftrace failed to modify ");
1500                 print_ip_sym(ip);
1501                 print_ip_ins(" actual: ", (unsigned char *)ip);
1502                 printk(KERN_CONT "\n");
1503                 break;
1504         case -EPERM:
1505                 FTRACE_WARN_ON_ONCE(1);
1506                 pr_info("ftrace faulted on writing ");
1507                 print_ip_sym(ip);
1508                 break;
1509         default:
1510                 FTRACE_WARN_ON_ONCE(1);
1511                 pr_info("ftrace faulted on unknown error ");
1512                 print_ip_sym(ip);
1513         }
1514 }
1515
1516
1517 /* Return 1 if the address range is reserved for ftrace */
1518 int ftrace_text_reserved(void *start, void *end)
1519 {
1520         struct dyn_ftrace *rec;
1521         struct ftrace_page *pg;
1522
1523         do_for_each_ftrace_rec(pg, rec) {
1524                 if (rec->ip <= (unsigned long)end &&
1525                     rec->ip + MCOUNT_INSN_SIZE > (unsigned long)start)
1526                         return 1;
1527         } while_for_each_ftrace_rec();
1528         return 0;
1529 }
1530
1531
1532 static int
1533 __ftrace_replace_code(struct dyn_ftrace *rec, int update)
1534 {
1535         unsigned long ftrace_addr;
1536         unsigned long flag = 0UL;
1537
1538         ftrace_addr = (unsigned long)FTRACE_ADDR;
1539
1540         /*
1541          * If we are updating calls:
1542          *
1543          *   If the record has a ref count, then we need to enable it
1544          *   because someone is using it.
1545          *
1546          *   Otherwise we make sure its disabled.
1547          *
1548          * If we are disabling calls, then disable all records that
1549          * are enabled.
1550          */
1551         if (update && (rec->flags & ~FTRACE_FL_MASK))
1552                 flag = FTRACE_FL_ENABLED;
1553
1554         /* If the state of this record hasn't changed, then do nothing */
1555         if ((rec->flags & FTRACE_FL_ENABLED) == flag)
1556                 return 0;
1557
1558         if (flag) {
1559                 rec->flags |= FTRACE_FL_ENABLED;
1560                 return ftrace_make_call(rec, ftrace_addr);
1561         }
1562
1563         rec->flags &= ~FTRACE_FL_ENABLED;
1564         return ftrace_make_nop(NULL, rec, ftrace_addr);
1565 }
1566
1567 static void ftrace_replace_code(int update)
1568 {
1569         struct dyn_ftrace *rec;
1570         struct ftrace_page *pg;
1571         int failed;
1572
1573         if (unlikely(ftrace_disabled))
1574                 return;
1575
1576         do_for_each_ftrace_rec(pg, rec) {
1577                 /* Skip over free records */
1578                 if (rec->flags & FTRACE_FL_FREE)
1579                         continue;
1580
1581                 failed = __ftrace_replace_code(rec, update);
1582                 if (failed) {
1583                         ftrace_bug(failed, rec->ip);
1584                         /* Stop processing */
1585                         return;
1586                 }
1587         } while_for_each_ftrace_rec();
1588 }
1589
1590 static int
1591 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
1592 {
1593         unsigned long ip;
1594         int ret;
1595
1596         ip = rec->ip;
1597
1598         if (unlikely(ftrace_disabled))
1599                 return 0;
1600
1601         ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
1602         if (ret) {
1603                 ftrace_bug(ret, ip);
1604                 return 0;
1605         }
1606         return 1;
1607 }
1608
1609 /*
1610  * archs can override this function if they must do something
1611  * before the modifying code is performed.
1612  */
1613 int __weak ftrace_arch_code_modify_prepare(void)
1614 {
1615         return 0;
1616 }
1617
1618 /*
1619  * archs can override this function if they must do something
1620  * after the modifying code is performed.
1621  */
1622 int __weak ftrace_arch_code_modify_post_process(void)
1623 {
1624         return 0;
1625 }
1626
1627 static int __ftrace_modify_code(void *data)
1628 {
1629         int *command = data;
1630
1631         /*
1632          * Do not call function tracer while we update the code.
1633          * We are in stop machine, no worrying about races.
1634          */
1635         function_trace_stop++;
1636
1637         if (*command & FTRACE_UPDATE_CALLS)
1638                 ftrace_replace_code(1);
1639         else if (*command & FTRACE_DISABLE_CALLS)
1640                 ftrace_replace_code(0);
1641
1642         if (*command & FTRACE_UPDATE_TRACE_FUNC)
1643                 ftrace_update_ftrace_func(ftrace_trace_function);
1644
1645         if (*command & FTRACE_START_FUNC_RET)
1646                 ftrace_enable_ftrace_graph_caller();
1647         else if (*command & FTRACE_STOP_FUNC_RET)
1648                 ftrace_disable_ftrace_graph_caller();
1649
1650 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
1651         /*
1652          * For archs that call ftrace_test_stop_func(), we must
1653          * wait till after we update all the function callers
1654          * before we update the callback. This keeps different
1655          * ops that record different functions from corrupting
1656          * each other.
1657          */
1658         __ftrace_trace_function = __ftrace_trace_function_delay;
1659 #endif
1660         function_trace_stop--;
1661
1662         return 0;
1663 }
1664
1665 static void ftrace_run_update_code(int command)
1666 {
1667         int ret;
1668
1669         ret = ftrace_arch_code_modify_prepare();
1670         FTRACE_WARN_ON(ret);
1671         if (ret)
1672                 return;
1673
1674         stop_machine(__ftrace_modify_code, &command, NULL);
1675
1676         ret = ftrace_arch_code_modify_post_process();
1677         FTRACE_WARN_ON(ret);
1678 }
1679
1680 static ftrace_func_t saved_ftrace_func;
1681 static int ftrace_start_up;
1682 static int global_start_up;
1683
1684 static void ftrace_startup_enable(int command)
1685 {
1686         if (saved_ftrace_func != ftrace_trace_function) {
1687                 saved_ftrace_func = ftrace_trace_function;
1688                 command |= FTRACE_UPDATE_TRACE_FUNC;
1689         }
1690
1691         if (!command || !ftrace_enabled)
1692                 return;
1693
1694         ftrace_run_update_code(command);
1695 }
1696
1697 static int ftrace_startup(struct ftrace_ops *ops, int command)
1698 {
1699         bool hash_enable = true;
1700         int ret;
1701
1702         if (unlikely(ftrace_disabled))
1703                 return -ENODEV;
1704
1705         ret = __register_ftrace_function(ops);
1706         if (ret)
1707                 return ret;
1708
1709         ftrace_start_up++;
1710         command |= FTRACE_UPDATE_CALLS;
1711
1712         /* ops marked global share the filter hashes */
1713         if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
1714                 ops = &global_ops;
1715                 /* Don't update hash if global is already set */
1716                 if (global_start_up)
1717                         hash_enable = false;
1718                 global_start_up++;
1719         }
1720
1721         ops->flags |= FTRACE_OPS_FL_ENABLED;
1722         if (hash_enable)
1723                 ftrace_hash_rec_enable(ops, 1);
1724
1725         ftrace_startup_enable(command);
1726
1727         return 0;
1728 }
1729
1730 static int ftrace_shutdown(struct ftrace_ops *ops, int command)
1731 {
1732         bool hash_disable = true;
1733         int ret;
1734
1735         if (unlikely(ftrace_disabled))
1736                 return -ENODEV;
1737
1738         ret = __unregister_ftrace_function(ops);
1739         if (ret)
1740                 return ret;
1741
1742         ftrace_start_up--;
1743         /*
1744          * Just warn in case of unbalance, no need to kill ftrace, it's not
1745          * critical but the ftrace_call callers may be never nopped again after
1746          * further ftrace uses.
1747          */
1748         WARN_ON_ONCE(ftrace_start_up < 0);
1749
1750         if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
1751                 ops = &global_ops;
1752                 global_start_up--;
1753                 WARN_ON_ONCE(global_start_up < 0);
1754                 /* Don't update hash if global still has users */
1755                 if (global_start_up) {
1756                         WARN_ON_ONCE(!ftrace_start_up);
1757                         hash_disable = false;
1758                 }
1759         }
1760
1761         if (hash_disable)
1762                 ftrace_hash_rec_disable(ops, 1);
1763
1764         if (ops != &global_ops || !global_start_up)
1765                 ops->flags &= ~FTRACE_OPS_FL_ENABLED;
1766
1767         command |= FTRACE_UPDATE_CALLS;
1768
1769         if (saved_ftrace_func != ftrace_trace_function) {
1770                 saved_ftrace_func = ftrace_trace_function;
1771                 command |= FTRACE_UPDATE_TRACE_FUNC;
1772         }
1773
1774         if (!command || !ftrace_enabled)
1775                 return 0;
1776
1777         ftrace_run_update_code(command);
1778         return 0;
1779 }
1780
1781 static void ftrace_startup_sysctl(void)
1782 {
1783         if (unlikely(ftrace_disabled))
1784                 return;
1785
1786         /* Force update next time */
1787         saved_ftrace_func = NULL;
1788         /* ftrace_start_up is true if we want ftrace running */
1789         if (ftrace_start_up)
1790                 ftrace_run_update_code(FTRACE_UPDATE_CALLS);
1791 }
1792
1793 static void ftrace_shutdown_sysctl(void)
1794 {
1795         if (unlikely(ftrace_disabled))
1796                 return;
1797
1798         /* ftrace_start_up is true if ftrace is running */
1799         if (ftrace_start_up)
1800                 ftrace_run_update_code(FTRACE_DISABLE_CALLS);
1801 }
1802
1803 static cycle_t          ftrace_update_time;
1804 static unsigned long    ftrace_update_cnt;
1805 unsigned long           ftrace_update_tot_cnt;
1806
1807 static inline int ops_traces_mod(struct ftrace_ops *ops)
1808 {
1809         /*
1810          * Filter_hash being empty will default to trace module.
1811          * But notrace hash requires a test of individual module functions.
1812          */
1813         return ftrace_hash_empty(ops->filter_hash) &&
1814                 ftrace_hash_empty(ops->notrace_hash);
1815 }
1816
1817 /*
1818  * Check if the current ops references the record.
1819  *
1820  * If the ops traces all functions, then it was already accounted for.
1821  * If the ops does not trace the current record function, skip it.
1822  * If the ops ignores the function via notrace filter, skip it.
1823  */
1824 static inline bool
1825 ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec)
1826 {
1827         /* If ops isn't enabled, ignore it */
1828         if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1829                 return 0;
1830
1831         /* If ops traces all mods, we already accounted for it */
1832         if (ops_traces_mod(ops))
1833                 return 0;
1834
1835         /* The function must be in the filter */
1836         if (!ftrace_hash_empty(ops->filter_hash) &&
1837             !ftrace_lookup_ip(ops->filter_hash, rec->ip))
1838                 return 0;
1839
1840         /* If in notrace hash, we ignore it too */
1841         if (ftrace_lookup_ip(ops->notrace_hash, rec->ip))
1842                 return 0;
1843
1844         return 1;
1845 }
1846
1847 static int referenced_filters(struct dyn_ftrace *rec)
1848 {
1849         struct ftrace_ops *ops;
1850         int cnt = 0;
1851
1852         for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {
1853                 if (ops_references_rec(ops, rec))
1854                     cnt++;
1855         }
1856
1857         return cnt;
1858 }
1859
1860 static int ftrace_update_code(struct module *mod)
1861 {
1862         struct dyn_ftrace *p;
1863         cycle_t start, stop;
1864         unsigned long ref = 0;
1865         bool test = false;
1866
1867         /*
1868          * When adding a module, we need to check if tracers are
1869          * currently enabled and if they are set to trace all functions.
1870          * If they are, we need to enable the module functions as well
1871          * as update the reference counts for those function records.
1872          */
1873         if (mod) {
1874                 struct ftrace_ops *ops;
1875
1876                 for (ops = ftrace_ops_list;
1877                      ops != &ftrace_list_end; ops = ops->next) {
1878                         if (ops->flags & FTRACE_OPS_FL_ENABLED) {
1879                                 if (ops_traces_mod(ops))
1880                                         ref++;
1881                                 else
1882                                         test = true;
1883                         }
1884                 }
1885         }
1886
1887         start = ftrace_now(raw_smp_processor_id());
1888         ftrace_update_cnt = 0;
1889
1890         while (ftrace_new_addrs) {
1891                 int cnt = ref;
1892
1893                 /* If something went wrong, bail without enabling anything */
1894                 if (unlikely(ftrace_disabled))
1895                         return -1;
1896
1897                 p = ftrace_new_addrs;
1898                 ftrace_new_addrs = p->newlist;
1899                 if (test)
1900                         cnt += referenced_filters(p);
1901                 p->flags = cnt;
1902
1903                 /*
1904                  * Do the initial record conversion from mcount jump
1905                  * to the NOP instructions.
1906                  */
1907                 if (!ftrace_code_disable(mod, p)) {
1908                         ftrace_free_rec(p);
1909                         /* Game over */
1910                         break;
1911                 }
1912
1913                 ftrace_update_cnt++;
1914
1915                 /*
1916                  * If the tracing is enabled, go ahead and enable the record.
1917                  *
1918                  * The reason not to enable the record immediatelly is the
1919                  * inherent check of ftrace_make_nop/ftrace_make_call for
1920                  * correct previous instructions.  Making first the NOP
1921                  * conversion puts the module to the correct state, thus
1922                  * passing the ftrace_make_call check.
1923                  */
1924                 if (ftrace_start_up && cnt) {
1925                         int failed = __ftrace_replace_code(p, 1);
1926                         if (failed) {
1927                                 ftrace_bug(failed, p->ip);
1928                                 ftrace_free_rec(p);
1929                         }
1930                 }
1931         }
1932
1933         stop = ftrace_now(raw_smp_processor_id());
1934         ftrace_update_time = stop - start;
1935         ftrace_update_tot_cnt += ftrace_update_cnt;
1936
1937         return 0;
1938 }
1939
1940 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
1941 {
1942         struct ftrace_page *pg;
1943         int cnt;
1944         int i;
1945
1946         /* allocate a few pages */
1947         ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
1948         if (!ftrace_pages_start)
1949                 return -1;
1950
1951         /*
1952          * Allocate a few more pages.
1953          *
1954          * TODO: have some parser search vmlinux before
1955          *   final linking to find all calls to ftrace.
1956          *   Then we can:
1957          *    a) know how many pages to allocate.
1958          *     and/or
1959          *    b) set up the table then.
1960          *
1961          *  The dynamic code is still necessary for
1962          *  modules.
1963          */
1964
1965         pg = ftrace_pages = ftrace_pages_start;
1966
1967         cnt = num_to_init / ENTRIES_PER_PAGE;
1968         pr_info("ftrace: allocating %ld entries in %d pages\n",
1969                 num_to_init, cnt + 1);
1970
1971         for (i = 0; i < cnt; i++) {
1972                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
1973
1974                 /* If we fail, we'll try later anyway */
1975                 if (!pg->next)
1976                         break;
1977
1978                 pg = pg->next;
1979         }
1980
1981         return 0;
1982 }
1983
1984 enum {
1985         FTRACE_ITER_FILTER      = (1 << 0),
1986         FTRACE_ITER_NOTRACE     = (1 << 1),
1987         FTRACE_ITER_PRINTALL    = (1 << 2),
1988         FTRACE_ITER_HASH        = (1 << 3),
1989         FTRACE_ITER_ENABLED     = (1 << 4),
1990 };
1991
1992 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
1993
1994 struct ftrace_iterator {
1995         loff_t                          pos;
1996         loff_t                          func_pos;
1997         struct ftrace_page              *pg;
1998         struct dyn_ftrace               *func;
1999         struct ftrace_func_probe        *probe;
2000         struct trace_parser             parser;
2001         struct ftrace_hash              *hash;
2002         struct ftrace_ops               *ops;
2003         int                             hidx;
2004         int                             idx;
2005         unsigned                        flags;
2006 };
2007
2008 static void *
2009 t_hash_next(struct seq_file *m, loff_t *pos)
2010 {
2011         struct ftrace_iterator *iter = m->private;
2012         struct hlist_node *hnd = NULL;
2013         struct hlist_head *hhd;
2014
2015         (*pos)++;
2016         iter->pos = *pos;
2017
2018         if (iter->probe)
2019                 hnd = &iter->probe->node;
2020  retry:
2021         if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
2022                 return NULL;
2023
2024         hhd = &ftrace_func_hash[iter->hidx];
2025
2026         if (hlist_empty(hhd)) {
2027                 iter->hidx++;
2028                 hnd = NULL;
2029                 goto retry;
2030         }
2031
2032         if (!hnd)
2033                 hnd = hhd->first;
2034         else {
2035                 hnd = hnd->next;
2036                 if (!hnd) {
2037                         iter->hidx++;
2038                         goto retry;
2039                 }
2040         }
2041
2042         if (WARN_ON_ONCE(!hnd))
2043                 return NULL;
2044
2045         iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node);
2046
2047         return iter;
2048 }
2049
2050 static void *t_hash_start(struct seq_file *m, loff_t *pos)
2051 {
2052         struct ftrace_iterator *iter = m->private;
2053         void *p = NULL;
2054         loff_t l;
2055
2056         if (iter->func_pos > *pos)
2057                 return NULL;
2058
2059         iter->hidx = 0;
2060         for (l = 0; l <= (*pos - iter->func_pos); ) {
2061                 p = t_hash_next(m, &l);
2062                 if (!p)
2063                         break;
2064         }
2065         if (!p)
2066                 return NULL;
2067
2068         /* Only set this if we have an item */
2069         iter->flags |= FTRACE_ITER_HASH;
2070
2071         return iter;
2072 }
2073
2074 static int
2075 t_hash_show(struct seq_file *m, struct ftrace_iterator *iter)
2076 {
2077         struct ftrace_func_probe *rec;
2078
2079         rec = iter->probe;
2080         if (WARN_ON_ONCE(!rec))
2081                 return -EIO;
2082
2083         if (rec->ops->print)
2084                 return rec->ops->print(m, rec->ip, rec->ops, rec->data);
2085
2086         seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
2087
2088         if (rec->data)
2089                 seq_printf(m, ":%p", rec->data);
2090         seq_putc(m, '\n');
2091
2092         return 0;
2093 }
2094
2095 static void *
2096 t_next(struct seq_file *m, void *v, loff_t *pos)
2097 {
2098         struct ftrace_iterator *iter = m->private;
2099         struct ftrace_ops *ops = &global_ops;
2100         struct dyn_ftrace *rec = NULL;
2101
2102         if (unlikely(ftrace_disabled))
2103                 return NULL;
2104
2105         if (iter->flags & FTRACE_ITER_HASH)
2106                 return t_hash_next(m, pos);
2107
2108         (*pos)++;
2109         iter->pos = iter->func_pos = *pos;
2110
2111         if (iter->flags & FTRACE_ITER_PRINTALL)
2112                 return t_hash_start(m, pos);
2113
2114  retry:
2115         if (iter->idx >= iter->pg->index) {
2116                 if (iter->pg->next) {
2117                         iter->pg = iter->pg->next;
2118                         iter->idx = 0;
2119                         goto retry;
2120                 }
2121         } else {
2122                 rec = &iter->pg->records[iter->idx++];
2123                 if ((rec->flags & FTRACE_FL_FREE) ||
2124
2125                     ((iter->flags & FTRACE_ITER_FILTER) &&
2126                      !(ftrace_lookup_ip(ops->filter_hash, rec->ip))) ||
2127
2128                     ((iter->flags & FTRACE_ITER_NOTRACE) &&
2129                      !ftrace_lookup_ip(ops->notrace_hash, rec->ip)) ||
2130
2131                     ((iter->flags & FTRACE_ITER_ENABLED) &&
2132                      !(rec->flags & ~FTRACE_FL_MASK))) {
2133
2134                         rec = NULL;
2135                         goto retry;
2136                 }
2137         }
2138
2139         if (!rec)
2140                 return t_hash_start(m, pos);
2141
2142         iter->func = rec;
2143
2144         return iter;
2145 }
2146
2147 static void reset_iter_read(struct ftrace_iterator *iter)
2148 {
2149         iter->pos = 0;
2150         iter->func_pos = 0;
2151         iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_HASH);
2152 }
2153
2154 static void *t_start(struct seq_file *m, loff_t *pos)
2155 {
2156         struct ftrace_iterator *iter = m->private;
2157         struct ftrace_ops *ops = &global_ops;
2158         void *p = NULL;
2159         loff_t l;
2160
2161         mutex_lock(&ftrace_lock);
2162
2163         if (unlikely(ftrace_disabled))
2164                 return NULL;
2165
2166         /*
2167          * If an lseek was done, then reset and start from beginning.
2168          */
2169         if (*pos < iter->pos)
2170                 reset_iter_read(iter);
2171
2172         /*
2173          * For set_ftrace_filter reading, if we have the filter
2174          * off, we can short cut and just print out that all
2175          * functions are enabled.
2176          */
2177         if (iter->flags & FTRACE_ITER_FILTER &&
2178             ftrace_hash_empty(ops->filter_hash)) {
2179                 if (*pos > 0)
2180                         return t_hash_start(m, pos);
2181                 iter->flags |= FTRACE_ITER_PRINTALL;
2182                 /* reset in case of seek/pread */
2183                 iter->flags &= ~FTRACE_ITER_HASH;
2184                 return iter;
2185         }
2186
2187         if (iter->flags & FTRACE_ITER_HASH)
2188                 return t_hash_start(m, pos);
2189
2190         /*
2191          * Unfortunately, we need to restart at ftrace_pages_start
2192          * every time we let go of the ftrace_mutex. This is because
2193          * those pointers can change without the lock.
2194          */
2195         iter->pg = ftrace_pages_start;
2196         iter->idx = 0;
2197         for (l = 0; l <= *pos; ) {
2198                 p = t_next(m, p, &l);
2199                 if (!p)
2200                         break;
2201         }
2202
2203         if (!p) {
2204                 if (iter->flags & FTRACE_ITER_FILTER)
2205                         return t_hash_start(m, pos);
2206
2207                 return NULL;
2208         }
2209
2210         return iter;
2211 }
2212
2213 static void t_stop(struct seq_file *m, void *p)
2214 {
2215         mutex_unlock(&ftrace_lock);
2216 }
2217
2218 static int t_show(struct seq_file *m, void *v)
2219 {
2220         struct ftrace_iterator *iter = m->private;
2221         struct dyn_ftrace *rec;
2222
2223         if (iter->flags & FTRACE_ITER_HASH)
2224                 return t_hash_show(m, iter);
2225
2226         if (iter->flags & FTRACE_ITER_PRINTALL) {
2227                 seq_printf(m, "#### all functions enabled ####\n");
2228                 return 0;
2229         }
2230
2231         rec = iter->func;
2232
2233         if (!rec)
2234                 return 0;
2235
2236         seq_printf(m, "%ps", (void *)rec->ip);
2237         if (iter->flags & FTRACE_ITER_ENABLED)
2238                 seq_printf(m, " (%ld)",
2239                            rec->flags & ~FTRACE_FL_MASK);
2240         seq_printf(m, "\n");
2241
2242         return 0;
2243 }
2244
2245 static const struct seq_operations show_ftrace_seq_ops = {
2246         .start = t_start,
2247         .next = t_next,
2248         .stop = t_stop,
2249         .show = t_show,
2250 };
2251
2252 static int
2253 ftrace_avail_open(struct inode *inode, struct file *file)
2254 {
2255         struct ftrace_iterator *iter;
2256         int ret;
2257
2258         if (unlikely(ftrace_disabled))
2259                 return -ENODEV;
2260
2261         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2262         if (!iter)
2263                 return -ENOMEM;
2264
2265         iter->pg = ftrace_pages_start;
2266
2267         ret = seq_open(file, &show_ftrace_seq_ops);
2268         if (!ret) {
2269                 struct seq_file *m = file->private_data;
2270
2271                 m->private = iter;
2272         } else {
2273                 kfree(iter);
2274         }
2275
2276         return ret;
2277 }
2278
2279 static int
2280 ftrace_enabled_open(struct inode *inode, struct file *file)
2281 {
2282         struct ftrace_iterator *iter;
2283         int ret;
2284
2285         if (unlikely(ftrace_disabled))
2286                 return -ENODEV;
2287
2288         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2289         if (!iter)
2290                 return -ENOMEM;
2291
2292         iter->pg = ftrace_pages_start;
2293         iter->flags = FTRACE_ITER_ENABLED;
2294
2295         ret = seq_open(file, &show_ftrace_seq_ops);
2296         if (!ret) {
2297                 struct seq_file *m = file->private_data;
2298
2299                 m->private = iter;
2300         } else {
2301                 kfree(iter);
2302         }
2303
2304         return ret;
2305 }
2306
2307 static void ftrace_filter_reset(struct ftrace_hash *hash)
2308 {
2309         mutex_lock(&ftrace_lock);
2310         ftrace_hash_clear(hash);
2311         mutex_unlock(&ftrace_lock);
2312 }
2313
2314 static int
2315 ftrace_regex_open(struct ftrace_ops *ops, int flag,
2316                   struct inode *inode, struct file *file)
2317 {
2318         struct ftrace_iterator *iter;
2319         struct ftrace_hash *hash;
2320         int ret = 0;
2321
2322         if (unlikely(ftrace_disabled))
2323                 return -ENODEV;
2324
2325         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2326         if (!iter)
2327                 return -ENOMEM;
2328
2329         if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
2330                 kfree(iter);
2331                 return -ENOMEM;
2332         }
2333
2334         if (flag & FTRACE_ITER_NOTRACE)
2335                 hash = ops->notrace_hash;
2336         else
2337                 hash = ops->filter_hash;
2338
2339         iter->ops = ops;
2340         iter->flags = flag;
2341
2342         if (file->f_mode & FMODE_WRITE) {
2343                 mutex_lock(&ftrace_lock);
2344                 iter->hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, hash);
2345                 mutex_unlock(&ftrace_lock);
2346
2347                 if (!iter->hash) {
2348                         trace_parser_put(&iter->parser);
2349                         kfree(iter);
2350                         return -ENOMEM;
2351                 }
2352         }
2353
2354         mutex_lock(&ftrace_regex_lock);
2355
2356         if ((file->f_mode & FMODE_WRITE) &&
2357             (file->f_flags & O_TRUNC))
2358                 ftrace_filter_reset(iter->hash);
2359
2360         if (file->f_mode & FMODE_READ) {
2361                 iter->pg = ftrace_pages_start;
2362
2363                 ret = seq_open(file, &show_ftrace_seq_ops);
2364                 if (!ret) {
2365                         struct seq_file *m = file->private_data;
2366                         m->private = iter;
2367                 } else {
2368                         /* Failed */
2369                         free_ftrace_hash(iter->hash);
2370                         trace_parser_put(&iter->parser);
2371                         kfree(iter);
2372                 }
2373         } else
2374                 file->private_data = iter;
2375         mutex_unlock(&ftrace_regex_lock);
2376
2377         return ret;
2378 }
2379
2380 static int
2381 ftrace_filter_open(struct inode *inode, struct file *file)
2382 {
2383         return ftrace_regex_open(&global_ops, FTRACE_ITER_FILTER,
2384                                  inode, file);
2385 }
2386
2387 static int
2388 ftrace_notrace_open(struct inode *inode, struct file *file)
2389 {
2390         return ftrace_regex_open(&global_ops, FTRACE_ITER_NOTRACE,
2391                                  inode, file);
2392 }
2393
2394 static int ftrace_match(char *str, char *regex, int len, int type)
2395 {
2396         int matched = 0;
2397         int slen;
2398
2399         switch (type) {
2400         case MATCH_FULL:
2401                 if (strcmp(str, regex) == 0)
2402                         matched = 1;
2403                 break;
2404         case MATCH_FRONT_ONLY:
2405                 if (strncmp(str, regex, len) == 0)
2406                         matched = 1;
2407                 break;
2408         case MATCH_MIDDLE_ONLY:
2409                 if (strstr(str, regex))
2410                         matched = 1;
2411                 break;
2412         case MATCH_END_ONLY:
2413                 slen = strlen(str);
2414                 if (slen >= len && memcmp(str + slen - len, regex, len) == 0)
2415                         matched = 1;
2416                 break;
2417         }
2418
2419         return matched;
2420 }
2421
2422 static int
2423 enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int not)
2424 {
2425         struct ftrace_func_entry *entry;
2426         int ret = 0;
2427
2428         entry = ftrace_lookup_ip(hash, rec->ip);
2429         if (not) {
2430                 /* Do nothing if it doesn't exist */
2431                 if (!entry)
2432                         return 0;
2433
2434                 free_hash_entry(hash, entry);
2435         } else {
2436                 /* Do nothing if it exists */
2437                 if (entry)
2438                         return 0;
2439
2440                 ret = add_hash_entry(hash, rec->ip);
2441         }
2442         return ret;
2443 }
2444
2445 static int
2446 ftrace_match_record(struct dyn_ftrace *rec, char *mod,
2447                     char *regex, int len, int type)
2448 {
2449         char str[KSYM_SYMBOL_LEN];
2450         char *modname;
2451
2452         kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
2453
2454         if (mod) {
2455                 /* module lookup requires matching the module */
2456                 if (!modname || strcmp(modname, mod))
2457                         return 0;
2458
2459                 /* blank search means to match all funcs in the mod */
2460                 if (!len)
2461                         return 1;
2462         }
2463
2464         return ftrace_match(str, regex, len, type);
2465 }
2466
2467 static int
2468 match_records(struct ftrace_hash *hash, char *buff,
2469               int len, char *mod, int not)
2470 {
2471         unsigned search_len = 0;
2472         struct ftrace_page *pg;
2473         struct dyn_ftrace *rec;
2474         int type = MATCH_FULL;
2475         char *search = buff;
2476         int found = 0;
2477         int ret;
2478
2479         if (len) {
2480                 type = filter_parse_regex(buff, len, &search, &not);
2481                 search_len = strlen(search);
2482         }
2483
2484         mutex_lock(&ftrace_lock);
2485
2486         if (unlikely(ftrace_disabled))
2487                 goto out_unlock;
2488
2489         do_for_each_ftrace_rec(pg, rec) {
2490
2491                 if (ftrace_match_record(rec, mod, search, search_len, type)) {
2492                         ret = enter_record(hash, rec, not);
2493                         if (ret < 0) {
2494                                 found = ret;
2495                                 goto out_unlock;
2496                         }
2497                         found = 1;
2498                 }
2499         } while_for_each_ftrace_rec();
2500  out_unlock:
2501         mutex_unlock(&ftrace_lock);
2502
2503         return found;
2504 }
2505
2506 static int
2507 ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
2508 {
2509         return match_records(hash, buff, len, NULL, 0);
2510 }
2511
2512 static int
2513 ftrace_match_module_records(struct ftrace_hash *hash, char *buff, char *mod)
2514 {
2515         int not = 0;
2516
2517         /* blank or '*' mean the same */
2518         if (strcmp(buff, "*") == 0)
2519                 buff[0] = 0;
2520
2521         /* handle the case of 'dont filter this module' */
2522         if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
2523                 buff[0] = 0;
2524                 not = 1;
2525         }
2526
2527         return match_records(hash, buff, strlen(buff), mod, not);
2528 }
2529
2530 /*
2531  * We register the module command as a template to show others how
2532  * to register the a command as well.
2533  */
2534
2535 static int
2536 ftrace_mod_callback(struct ftrace_hash *hash,
2537                     char *func, char *cmd, char *param, int enable)
2538 {
2539         char *mod;
2540         int ret = -EINVAL;
2541
2542         /*
2543          * cmd == 'mod' because we only registered this func
2544          * for the 'mod' ftrace_func_command.
2545          * But if you register one func with multiple commands,
2546          * you can tell which command was used by the cmd
2547          * parameter.
2548          */
2549
2550         /* we must have a module name */
2551         if (!param)
2552                 return ret;
2553
2554         mod = strsep(&param, ":");
2555         if (!strlen(mod))
2556                 return ret;
2557
2558         ret = ftrace_match_module_records(hash, func, mod);
2559         if (!ret)
2560                 ret = -EINVAL;
2561         if (ret < 0)
2562                 return ret;
2563
2564         return 0;
2565 }
2566
2567 static struct ftrace_func_command ftrace_mod_cmd = {
2568         .name                   = "mod",
2569         .func                   = ftrace_mod_callback,
2570 };
2571
2572 static int __init ftrace_mod_cmd_init(void)
2573 {
2574         return register_ftrace_command(&ftrace_mod_cmd);
2575 }
2576 device_initcall(ftrace_mod_cmd_init);
2577
2578 static void
2579 function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
2580 {
2581         struct ftrace_func_probe *entry;
2582         struct hlist_head *hhd;
2583         struct hlist_node *n;
2584         unsigned long key;
2585
2586         key = hash_long(ip, FTRACE_HASH_BITS);
2587
2588         hhd = &ftrace_func_hash[key];
2589
2590         if (hlist_empty(hhd))
2591                 return;
2592
2593         /*
2594          * Disable preemption for these calls to prevent a RCU grace
2595          * period. This syncs the hash iteration and freeing of items
2596          * on the hash. rcu_read_lock is too dangerous here.
2597          */
2598         preempt_disable_notrace();
2599         hlist_for_each_entry_rcu(entry, n, hhd, node) {
2600                 if (entry->ip == ip)
2601                         entry->ops->func(ip, parent_ip, &entry->data);
2602         }
2603         preempt_enable_notrace();
2604 }
2605
2606 static struct ftrace_ops trace_probe_ops __read_mostly =
2607 {
2608         .func           = function_trace_probe_call,
2609 };
2610
2611 static int ftrace_probe_registered;
2612
2613 static void __enable_ftrace_function_probe(void)
2614 {
2615         int ret;
2616         int i;
2617
2618         if (ftrace_probe_registered)
2619                 return;
2620
2621         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2622                 struct hlist_head *hhd = &ftrace_func_hash[i];
2623                 if (hhd->first)
2624                         break;
2625         }
2626         /* Nothing registered? */
2627         if (i == FTRACE_FUNC_HASHSIZE)
2628                 return;
2629
2630         ret = ftrace_startup(&trace_probe_ops, 0);
2631
2632         ftrace_probe_registered = 1;
2633 }
2634
2635 static void __disable_ftrace_function_probe(void)
2636 {
2637         int i;
2638
2639         if (!ftrace_probe_registered)
2640                 return;
2641
2642         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2643                 struct hlist_head *hhd = &ftrace_func_hash[i];
2644                 if (hhd->first)
2645                         return;
2646         }
2647
2648         /* no more funcs left */
2649         ftrace_shutdown(&trace_probe_ops, 0);
2650
2651         ftrace_probe_registered = 0;
2652 }
2653
2654
2655 static void ftrace_free_entry_rcu(struct rcu_head *rhp)
2656 {
2657         struct ftrace_func_probe *entry =
2658                 container_of(rhp, struct ftrace_func_probe, rcu);
2659
2660         if (entry->ops->free)
2661                 entry->ops->free(&entry->data);
2662         kfree(entry);
2663 }
2664
2665
2666 int
2667 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2668                               void *data)
2669 {
2670         struct ftrace_func_probe *entry;
2671         struct ftrace_page *pg;
2672         struct dyn_ftrace *rec;
2673         int type, len, not;
2674         unsigned long key;
2675         int count = 0;
2676         char *search;
2677
2678         type = filter_parse_regex(glob, strlen(glob), &search, &not);
2679         len = strlen(search);
2680
2681         /* we do not support '!' for function probes */
2682         if (WARN_ON(not))
2683                 return -EINVAL;
2684
2685         mutex_lock(&ftrace_lock);
2686
2687         if (unlikely(ftrace_disabled))
2688                 goto out_unlock;
2689
2690         do_for_each_ftrace_rec(pg, rec) {
2691
2692                 if (!ftrace_match_record(rec, NULL, search, len, type))
2693                         continue;
2694
2695                 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
2696                 if (!entry) {
2697                         /* If we did not process any, then return error */
2698                         if (!count)
2699                                 count = -ENOMEM;
2700                         goto out_unlock;
2701                 }
2702
2703                 count++;
2704
2705                 entry->data = data;
2706
2707                 /*
2708                  * The caller might want to do something special
2709                  * for each function we find. We call the callback
2710                  * to give the caller an opportunity to do so.
2711                  */
2712                 if (ops->callback) {
2713                         if (ops->callback(rec->ip, &entry->data) < 0) {
2714                                 /* caller does not like this func */
2715                                 kfree(entry);
2716                                 continue;
2717                         }
2718                 }
2719
2720                 entry->ops = ops;
2721                 entry->ip = rec->ip;
2722
2723                 key = hash_long(entry->ip, FTRACE_HASH_BITS);
2724                 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
2725
2726         } while_for_each_ftrace_rec();
2727         __enable_ftrace_function_probe();
2728
2729  out_unlock:
2730         mutex_unlock(&ftrace_lock);
2731
2732         return count;
2733 }
2734
2735 enum {
2736         PROBE_TEST_FUNC         = 1,
2737         PROBE_TEST_DATA         = 2
2738 };
2739
2740 static void
2741 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2742                                   void *data, int flags)
2743 {
2744         struct ftrace_func_probe *entry;
2745         struct hlist_node *n, *tmp;
2746         char str[KSYM_SYMBOL_LEN];
2747         int type = MATCH_FULL;
2748         int i, len = 0;
2749         char *search;
2750
2751         if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
2752                 glob = NULL;
2753         else if (glob) {
2754                 int not;
2755
2756                 type = filter_parse_regex(glob, strlen(glob), &search, &not);
2757                 len = strlen(search);
2758
2759                 /* we do not support '!' for function probes */
2760                 if (WARN_ON(not))
2761                         return;
2762         }
2763
2764         mutex_lock(&ftrace_lock);
2765         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2766                 struct hlist_head *hhd = &ftrace_func_hash[i];
2767
2768                 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
2769
2770                         /* break up if statements for readability */
2771                         if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
2772                                 continue;
2773
2774                         if ((flags & PROBE_TEST_DATA) && entry->data != data)
2775                                 continue;
2776
2777                         /* do this last, since it is the most expensive */
2778                         if (glob) {
2779                                 kallsyms_lookup(entry->ip, NULL, NULL,
2780                                                 NULL, str);
2781                                 if (!ftrace_match(str, glob, len, type))
2782                                         continue;
2783                         }
2784
2785                         hlist_del_rcu(&entry->node);
2786                         call_rcu_sched(&entry->rcu, ftrace_free_entry_rcu);
2787                 }
2788         }
2789         __disable_ftrace_function_probe();
2790         mutex_unlock(&ftrace_lock);
2791 }
2792
2793 void
2794 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2795                                 void *data)
2796 {
2797         __unregister_ftrace_function_probe(glob, ops, data,
2798                                           PROBE_TEST_FUNC | PROBE_TEST_DATA);
2799 }
2800
2801 void
2802 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
2803 {
2804         __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
2805 }
2806
2807 void unregister_ftrace_function_probe_all(char *glob)
2808 {
2809         __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
2810 }
2811
2812 static LIST_HEAD(ftrace_commands);
2813 static DEFINE_MUTEX(ftrace_cmd_mutex);
2814
2815 int register_ftrace_command(struct ftrace_func_command *cmd)
2816 {
2817         struct ftrace_func_command *p;
2818         int ret = 0;
2819
2820         mutex_lock(&ftrace_cmd_mutex);
2821         list_for_each_entry(p, &ftrace_commands, list) {
2822                 if (strcmp(cmd->name, p->name) == 0) {
2823                         ret = -EBUSY;
2824                         goto out_unlock;
2825                 }
2826         }
2827         list_add(&cmd->list, &ftrace_commands);
2828  out_unlock:
2829         mutex_unlock(&ftrace_cmd_mutex);
2830
2831         return ret;
2832 }
2833
2834 int unregister_ftrace_command(struct ftrace_func_command *cmd)
2835 {
2836         struct ftrace_func_command *p, *n;
2837         int ret = -ENODEV;
2838
2839         mutex_lock(&ftrace_cmd_mutex);
2840         list_for_each_entry_safe(p, n, &ftrace_commands, list) {
2841                 if (strcmp(cmd->name, p->name) == 0) {
2842                         ret = 0;
2843                         list_del_init(&p->list);
2844                         goto out_unlock;
2845                 }
2846         }
2847  out_unlock:
2848         mutex_unlock(&ftrace_cmd_mutex);
2849
2850         return ret;
2851 }
2852
2853 static int ftrace_process_regex(struct ftrace_hash *hash,
2854                                 char *buff, int len, int enable)
2855 {
2856         char *func, *command, *next = buff;
2857         struct ftrace_func_command *p;
2858         int ret = -EINVAL;
2859
2860         func = strsep(&next, ":");
2861
2862         if (!next) {
2863                 ret = ftrace_match_records(hash, func, len);
2864                 if (!ret)
2865                         ret = -EINVAL;
2866                 if (ret < 0)
2867                         return ret;
2868                 return 0;
2869         }
2870
2871         /* command found */
2872
2873         command = strsep(&next, ":");
2874
2875         mutex_lock(&ftrace_cmd_mutex);
2876         list_for_each_entry(p, &ftrace_commands, list) {
2877                 if (strcmp(p->name, command) == 0) {
2878                         ret = p->func(hash, func, command, next, enable);
2879                         goto out_unlock;
2880                 }
2881         }
2882  out_unlock:
2883         mutex_unlock(&ftrace_cmd_mutex);
2884
2885         return ret;
2886 }
2887
2888 static ssize_t
2889 ftrace_regex_write(struct file *file, const char __user *ubuf,
2890                    size_t cnt, loff_t *ppos, int enable)
2891 {
2892         struct ftrace_iterator *iter;
2893         struct trace_parser *parser;
2894         ssize_t ret, read;
2895
2896         if (!cnt)
2897                 return 0;
2898
2899         mutex_lock(&ftrace_regex_lock);
2900
2901         ret = -ENODEV;
2902         if (unlikely(ftrace_disabled))
2903                 goto out_unlock;
2904
2905         if (file->f_mode & FMODE_READ) {
2906                 struct seq_file *m = file->private_data;
2907                 iter = m->private;
2908         } else
2909                 iter = file->private_data;
2910
2911         parser = &iter->parser;
2912         read = trace_get_user(parser, ubuf, cnt, ppos);
2913
2914         if (read >= 0 && trace_parser_loaded(parser) &&
2915             !trace_parser_cont(parser)) {
2916                 ret = ftrace_process_regex(iter->hash, parser->buffer,
2917                                            parser->idx, enable);
2918                 trace_parser_clear(parser);
2919                 if (ret)
2920                         goto out_unlock;
2921         }
2922
2923         ret = read;
2924 out_unlock:
2925         mutex_unlock(&ftrace_regex_lock);
2926
2927         return ret;
2928 }
2929
2930 static ssize_t
2931 ftrace_filter_write(struct file *file, const char __user *ubuf,
2932                     size_t cnt, loff_t *ppos)
2933 {
2934         return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
2935 }
2936
2937 static ssize_t
2938 ftrace_notrace_write(struct file *file, const char __user *ubuf,
2939                      size_t cnt, loff_t *ppos)
2940 {
2941         return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
2942 }
2943
2944 static int
2945 ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
2946                  int reset, int enable)
2947 {
2948         struct ftrace_hash **orig_hash;
2949         struct ftrace_hash *hash;
2950         int ret;
2951
2952         /* All global ops uses the global ops filters */
2953         if (ops->flags & FTRACE_OPS_FL_GLOBAL)
2954                 ops = &global_ops;
2955
2956         if (unlikely(ftrace_disabled))
2957                 return -ENODEV;
2958
2959         if (enable)
2960                 orig_hash = &ops->filter_hash;
2961         else
2962                 orig_hash = &ops->notrace_hash;
2963
2964         hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
2965         if (!hash)
2966                 return -ENOMEM;
2967
2968         mutex_lock(&ftrace_regex_lock);
2969         if (reset)
2970                 ftrace_filter_reset(hash);
2971         if (buf)
2972                 ftrace_match_records(hash, buf, len);
2973
2974         mutex_lock(&ftrace_lock);
2975         ret = ftrace_hash_move(ops, enable, orig_hash, hash);
2976         if (!ret && ops->flags & FTRACE_OPS_FL_ENABLED
2977             && ftrace_enabled)
2978                 ftrace_run_update_code(FTRACE_UPDATE_CALLS);
2979
2980         mutex_unlock(&ftrace_lock);
2981
2982         mutex_unlock(&ftrace_regex_lock);
2983
2984         free_ftrace_hash(hash);
2985         return ret;
2986 }
2987
2988 /**
2989  * ftrace_set_filter - set a function to filter on in ftrace
2990  * @ops - the ops to set the filter with
2991  * @buf - the string that holds the function filter text.
2992  * @len - the length of the string.
2993  * @reset - non zero to reset all filters before applying this filter.
2994  *
2995  * Filters denote which functions should be enabled when tracing is enabled.
2996  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
2997  */
2998 void ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
2999                        int len, int reset)
3000 {
3001         ftrace_set_regex(ops, buf, len, reset, 1);
3002 }
3003 EXPORT_SYMBOL_GPL(ftrace_set_filter);
3004
3005 /**
3006  * ftrace_set_notrace - set a function to not trace in ftrace
3007  * @ops - the ops to set the notrace filter with
3008  * @buf - the string that holds the function notrace text.
3009  * @len - the length of the string.
3010  * @reset - non zero to reset all filters before applying this filter.
3011  *
3012  * Notrace Filters denote which functions should not be enabled when tracing
3013  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
3014  * for tracing.
3015  */
3016 void ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
3017                         int len, int reset)
3018 {
3019         ftrace_set_regex(ops, buf, len, reset, 0);
3020 }
3021 EXPORT_SYMBOL_GPL(ftrace_set_notrace);
3022 /**
3023  * ftrace_set_filter - set a function to filter on in ftrace
3024  * @ops - the ops to set the filter with
3025  * @buf - the string that holds the function filter text.
3026  * @len - the length of the string.
3027  * @reset - non zero to reset all filters before applying this filter.
3028  *
3029  * Filters denote which functions should be enabled when tracing is enabled.
3030  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
3031  */
3032 void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
3033 {
3034         ftrace_set_regex(&global_ops, buf, len, reset, 1);
3035 }
3036 EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
3037
3038 /**
3039  * ftrace_set_notrace - set a function to not trace in ftrace
3040  * @ops - the ops to set the notrace filter with
3041  * @buf - the string that holds the function notrace text.
3042  * @len - the length of the string.
3043  * @reset - non zero to reset all filters before applying this filter.
3044  *
3045  * Notrace Filters denote which functions should not be enabled when tracing
3046  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
3047  * for tracing.
3048  */
3049 void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
3050 {
3051         ftrace_set_regex(&global_ops, buf, len, reset, 0);
3052 }
3053 EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
3054
3055 /*
3056  * command line interface to allow users to set filters on boot up.
3057  */
3058 #define FTRACE_FILTER_SIZE              COMMAND_LINE_SIZE
3059 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
3060 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
3061
3062 static int __init set_ftrace_notrace(char *str)
3063 {
3064         strncpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
3065         return 1;
3066 }
3067 __setup("ftrace_notrace=", set_ftrace_notrace);
3068
3069 static int __init set_ftrace_filter(char *str)
3070 {
3071         strncpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
3072         return 1;
3073 }
3074 __setup("ftrace_filter=", set_ftrace_filter);
3075
3076 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3077 static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
3078 static int ftrace_set_func(unsigned long *array, int *idx, char *buffer);
3079
3080 static int __init set_graph_function(char *str)
3081 {
3082         strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
3083         return 1;
3084 }
3085 __setup("ftrace_graph_filter=", set_graph_function);
3086
3087 static void __init set_ftrace_early_graph(char *buf)
3088 {
3089         int ret;
3090         char *func;
3091
3092         while (buf) {
3093                 func = strsep(&buf, ",");
3094                 /* we allow only one expression at a time */
3095                 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
3096                                       func);
3097                 if (ret)
3098                         printk(KERN_DEBUG "ftrace: function %s not "
3099                                           "traceable\n", func);
3100         }
3101 }
3102 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3103
3104 static void __init
3105 set_ftrace_early_filter(struct ftrace_ops *ops, char *buf, int enable)
3106 {
3107         char *func;
3108
3109         while (buf) {
3110                 func = strsep(&buf, ",");
3111                 ftrace_set_regex(ops, func, strlen(func), 0, enable);
3112         }
3113 }
3114
3115 static void __init set_ftrace_early_filters(void)
3116 {
3117         if (ftrace_filter_buf[0])
3118                 set_ftrace_early_filter(&global_ops, ftrace_filter_buf, 1);
3119         if (ftrace_notrace_buf[0])
3120                 set_ftrace_early_filter(&global_ops, ftrace_notrace_buf, 0);
3121 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3122         if (ftrace_graph_buf[0])
3123                 set_ftrace_early_graph(ftrace_graph_buf);
3124 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3125 }
3126
3127 static int
3128 ftrace_regex_release(struct inode *inode, struct file *file)
3129 {
3130         struct seq_file *m = (struct seq_file *)file->private_data;
3131         struct ftrace_iterator *iter;
3132         struct ftrace_hash **orig_hash;
3133         struct trace_parser *parser;
3134         int filter_hash;
3135         int ret;
3136
3137         mutex_lock(&ftrace_regex_lock);
3138         if (file->f_mode & FMODE_READ) {
3139                 iter = m->private;
3140
3141                 seq_release(inode, file);
3142         } else
3143                 iter = file->private_data;
3144
3145         parser = &iter->parser;
3146         if (trace_parser_loaded(parser)) {
3147                 parser->buffer[parser->idx] = 0;
3148                 ftrace_match_records(iter->hash, parser->buffer, parser->idx);
3149         }
3150
3151         trace_parser_put(parser);
3152
3153         if (file->f_mode & FMODE_WRITE) {
3154                 filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
3155
3156                 if (filter_hash)
3157                         orig_hash = &iter->ops->filter_hash;
3158                 else
3159                         orig_hash = &iter->ops->notrace_hash;
3160
3161                 mutex_lock(&ftrace_lock);
3162                 ret = ftrace_hash_move(iter->ops, filter_hash,
3163                                        orig_hash, iter->hash);
3164                 if (!ret && (iter->ops->flags & FTRACE_OPS_FL_ENABLED)
3165                     && ftrace_enabled)
3166                         ftrace_run_update_code(FTRACE_UPDATE_CALLS);
3167
3168                 mutex_unlock(&ftrace_lock);
3169         }
3170         free_ftrace_hash(iter->hash);
3171         kfree(iter);
3172
3173         mutex_unlock(&ftrace_regex_lock);
3174         return 0;
3175 }
3176
3177 static const struct file_operations ftrace_avail_fops = {
3178         .open = ftrace_avail_open,
3179         .read = seq_read,
3180         .llseek = seq_lseek,
3181         .release = seq_release_private,
3182 };
3183
3184 static const struct file_operations ftrace_enabled_fops = {
3185         .open = ftrace_enabled_open,
3186         .read = seq_read,
3187         .llseek = seq_lseek,
3188         .release = seq_release_private,
3189 };
3190
3191 static const struct file_operations ftrace_filter_fops = {
3192         .open = ftrace_filter_open,
3193         .read = seq_read,
3194         .write = ftrace_filter_write,
3195         .llseek = ftrace_filter_lseek,
3196         .release = ftrace_regex_release,
3197 };
3198
3199 static const struct file_operations ftrace_notrace_fops = {
3200         .open = ftrace_notrace_open,
3201         .read = seq_read,
3202         .write = ftrace_notrace_write,
3203         .llseek = ftrace_filter_lseek,
3204         .release = ftrace_regex_release,
3205 };
3206
3207 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3208
3209 static DEFINE_MUTEX(graph_lock);
3210
3211 int ftrace_graph_count;
3212 int ftrace_graph_filter_enabled;
3213 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
3214
3215 static void *
3216 __g_next(struct seq_file *m, loff_t *pos)
3217 {
3218         if (*pos >= ftrace_graph_count)
3219                 return NULL;
3220         return &ftrace_graph_funcs[*pos];
3221 }
3222
3223 static void *
3224 g_next(struct seq_file *m, void *v, loff_t *pos)
3225 {
3226         (*pos)++;
3227         return __g_next(m, pos);
3228 }
3229
3230 static void *g_start(struct seq_file *m, loff_t *pos)
3231 {
3232         mutex_lock(&graph_lock);
3233
3234         /* Nothing, tell g_show to print all functions are enabled */
3235         if (!ftrace_graph_filter_enabled && !*pos)
3236                 return (void *)1;
3237
3238         return __g_next(m, pos);
3239 }
3240
3241 static void g_stop(struct seq_file *m, void *p)
3242 {
3243         mutex_unlock(&graph_lock);
3244 }
3245
3246 static int g_show(struct seq_file *m, void *v)
3247 {
3248         unsigned long *ptr = v;
3249
3250         if (!ptr)
3251                 return 0;
3252
3253         if (ptr == (unsigned long *)1) {
3254                 seq_printf(m, "#### all functions enabled ####\n");
3255                 return 0;
3256         }
3257
3258         seq_printf(m, "%ps\n", (void *)*ptr);
3259
3260         return 0;
3261 }
3262
3263 static const struct seq_operations ftrace_graph_seq_ops = {
3264         .start = g_start,
3265         .next = g_next,
3266         .stop = g_stop,
3267         .show = g_show,
3268 };
3269
3270 static int
3271 ftrace_graph_open(struct inode *inode, struct file *file)
3272 {
3273         int ret = 0;
3274
3275         if (unlikely(ftrace_disabled))
3276                 return -ENODEV;
3277
3278         mutex_lock(&graph_lock);
3279         if ((file->f_mode & FMODE_WRITE) &&
3280             (file->f_flags & O_TRUNC)) {
3281                 ftrace_graph_filter_enabled = 0;
3282                 ftrace_graph_count = 0;
3283                 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
3284         }
3285         mutex_unlock(&graph_lock);
3286
3287         if (file->f_mode & FMODE_READ)
3288                 ret = seq_open(file, &ftrace_graph_seq_ops);
3289
3290         return ret;
3291 }
3292
3293 static int
3294 ftrace_graph_release(struct inode *inode, struct file *file)
3295 {
3296         if (file->f_mode & FMODE_READ)
3297                 seq_release(inode, file);
3298         return 0;
3299 }
3300
3301 static int
3302 ftrace_set_func(unsigned long *array, int *idx, char *buffer)
3303 {
3304         struct dyn_ftrace *rec;
3305         struct ftrace_page *pg;
3306         int search_len;
3307         int fail = 1;
3308         int type, not;
3309         char *search;
3310         bool exists;
3311         int i;
3312
3313         /* decode regex */
3314         type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
3315         if (!not && *idx >= FTRACE_GRAPH_MAX_FUNCS)
3316                 return -EBUSY;
3317
3318         search_len = strlen(search);
3319
3320         mutex_lock(&ftrace_lock);
3321
3322         if (unlikely(ftrace_disabled)) {
3323                 mutex_unlock(&ftrace_lock);
3324                 return -ENODEV;
3325         }
3326
3327         do_for_each_ftrace_rec(pg, rec) {
3328
3329                 if (rec->flags & FTRACE_FL_FREE)
3330                         continue;
3331
3332                 if (ftrace_match_record(rec, NULL, search, search_len, type)) {
3333                         /* if it is in the array */
3334                         exists = false;
3335                         for (i = 0; i < *idx; i++) {
3336                                 if (array[i] == rec->ip) {
3337                                         exists = true;
3338                                         break;
3339                                 }
3340                         }
3341
3342                         if (!not) {
3343                                 fail = 0;
3344                                 if (!exists) {
3345                                         array[(*idx)++] = rec->ip;
3346                                         if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
3347                                                 goto out;
3348                                 }
3349                         } else {
3350                                 if (exists) {
3351                                         array[i] = array[--(*idx)];
3352                                         array[*idx] = 0;
3353                                         fail = 0;
3354                                 }
3355                         }
3356                 }
3357         } while_for_each_ftrace_rec();
3358 out:
3359         mutex_unlock(&ftrace_lock);
3360
3361         if (fail)
3362                 return -EINVAL;
3363
3364         ftrace_graph_filter_enabled = !!(*idx);
3365
3366         return 0;
3367 }
3368
3369 static ssize_t
3370 ftrace_graph_write(struct file *file, const char __user *ubuf,
3371                    size_t cnt, loff_t *ppos)
3372 {
3373         struct trace_parser parser;
3374         ssize_t read, ret;
3375
3376         if (!cnt)
3377                 return 0;
3378
3379         mutex_lock(&graph_lock);
3380
3381         if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) {
3382                 ret = -ENOMEM;
3383                 goto out_unlock;
3384         }
3385
3386         read = trace_get_user(&parser, ubuf, cnt, ppos);
3387
3388         if (read >= 0 && trace_parser_loaded((&parser))) {
3389                 parser.buffer[parser.idx] = 0;
3390
3391                 /* we allow only one expression at a time */
3392                 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
3393                                         parser.buffer);
3394                 if (ret)
3395                         goto out_free;
3396         }
3397
3398         ret = read;
3399
3400 out_free:
3401         trace_parser_put(&parser);
3402 out_unlock:
3403         mutex_unlock(&graph_lock);
3404
3405         return ret;
3406 }
3407
3408 static const struct file_operations ftrace_graph_fops = {
3409         .open           = ftrace_graph_open,
3410         .read           = seq_read,
3411         .write          = ftrace_graph_write,
3412         .llseek         = ftrace_filter_lseek,
3413         .release        = ftrace_graph_release,
3414 };
3415 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3416
3417 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
3418 {
3419
3420         trace_create_file("available_filter_functions", 0444,
3421                         d_tracer, NULL, &ftrace_avail_fops);
3422
3423         trace_create_file("enabled_functions", 0444,
3424                         d_tracer, NULL, &ftrace_enabled_fops);
3425
3426         trace_create_file("set_ftrace_filter", 0644, d_tracer,
3427                         NULL, &ftrace_filter_fops);
3428
3429         trace_create_file("set_ftrace_notrace", 0644, d_tracer,
3430                                     NULL, &ftrace_notrace_fops);
3431
3432 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3433         trace_create_file("set_graph_function", 0444, d_tracer,
3434                                     NULL,
3435                                     &ftrace_graph_fops);
3436 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3437
3438         return 0;
3439 }
3440
3441 static int ftrace_process_locs(struct module *mod,
3442                                unsigned long *start,
3443                                unsigned long *end)
3444 {
3445         unsigned long *p;
3446         unsigned long addr;
3447         unsigned long flags = 0; /* Shut up gcc */
3448
3449         mutex_lock(&ftrace_lock);
3450         p = start;
3451         while (p < end) {
3452                 addr = ftrace_call_adjust(*p++);
3453                 /*
3454                  * Some architecture linkers will pad between
3455                  * the different mcount_loc sections of different
3456                  * object files to satisfy alignments.
3457                  * Skip any NULL pointers.
3458                  */
3459                 if (!addr)
3460                         continue;
3461                 ftrace_record_ip(addr);
3462         }
3463
3464         /*
3465          * We only need to disable interrupts on start up
3466          * because we are modifying code that an interrupt
3467          * may execute, and the modification is not atomic.
3468          * But for modules, nothing runs the code we modify
3469          * until we are finished with it, and there's no
3470          * reason to cause large interrupt latencies while we do it.
3471          */
3472         if (!mod)
3473                 local_irq_save(flags);
3474         ftrace_update_code(mod);
3475         if (!mod)
3476                 local_irq_restore(flags);
3477         mutex_unlock(&ftrace_lock);
3478
3479         return 0;
3480 }
3481
3482 #ifdef CONFIG_MODULES
3483 void ftrace_release_mod(struct module *mod)
3484 {
3485         struct dyn_ftrace *rec;
3486         struct ftrace_page *pg;
3487
3488         mutex_lock(&ftrace_lock);
3489
3490         if (ftrace_disabled)
3491                 goto out_unlock;
3492
3493         do_for_each_ftrace_rec(pg, rec) {
3494                 if (within_module_core(rec->ip, mod)) {
3495                         /*
3496                          * rec->ip is changed in ftrace_free_rec()
3497                          * It should not between s and e if record was freed.
3498                          */
3499                         FTRACE_WARN_ON(rec->flags & FTRACE_FL_FREE);
3500                         ftrace_free_rec(rec);
3501                 }
3502         } while_for_each_ftrace_rec();
3503  out_unlock:
3504         mutex_unlock(&ftrace_lock);
3505 }
3506
3507 static void ftrace_init_module(struct module *mod,
3508                                unsigned long *start, unsigned long *end)
3509 {
3510         if (ftrace_disabled || start == end)
3511                 return;
3512         ftrace_process_locs(mod, start, end);
3513 }
3514
3515 static int ftrace_module_notify_enter(struct notifier_block *self,
3516                                       unsigned long val, void *data)
3517 {
3518         struct module *mod = data;
3519
3520         if (val == MODULE_STATE_COMING)
3521                 ftrace_init_module(mod, mod->ftrace_callsites,
3522                                    mod->ftrace_callsites +
3523                                    mod->num_ftrace_callsites);
3524         return 0;
3525 }
3526
3527 static int ftrace_module_notify_exit(struct notifier_block *self,
3528                                      unsigned long val, void *data)
3529 {
3530         struct module *mod = data;
3531
3532         if (val == MODULE_STATE_GOING)
3533                 ftrace_release_mod(mod);
3534
3535         return 0;
3536 }
3537 #else
3538 static int ftrace_module_notify_enter(struct notifier_block *self,
3539                                       unsigned long val, void *data)
3540 {
3541         return 0;
3542 }
3543 static int ftrace_module_notify_exit(struct notifier_block *self,
3544                                      unsigned long val, void *data)
3545 {
3546         return 0;
3547 }
3548 #endif /* CONFIG_MODULES */
3549
3550 struct notifier_block ftrace_module_enter_nb = {
3551         .notifier_call = ftrace_module_notify_enter,
3552         .priority = INT_MAX,    /* Run before anything that can use kprobes */
3553 };
3554
3555 struct notifier_block ftrace_module_exit_nb = {
3556         .notifier_call = ftrace_module_notify_exit,
3557         .priority = INT_MIN,    /* Run after anything that can remove kprobes */
3558 };
3559
3560 extern unsigned long __start_mcount_loc[];
3561 extern unsigned long __stop_mcount_loc[];
3562
3563 void __init ftrace_init(void)
3564 {
3565         unsigned long count, addr, flags;
3566         int ret;
3567
3568         /* Keep the ftrace pointer to the stub */
3569         addr = (unsigned long)ftrace_stub;
3570
3571         local_irq_save(flags);
3572         ftrace_dyn_arch_init(&addr);
3573         local_irq_restore(flags);
3574
3575         /* ftrace_dyn_arch_init places the return code in addr */
3576         if (addr)
3577                 goto failed;
3578
3579         count = __stop_mcount_loc - __start_mcount_loc;
3580
3581         ret = ftrace_dyn_table_alloc(count);
3582         if (ret)
3583                 goto failed;
3584
3585         last_ftrace_enabled = ftrace_enabled = 1;
3586
3587         ret = ftrace_process_locs(NULL,
3588                                   __start_mcount_loc,
3589                                   __stop_mcount_loc);
3590
3591         ret = register_module_notifier(&ftrace_module_enter_nb);
3592         if (ret)
3593                 pr_warning("Failed to register trace ftrace module enter notifier\n");
3594
3595         ret = register_module_notifier(&ftrace_module_exit_nb);
3596         if (ret)
3597                 pr_warning("Failed to register trace ftrace module exit notifier\n");
3598
3599         set_ftrace_early_filters();
3600
3601         return;
3602  failed:
3603         ftrace_disabled = 1;
3604 }
3605
3606 #else
3607
3608 static struct ftrace_ops global_ops = {
3609         .func                   = ftrace_stub,
3610 };
3611
3612 static int __init ftrace_nodyn_init(void)
3613 {
3614         ftrace_enabled = 1;
3615         return 0;
3616 }
3617 device_initcall(ftrace_nodyn_init);
3618
3619 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
3620 static inline void ftrace_startup_enable(int command) { }
3621 /* Keep as macros so we do not need to define the commands */
3622 # define ftrace_startup(ops, command)                                   \
3623         ({                                                              \
3624                 int ___ret = __register_ftrace_function(ops);           \
3625                 if (!___ret)                                            \
3626                         (ops)->flags |= FTRACE_OPS_FL_ENABLED;          \
3627                 ___ret;                                                 \
3628         })
3629 # define ftrace_shutdown(ops, command) __unregister_ftrace_function(ops)
3630
3631 # define ftrace_startup_sysctl()        do { } while (0)
3632 # define ftrace_shutdown_sysctl()       do { } while (0)
3633
3634 static inline int
3635 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
3636 {
3637         return 1;
3638 }
3639
3640 #endif /* CONFIG_DYNAMIC_FTRACE */
3641
3642 static void
3643 ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip)
3644 {
3645         struct ftrace_ops *op;
3646
3647         if (unlikely(trace_recursion_test(TRACE_INTERNAL_BIT)))
3648                 return;
3649
3650         trace_recursion_set(TRACE_INTERNAL_BIT);
3651         /*
3652          * Some of the ops may be dynamically allocated,
3653          * they must be freed after a synchronize_sched().
3654          */
3655         preempt_disable_notrace();
3656         op = rcu_dereference_raw(ftrace_ops_list);
3657         while (op != &ftrace_list_end) {
3658                 if (ftrace_ops_test(op, ip))
3659                         op->func(ip, parent_ip);
3660                 op = rcu_dereference_raw(op->next);
3661         };
3662         preempt_enable_notrace();
3663         trace_recursion_clear(TRACE_INTERNAL_BIT);
3664 }
3665
3666 static void clear_ftrace_swapper(void)
3667 {
3668         struct task_struct *p;
3669         int cpu;
3670
3671         get_online_cpus();
3672         for_each_online_cpu(cpu) {
3673                 p = idle_task(cpu);
3674                 clear_tsk_trace_trace(p);
3675         }
3676         put_online_cpus();
3677 }
3678
3679 static void set_ftrace_swapper(void)
3680 {
3681         struct task_struct *p;
3682         int cpu;
3683
3684         get_online_cpus();
3685         for_each_online_cpu(cpu) {
3686                 p = idle_task(cpu);
3687                 set_tsk_trace_trace(p);
3688         }
3689         put_online_cpus();
3690 }
3691
3692 static void clear_ftrace_pid(struct pid *pid)
3693 {
3694         struct task_struct *p;
3695
3696         rcu_read_lock();
3697         do_each_pid_task(pid, PIDTYPE_PID, p) {
3698                 clear_tsk_trace_trace(p);
3699         } while_each_pid_task(pid, PIDTYPE_PID, p);
3700         rcu_read_unlock();
3701
3702         put_pid(pid);
3703 }
3704
3705 static void set_ftrace_pid(struct pid *pid)
3706 {
3707         struct task_struct *p;
3708
3709         rcu_read_lock();
3710         do_each_pid_task(pid, PIDTYPE_PID, p) {
3711                 set_tsk_trace_trace(p);
3712         } while_each_pid_task(pid, PIDTYPE_PID, p);
3713         rcu_read_unlock();
3714 }
3715
3716 static void clear_ftrace_pid_task(struct pid *pid)
3717 {
3718         if (pid == ftrace_swapper_pid)
3719                 clear_ftrace_swapper();
3720         else
3721                 clear_ftrace_pid(pid);
3722 }
3723
3724 static void set_ftrace_pid_task(struct pid *pid)
3725 {
3726         if (pid == ftrace_swapper_pid)
3727                 set_ftrace_swapper();
3728         else
3729                 set_ftrace_pid(pid);
3730 }
3731
3732 static int ftrace_pid_add(int p)
3733 {
3734         struct pid *pid;
3735         struct ftrace_pid *fpid;
3736         int ret = -EINVAL;
3737
3738         mutex_lock(&ftrace_lock);
3739
3740         if (!p)
3741                 pid = ftrace_swapper_pid;
3742         else
3743                 pid = find_get_pid(p);
3744
3745         if (!pid)
3746                 goto out;
3747
3748         ret = 0;
3749
3750         list_for_each_entry(fpid, &ftrace_pids, list)
3751                 if (fpid->pid == pid)
3752                         goto out_put;
3753
3754         ret = -ENOMEM;
3755
3756         fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
3757         if (!fpid)
3758                 goto out_put;
3759
3760         list_add(&fpid->list, &ftrace_pids);
3761         fpid->pid = pid;
3762
3763         set_ftrace_pid_task(pid);
3764
3765         ftrace_update_pid_func();
3766         ftrace_startup_enable(0);
3767
3768         mutex_unlock(&ftrace_lock);
3769         return 0;
3770
3771 out_put:
3772         if (pid != ftrace_swapper_pid)
3773                 put_pid(pid);
3774
3775 out:
3776         mutex_unlock(&ftrace_lock);
3777         return ret;
3778 }
3779
3780 static void ftrace_pid_reset(void)
3781 {
3782         struct ftrace_pid *fpid, *safe;
3783
3784         mutex_lock(&ftrace_lock);
3785         list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
3786                 struct pid *pid = fpid->pid;
3787
3788                 clear_ftrace_pid_task(pid);
3789
3790                 list_del(&fpid->list);
3791                 kfree(fpid);
3792         }
3793
3794         ftrace_update_pid_func();
3795         ftrace_startup_enable(0);
3796
3797         mutex_unlock(&ftrace_lock);
3798 }
3799
3800 static void *fpid_start(struct seq_file *m, loff_t *pos)
3801 {
3802         mutex_lock(&ftrace_lock);
3803
3804         if (list_empty(&ftrace_pids) && (!*pos))
3805                 return (void *) 1;
3806
3807         return seq_list_start(&ftrace_pids, *pos);
3808 }
3809
3810 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
3811 {
3812         if (v == (void *)1)
3813                 return NULL;
3814
3815         return seq_list_next(v, &ftrace_pids, pos);
3816 }
3817
3818 static void fpid_stop(struct seq_file *m, void *p)
3819 {
3820         mutex_unlock(&ftrace_lock);
3821 }
3822
3823 static int fpid_show(struct seq_file *m, void *v)
3824 {
3825         const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);
3826
3827         if (v == (void *)1) {
3828                 seq_printf(m, "no pid\n");
3829                 return 0;
3830         }
3831
3832         if (fpid->pid == ftrace_swapper_pid)
3833                 seq_printf(m, "swapper tasks\n");
3834         else
3835                 seq_printf(m, "%u\n", pid_vnr(fpid->pid));
3836
3837         return 0;
3838 }
3839
3840 static const struct seq_operations ftrace_pid_sops = {
3841         .start = fpid_start,
3842         .next = fpid_next,
3843         .stop = fpid_stop,
3844         .show = fpid_show,
3845 };
3846
3847 static int
3848 ftrace_pid_open(struct inode *inode, struct file *file)
3849 {
3850         int ret = 0;
3851
3852         if ((file->f_mode & FMODE_WRITE) &&
3853             (file->f_flags & O_TRUNC))
3854                 ftrace_pid_reset();
3855
3856         if (file->f_mode & FMODE_READ)
3857                 ret = seq_open(file, &ftrace_pid_sops);
3858
3859         return ret;
3860 }
3861
3862 static ssize_t
3863 ftrace_pid_write(struct file *filp, const char __user *ubuf,
3864                    size_t cnt, loff_t *ppos)
3865 {
3866         char buf[64], *tmp;
3867         long val;
3868         int ret;
3869
3870         if (cnt >= sizeof(buf))
3871                 return -EINVAL;
3872
3873         if (copy_from_user(&buf, ubuf, cnt))
3874                 return -EFAULT;
3875
3876         buf[cnt] = 0;
3877
3878         /*
3879          * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
3880          * to clean the filter quietly.
3881          */
3882         tmp = strstrip(buf);
3883         if (strlen(tmp) == 0)
3884                 return 1;
3885
3886         ret = strict_strtol(tmp, 10, &val);
3887         if (ret < 0)
3888                 return ret;
3889
3890         ret = ftrace_pid_add(val);
3891
3892         return ret ? ret : cnt;
3893 }
3894
3895 static int
3896 ftrace_pid_release(struct inode *inode, struct file *file)
3897 {
3898         if (file->f_mode & FMODE_READ)
3899                 seq_release(inode, file);
3900
3901         return 0;
3902 }
3903
3904 static const struct file_operations ftrace_pid_fops = {
3905         .open           = ftrace_pid_open,
3906         .write          = ftrace_pid_write,
3907         .read           = seq_read,
3908         .llseek         = ftrace_filter_lseek,
3909         .release        = ftrace_pid_release,
3910 };
3911
3912 static __init int ftrace_init_debugfs(void)
3913 {
3914         struct dentry *d_tracer;
3915
3916         d_tracer = tracing_init_dentry();
3917         if (!d_tracer)
3918                 return 0;
3919
3920         ftrace_init_dyn_debugfs(d_tracer);
3921
3922         trace_create_file("set_ftrace_pid", 0644, d_tracer,
3923                             NULL, &ftrace_pid_fops);
3924
3925         ftrace_profile_debugfs(d_tracer);
3926
3927         return 0;
3928 }
3929 fs_initcall(ftrace_init_debugfs);
3930
3931 /**
3932  * ftrace_kill - kill ftrace
3933  *
3934  * This function should be used by panic code. It stops ftrace
3935  * but in a not so nice way. If you need to simply kill ftrace
3936  * from a non-atomic section, use ftrace_kill.
3937  */
3938 void ftrace_kill(void)
3939 {
3940         ftrace_disabled = 1;
3941         ftrace_enabled = 0;
3942         clear_ftrace_function();
3943 }
3944
3945 /**
3946  * Test if ftrace is dead or not.
3947  */
3948 int ftrace_is_dead(void)
3949 {
3950         return ftrace_disabled;
3951 }
3952
3953 /**
3954  * register_ftrace_function - register a function for profiling
3955  * @ops - ops structure that holds the function for profiling.
3956  *
3957  * Register a function to be called by all functions in the
3958  * kernel.
3959  *
3960  * Note: @ops->func and all the functions it calls must be labeled
3961  *       with "notrace", otherwise it will go into a
3962  *       recursive loop.
3963  */
3964 int register_ftrace_function(struct ftrace_ops *ops)
3965 {
3966         int ret = -1;
3967
3968         mutex_lock(&ftrace_lock);
3969
3970         ret = ftrace_startup(ops, 0);
3971
3972         mutex_unlock(&ftrace_lock);
3973         return ret;
3974 }
3975 EXPORT_SYMBOL_GPL(register_ftrace_function);
3976
3977 /**
3978  * unregister_ftrace_function - unregister a function for profiling.
3979  * @ops - ops structure that holds the function to unregister
3980  *
3981  * Unregister a function that was added to be called by ftrace profiling.
3982  */
3983 int unregister_ftrace_function(struct ftrace_ops *ops)
3984 {
3985         int ret;
3986
3987         mutex_lock(&ftrace_lock);
3988         ret = ftrace_shutdown(ops, 0);
3989         mutex_unlock(&ftrace_lock);
3990
3991         return ret;
3992 }
3993 EXPORT_SYMBOL_GPL(unregister_ftrace_function);
3994
3995 int
3996 ftrace_enable_sysctl(struct ctl_table *table, int write,
3997                      void __user *buffer, size_t *lenp,
3998                      loff_t *ppos)
3999 {
4000         int ret = -ENODEV;
4001
4002         mutex_lock(&ftrace_lock);
4003
4004         if (unlikely(ftrace_disabled))
4005                 goto out;
4006
4007         ret = proc_dointvec(table, write, buffer, lenp, ppos);
4008
4009         if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
4010                 goto out;
4011
4012         last_ftrace_enabled = !!ftrace_enabled;
4013
4014         if (ftrace_enabled) {
4015
4016                 ftrace_startup_sysctl();
4017
4018                 /* we are starting ftrace again */
4019                 if (ftrace_ops_list != &ftrace_list_end)
4020                         update_ftrace_function();
4021
4022         } else {
4023                 /* stopping ftrace calls (just send to ftrace_stub) */
4024                 ftrace_trace_function = ftrace_stub;
4025
4026                 ftrace_shutdown_sysctl();
4027         }
4028
4029  out:
4030         mutex_unlock(&ftrace_lock);
4031         return ret;
4032 }
4033
4034 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
4035
4036 static int ftrace_graph_active;
4037 static struct notifier_block ftrace_suspend_notifier;
4038
4039 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
4040 {
4041         return 0;
4042 }
4043
4044 /* The callbacks that hook a function */
4045 trace_func_graph_ret_t ftrace_graph_return =
4046                         (trace_func_graph_ret_t)ftrace_stub;
4047 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
4048
4049 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
4050 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
4051 {
4052         int i;
4053         int ret = 0;
4054         unsigned long flags;
4055         int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
4056         struct task_struct *g, *t;
4057
4058         for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
4059                 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
4060                                         * sizeof(struct ftrace_ret_stack),
4061                                         GFP_KERNEL);
4062                 if (!ret_stack_list[i]) {
4063                         start = 0;
4064                         end = i;
4065                         ret = -ENOMEM;
4066                         goto free;
4067                 }
4068         }
4069
4070         read_lock_irqsave(&tasklist_lock, flags);
4071         do_each_thread(g, t) {
4072                 if (start == end) {
4073                         ret = -EAGAIN;
4074                         goto unlock;
4075                 }
4076
4077                 if (t->ret_stack == NULL) {
4078                         atomic_set(&t->tracing_graph_pause, 0);
4079                         atomic_set(&t->trace_overrun, 0);
4080                         t->curr_ret_stack = -1;
4081                         /* Make sure the tasks see the -1 first: */
4082                         smp_wmb();
4083                         t->ret_stack = ret_stack_list[start++];
4084                 }
4085         } while_each_thread(g, t);
4086
4087 unlock:
4088         read_unlock_irqrestore(&tasklist_lock, flags);
4089 free:
4090         for (i = start; i < end; i++)
4091                 kfree(ret_stack_list[i]);
4092         return ret;
4093 }
4094
4095 static void
4096 ftrace_graph_probe_sched_switch(void *ignore,
4097                         struct task_struct *prev, struct task_struct *next)
4098 {
4099         unsigned long long timestamp;
4100         int index;
4101
4102         /*
4103          * Does the user want to count the time a function was asleep.
4104          * If so, do not update the time stamps.
4105          */
4106         if (trace_flags & TRACE_ITER_SLEEP_TIME)
4107                 return;
4108
4109         timestamp = trace_clock_local();
4110
4111         prev->ftrace_timestamp = timestamp;
4112
4113         /* only process tasks that we timestamped */
4114         if (!next->ftrace_timestamp)
4115                 return;
4116
4117         /*
4118          * Update all the counters in next to make up for the
4119          * time next was sleeping.
4120          */
4121         timestamp -= next->ftrace_timestamp;
4122
4123         for (index = next->curr_ret_stack; index >= 0; index--)
4124                 next->ret_stack[index].calltime += timestamp;
4125 }
4126
4127 /* Allocate a return stack for each task */
4128 static int start_graph_tracing(void)
4129 {
4130         struct ftrace_ret_stack **ret_stack_list;
4131         int ret, cpu;
4132
4133         ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
4134                                 sizeof(struct ftrace_ret_stack *),
4135                                 GFP_KERNEL);
4136
4137         if (!ret_stack_list)
4138                 return -ENOMEM;
4139
4140         /* The cpu_boot init_task->ret_stack will never be freed */
4141         for_each_online_cpu(cpu) {
4142                 if (!idle_task(cpu)->ret_stack)
4143                         ftrace_graph_init_idle_task(idle_task(cpu), cpu);
4144         }
4145
4146         do {
4147                 ret = alloc_retstack_tasklist(ret_stack_list);
4148         } while (ret == -EAGAIN);
4149
4150         if (!ret) {
4151                 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
4152                 if (ret)
4153                         pr_info("ftrace_graph: Couldn't activate tracepoint"
4154                                 " probe to kernel_sched_switch\n");
4155         }
4156
4157         kfree(ret_stack_list);
4158         return ret;
4159 }
4160
4161 /*
4162  * Hibernation protection.
4163  * The state of the current task is too much unstable during
4164  * suspend/restore to disk. We want to protect against that.
4165  */
4166 static int
4167 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
4168                                                         void *unused)
4169 {
4170         switch (state) {
4171         case PM_HIBERNATION_PREPARE:
4172                 pause_graph_tracing();
4173                 break;
4174
4175         case PM_POST_HIBERNATION:
4176                 unpause_graph_tracing();
4177                 break;
4178         }
4179         return NOTIFY_DONE;
4180 }
4181
4182 /* Just a place holder for function graph */
4183 static struct ftrace_ops fgraph_ops __read_mostly = {
4184         .func           = ftrace_stub,
4185         .flags          = FTRACE_OPS_FL_GLOBAL,
4186 };
4187
4188 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
4189                         trace_func_graph_ent_t entryfunc)
4190 {
4191         int ret = 0;
4192
4193         mutex_lock(&ftrace_lock);
4194
4195         /* we currently allow only one tracer registered at a time */
4196         if (ftrace_graph_active) {
4197                 ret = -EBUSY;
4198                 goto out;
4199         }
4200
4201         ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
4202         register_pm_notifier(&ftrace_suspend_notifier);
4203
4204         ftrace_graph_active++;
4205         ret = start_graph_tracing();
4206         if (ret) {
4207                 ftrace_graph_active--;
4208                 goto out;
4209         }
4210
4211         ftrace_graph_return = retfunc;
4212         ftrace_graph_entry = entryfunc;
4213
4214         ret = ftrace_startup(&fgraph_ops, FTRACE_START_FUNC_RET);
4215
4216 out:
4217         mutex_unlock(&ftrace_lock);
4218         return ret;
4219 }
4220
4221 void unregister_ftrace_graph(void)
4222 {
4223         mutex_lock(&ftrace_lock);
4224
4225         if (unlikely(!ftrace_graph_active))
4226                 goto out;
4227
4228         ftrace_graph_active--;
4229         ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
4230         ftrace_graph_entry = ftrace_graph_entry_stub;
4231         ftrace_shutdown(&fgraph_ops, FTRACE_STOP_FUNC_RET);
4232         unregister_pm_notifier(&ftrace_suspend_notifier);
4233         unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
4234
4235  out:
4236         mutex_unlock(&ftrace_lock);
4237 }
4238
4239 static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);
4240
4241 static void
4242 graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
4243 {
4244         atomic_set(&t->tracing_graph_pause, 0);
4245         atomic_set(&t->trace_overrun, 0);
4246         t->ftrace_timestamp = 0;
4247         /* make curr_ret_stack visible before we add the ret_stack */
4248         smp_wmb();
4249         t->ret_stack = ret_stack;
4250 }
4251
4252 /*
4253  * Allocate a return stack for the idle task. May be the first
4254  * time through, or it may be done by CPU hotplug online.
4255  */
4256 void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
4257 {
4258         t->curr_ret_stack = -1;
4259         /*
4260          * The idle task has no parent, it either has its own
4261          * stack or no stack at all.
4262          */
4263         if (t->ret_stack)
4264                 WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
4265
4266         if (ftrace_graph_active) {
4267                 struct ftrace_ret_stack *ret_stack;
4268
4269                 ret_stack = per_cpu(idle_ret_stack, cpu);
4270                 if (!ret_stack) {
4271                         ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
4272                                             * sizeof(struct ftrace_ret_stack),
4273                                             GFP_KERNEL);
4274                         if (!ret_stack)
4275                                 return;
4276                         per_cpu(idle_ret_stack, cpu) = ret_stack;
4277                 }
4278                 graph_init_task(t, ret_stack);
4279         }
4280 }
4281
4282 /* Allocate a return stack for newly created task */
4283 void ftrace_graph_init_task(struct task_struct *t)
4284 {
4285         /* Make sure we do not use the parent ret_stack */
4286         t->ret_stack = NULL;
4287         t->curr_ret_stack = -1;
4288
4289         if (ftrace_graph_active) {
4290                 struct ftrace_ret_stack *ret_stack;
4291
4292                 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
4293                                 * sizeof(struct ftrace_ret_stack),
4294                                 GFP_KERNEL);
4295                 if (!ret_stack)
4296                         return;
4297                 graph_init_task(t, ret_stack);
4298         }
4299 }
4300
4301 void ftrace_graph_exit_task(struct task_struct *t)
4302 {
4303         struct ftrace_ret_stack *ret_stack = t->ret_stack;
4304
4305         t->ret_stack = NULL;
4306         /* NULL must become visible to IRQs before we free it: */
4307         barrier();
4308
4309         kfree(ret_stack);
4310 }
4311
4312 void ftrace_graph_stop(void)
4313 {
4314         ftrace_stop();
4315 }
4316 #endif