kernel/trace/ftrace.c

   1 /*
   2  * Infrastructure for profiling code inserted by 'gcc -pg'.
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Originally ported from the -rt patch by:
   8  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
   9  *
  10  * Based on code in the latency_tracer, that is:
  11  *
  12  *  Copyright (C) 2004-2006 Ingo Molnar
  13  *  Copyright (C) 2004 William Lee Irwin III
  14  */
  15
  16 #include <linux/stop_machine.h>
  17 #include <linux/clocksource.h>
  18 #include <linux/kallsyms.h>
  19 #include <linux/seq_file.h>
  20 #include <linux/suspend.h>
  21 #include <linux/debugfs.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/kthread.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/module.h>
  26 #include <linux/ftrace.h>
  27 #include <linux/sysctl.h>
  28 #include <linux/slab.h>
  29 #include <linux/ctype.h>
  30 #include <linux/list.h>
  31 #include <linux/hash.h>
  32 #include <linux/rcupdate.h>
  33
  34 #include <trace/events/sched.h>
  35
  36 #include <asm/setup.h>
  37
  38 #include "trace_output.h"
  39 #include "trace_stat.h"
  40
  41 #define FTRACE_WARN_ON(cond)                    \
  42         ({                                      \
  43                 int ___r = cond;                \
  44                 if (WARN_ON(___r))              \
  45                         ftrace_kill();          \
  46                 ___r;                           \
  47         })
  48
  49 #define FTRACE_WARN_ON_ONCE(cond)               \
  50         ({                                      \
  51                 int ___r = cond;                \
  52                 if (WARN_ON_ONCE(___r))         \
  53                         ftrace_kill();          \
  54                 ___r;                           \
  55         })
  56
  57 /* hash bits for specific function selection */
  58 #define FTRACE_HASH_BITS 7
  59 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
  60 #define FTRACE_HASH_DEFAULT_BITS 10
  61 #define FTRACE_HASH_MAX_BITS 12
  62
  63 /* ftrace_enabled is a method to turn ftrace on or off */
  64 int ftrace_enabled __read_mostly;
  65 static int last_ftrace_enabled;
  66
  67 /* Quick disabling of function tracer. */
  68 int function_trace_stop;
  69
  70 /* List for set_ftrace_pid's pids. */
  71 LIST_HEAD(ftrace_pids);
  72 struct ftrace_pid {
  73         struct list_head list;
  74         struct pid *pid;
  75 };
  76
  77 /*
  78  * ftrace_disabled is set when an anomaly is discovered.
  79  * ftrace_disabled is much stronger than ftrace_enabled.
  80  */
  81 static int ftrace_disabled __read_mostly;
  82
  83 static DEFINE_MUTEX(ftrace_lock);
  84
  85 static struct ftrace_ops ftrace_list_end __read_mostly = {
  86         .func           = ftrace_stub,
  87 };
  88
  89 static struct ftrace_ops *ftrace_global_list __read_mostly = &ftrace_list_end;
  90 static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;
  91 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
  92 static ftrace_func_t __ftrace_trace_function_delay __read_mostly = ftrace_stub;
  93 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
  94 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
  95 static struct ftrace_ops global_ops;
  96
  97 static void
  98 ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip);
  99
 100 /*
 101  * Traverse the ftrace_global_list, invoking all entries.  The reason that we
 102  * can use rcu_dereference_raw() is that elements removed from this list
 103  * are simply leaked, so there is no need to interact with a grace-period
 104  * mechanism.  The rcu_dereference_raw() calls are needed to handle
 105  * concurrent insertions into the ftrace_global_list.
 106  *
 107  * Silly Alpha and silly pointer-speculation compiler optimizations!
 108  */
 109 static void ftrace_global_list_func(unsigned long ip,
 110                                     unsigned long parent_ip)
 111 {
 112         struct ftrace_ops *op;
 113
 114         if (unlikely(trace_recursion_test(TRACE_GLOBAL_BIT)))
 115                 return;
 116
 117         trace_recursion_set(TRACE_GLOBAL_BIT);
 118         op = rcu_dereference_raw(ftrace_global_list); /*see above*/
 119         while (op != &ftrace_list_end) {
 120                 op->func(ip, parent_ip);
 121                 op = rcu_dereference_raw(op->next); /*see above*/
 122         };
 123         trace_recursion_clear(TRACE_GLOBAL_BIT);
 124 }
 125
 126 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
 127 {
 128         if (!test_tsk_trace_trace(current))
 129                 return;
 130
 131         ftrace_pid_function(ip, parent_ip);
 132 }
 133
 134 static void set_ftrace_pid_function(ftrace_func_t func)
 135 {
 136         /* do not set ftrace_pid_function to itself! */
 137         if (func != ftrace_pid_func)
 138                 ftrace_pid_function = func;
 139 }
 140
 141 /**
 142  * clear_ftrace_function - reset the ftrace function
 143  *
 144  * This NULLs the ftrace function and in essence stops
 145  * tracing.  There may be lag
 146  */
 147 void clear_ftrace_function(void)
 148 {
 149         ftrace_trace_function = ftrace_stub;
 150         __ftrace_trace_function = ftrace_stub;
 151         __ftrace_trace_function_delay = ftrace_stub;
 152         ftrace_pid_function = ftrace_stub;
 153 }
 154
 155 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 156 /*
 157  * For those archs that do not test ftrace_trace_stop in their
 158  * mcount call site, we need to do it from C.
 159  */
 160 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
 161 {
 162         if (function_trace_stop)
 163                 return;
 164
 165         __ftrace_trace_function(ip, parent_ip);
 166 }
 167 #endif
 168
 169 static void update_global_ops(void)
 170 {
 171         ftrace_func_t func;
 172
 173         /*
 174          * If there's only one function registered, then call that
 175          * function directly. Otherwise, we need to iterate over the
 176          * registered callers.
 177          */
 178         if (ftrace_global_list == &ftrace_list_end ||
 179             ftrace_global_list->next == &ftrace_list_end)
 180                 func = ftrace_global_list->func;
 181         else
 182                 func = ftrace_global_list_func;
 183
 184         /* If we filter on pids, update to use the pid function */
 185         if (!list_empty(&ftrace_pids)) {
 186                 set_ftrace_pid_function(func);
 187                 func = ftrace_pid_func;
 188         }
 189
 190         global_ops.func = func;
 191 }
 192
 193 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 194 static void update_function_graph_func(void);
 195 #else
 196 static inline void update_function_graph_func(void) { }
 197 #endif
 198
 199 static void update_ftrace_function(void)
 200 {
 201         ftrace_func_t func;
 202
 203         update_global_ops();
 204
 205         /*
 206          * If we are at the end of the list and this ops is
 207          * not dynamic, then have the mcount trampoline call
 208          * the function directly
 209          */
 210         if (ftrace_ops_list == &ftrace_list_end ||
 211             (ftrace_ops_list->next == &ftrace_list_end &&
 212              !(ftrace_ops_list->flags & FTRACE_OPS_FL_DYNAMIC)))
 213                 func = ftrace_ops_list->func;
 214         else
 215                 func = ftrace_ops_list_func;
 216
 217 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 218         ftrace_trace_function = func;
 219 #else
 220 #ifdef CONFIG_DYNAMIC_FTRACE
 221         /* do not update till all functions have been modified */
 222         __ftrace_trace_function_delay = func;
 223 #else
 224         __ftrace_trace_function = func;
 225 #endif
 226         ftrace_trace_function = ftrace_test_stop_func;
 227 #endif
 228 }
 229
 230 static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
 231 {
 232         ops->next = *list;
 233         /*
 234          * We are entering ops into the list but another
 235          * CPU might be walking that list. We need to make sure
 236          * the ops->next pointer is valid before another CPU sees
 237          * the ops pointer included into the list.
 238          */
 239         rcu_assign_pointer(*list, ops);
 240 }
 241
 242 static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
 243 {
 244         struct ftrace_ops **p;
 245
 246         update_function_graph_func();
 247
 248         /*
 249          * If we are removing the last function, then simply point
 250          * to the ftrace_stub.
 251          */
 252         if (*list == ops && ops->next == &ftrace_list_end) {
 253                 *list = &ftrace_list_end;
 254                 return 0;
 255         }
 256
 257         for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
 258                 if (*p == ops)
 259                         break;
 260
 261         if (*p != ops)
 262                 return -1;
 263
 264         *p = (*p)->next;
 265         return 0;
 266 }
 267
 268 static int __register_ftrace_function(struct ftrace_ops *ops)
 269 {
 270         if (FTRACE_WARN_ON(ops == &global_ops))
 271                 return -EINVAL;
 272
 273         if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
 274                 return -EBUSY;
 275
 276         if (!core_kernel_data((unsigned long)ops))
 277                 ops->flags |= FTRACE_OPS_FL_DYNAMIC;
 278
 279         if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
 280                 int first = ftrace_global_list == &ftrace_list_end;
 281                 add_ftrace_ops(&ftrace_global_list, ops);
 282                 ops->flags |= FTRACE_OPS_FL_ENABLED;
 283                 if (first)
 284                         add_ftrace_ops(&ftrace_ops_list, &global_ops);
 285         } else
 286                 add_ftrace_ops(&ftrace_ops_list, ops);
 287
 288         if (ftrace_enabled)
 289                 update_ftrace_function();
 290
 291         return 0;
 292 }
 293
 294 static void ftrace_sync(struct work_struct *work)
 295 {
 296         /*
 297          * This function is just a stub to implement a hard force
 298          * of synchronize_sched(). This requires synchronizing
 299          * tasks even in userspace and idle.
 300          *
 301          * Yes, function tracing is rude.
 302          */
 303 }
 304
 305 static int __unregister_ftrace_function(struct ftrace_ops *ops)
 306 {
 307         int ret;
 308
 309         if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
 310                 return -EBUSY;
 311
 312         if (FTRACE_WARN_ON(ops == &global_ops))
 313                 return -EINVAL;
 314
 315         if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
 316                 ret = remove_ftrace_ops(&ftrace_global_list, ops);
 317                 if (!ret && ftrace_global_list == &ftrace_list_end)
 318                         ret = remove_ftrace_ops(&ftrace_ops_list, &global_ops);
 319                 if (!ret)
 320                         ops->flags &= ~FTRACE_OPS_FL_ENABLED;
 321         } else
 322                 ret = remove_ftrace_ops(&ftrace_ops_list, ops);
 323
 324         if (ret < 0)
 325                 return ret;
 326
 327         if (ftrace_enabled)
 328                 update_ftrace_function();
 329
 330         return 0;
 331 }
 332
 333 static void ftrace_update_pid_func(void)
 334 {
 335         /* Only do something if we are tracing something */
 336         if (ftrace_trace_function == ftrace_stub)
 337                 return;
 338
 339         update_ftrace_function();
 340 }
 341
 342 #ifdef CONFIG_FUNCTION_PROFILER
 343 struct ftrace_profile {
 344         struct hlist_node               node;
 345         unsigned long                   ip;
 346         unsigned long                   counter;
 347 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 348         unsigned long long              time;
 349         unsigned long long              time_squared;
 350 #endif
 351 };
 352
 353 struct ftrace_profile_page {
 354         struct ftrace_profile_page      *next;
 355         unsigned long                   index;
 356         struct ftrace_profile           records[];
 357 };
 358
 359 struct ftrace_profile_stat {
 360         atomic_t                        disabled;
 361         struct hlist_head               *hash;
 362         struct ftrace_profile_page      *pages;
 363         struct ftrace_profile_page      *start;
 364         struct tracer_stat              stat;
 365 };
 366
 367 #define PROFILE_RECORDS_SIZE                                            \
 368         (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
 369
 370 #define PROFILES_PER_PAGE                                       \
 371         (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
 372
 373 static int ftrace_profile_bits __read_mostly;
 374 static int ftrace_profile_enabled __read_mostly;
 375
 376 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
 377 static DEFINE_MUTEX(ftrace_profile_lock);
 378
 379 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
 380
 381 #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */
 382
 383 static void *
 384 function_stat_next(void *v, int idx)
 385 {
 386         struct ftrace_profile *rec = v;
 387         struct ftrace_profile_page *pg;
 388
 389         pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
 390
 391  again:
 392         if (idx != 0)
 393                 rec++;
 394
 395         if ((void *)rec >= (void *)&pg->records[pg->index]) {
 396                 pg = pg->next;
 397                 if (!pg)
 398                         return NULL;
 399                 rec = &pg->records[0];
 400                 if (!rec->counter)
 401                         goto again;
 402         }
 403
 404         return rec;
 405 }
 406
 407 static void *function_stat_start(struct tracer_stat *trace)
 408 {
 409         struct ftrace_profile_stat *stat =
 410                 container_of(trace, struct ftrace_profile_stat, stat);
 411
 412         if (!stat || !stat->start)
 413                 return NULL;
 414
 415         return function_stat_next(&stat->start->records[0], 0);
 416 }
 417
 418 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 419 /* function graph compares on total time */
 420 static int function_stat_cmp(void *p1, void *p2)
 421 {
 422         struct ftrace_profile *a = p1;
 423         struct ftrace_profile *b = p2;
 424
 425         if (a->time < b->time)
 426                 return -1;
 427         if (a->time > b->time)
 428                 return 1;
 429         else
 430                 return 0;
 431 }
 432 #else
 433 /* not function graph compares against hits */
 434 static int function_stat_cmp(void *p1, void *p2)
 435 {
 436         struct ftrace_profile *a = p1;
 437         struct ftrace_profile *b = p2;
 438
 439         if (a->counter < b->counter)
 440                 return -1;
 441         if (a->counter > b->counter)
 442                 return 1;
 443         else
 444                 return 0;
 445 }
 446 #endif
 447
 448 static int function_stat_headers(struct seq_file *m)
 449 {
 450 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 451         seq_printf(m, "  Function                               "
 452                    "Hit    Time            Avg             s^2\n"
 453                       "  --------                               "
 454                    "---    ----            ---             ---\n");
 455 #else
 456         seq_printf(m, "  Function                               Hit\n"
 457                       "  --------                               ---\n");
 458 #endif
 459         return 0;
 460 }
 461
 462 static int function_stat_show(struct seq_file *m, void *v)
 463 {
 464         struct ftrace_profile *rec = v;
 465         char str[KSYM_SYMBOL_LEN];
 466         int ret = 0;
 467 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 468         static struct trace_seq s;
 469         unsigned long long avg;
 470         unsigned long long stddev;
 471 #endif
 472         mutex_lock(&ftrace_profile_lock);
 473
 474         /* we raced with function_profile_reset() */
 475         if (unlikely(rec->counter == 0)) {
 476                 ret = -EBUSY;
 477                 goto out;
 478         }
 479
 480         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
 481         seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);
 482
 483 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 484         seq_printf(m, "    ");
 485         avg = rec->time;
 486         do_div(avg, rec->counter);
 487
 488         /* Sample standard deviation (s^2) */
 489         if (rec->counter <= 1)
 490                 stddev = 0;
 491         else {
 492                 stddev = rec->time_squared - rec->counter * avg * avg;
 493                 /*
 494                  * Divide only 1000 for ns^2 -> us^2 conversion.
 495                  * trace_print_graph_duration will divide 1000 again.
 496                  */
 497                 do_div(stddev, (rec->counter - 1) * 1000);
 498         }
 499
 500         trace_seq_init(&s);
 501         trace_print_graph_duration(rec->time, &s);
 502         trace_seq_puts(&s, "    ");
 503         trace_print_graph_duration(avg, &s);
 504         trace_seq_puts(&s, "    ");
 505         trace_print_graph_duration(stddev, &s);
 506         trace_print_seq(m, &s);
 507 #endif
 508         seq_putc(m, '\n');
 509 out:
 510         mutex_unlock(&ftrace_profile_lock);
 511
 512         return ret;
 513 }
 514
 515 static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
 516 {
 517         struct ftrace_profile_page *pg;
 518
 519         pg = stat->pages = stat->start;
 520
 521         while (pg) {
 522                 memset(pg->records, 0, PROFILE_RECORDS_SIZE);
 523                 pg->index = 0;
 524                 pg = pg->next;
 525         }
 526
 527         memset(stat->hash, 0,
 528                FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
 529 }
 530
 531 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
 532 {
 533         struct ftrace_profile_page *pg;
 534         int functions;
 535         int pages;
 536         int i;
 537
 538         /* If we already allocated, do nothing */
 539         if (stat->pages)
 540                 return 0;
 541
 542         stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
 543         if (!stat->pages)
 544                 return -ENOMEM;
 545
 546 #ifdef CONFIG_DYNAMIC_FTRACE
 547         functions = ftrace_update_tot_cnt;
 548 #else
 549         /*
 550          * We do not know the number of functions that exist because
 551          * dynamic tracing is what counts them. With past experience
 552          * we have around 20K functions. That should be more than enough.
 553          * It is highly unlikely we will execute every function in
 554          * the kernel.
 555          */
 556         functions = 20000;
 557 #endif
 558
 559         pg = stat->start = stat->pages;
 560
 561         pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
 562
 563         for (i = 1; i < pages; i++) {
 564                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
 565                 if (!pg->next)
 566                         goto out_free;
 567                 pg = pg->next;
 568         }
 569
 570         return 0;
 571
 572  out_free:
 573         pg = stat->start;
 574         while (pg) {
 575                 unsigned long tmp = (unsigned long)pg;
 576
 577                 pg = pg->next;
 578                 free_page(tmp);
 579         }
 580
 581         stat->pages = NULL;
 582         stat->start = NULL;
 583
 584         return -ENOMEM;
 585 }
 586
 587 static int ftrace_profile_init_cpu(int cpu)
 588 {
 589         struct ftrace_profile_stat *stat;
 590         int size;
 591
 592         stat = &per_cpu(ftrace_profile_stats, cpu);
 593
 594         if (stat->hash) {
 595                 /* If the profile is already created, simply reset it */
 596                 ftrace_profile_reset(stat);
 597                 return 0;
 598         }
 599
 600         /*
 601          * We are profiling all functions, but usually only a few thousand
 602          * functions are hit. We'll make a hash of 1024 items.
 603          */
 604         size = FTRACE_PROFILE_HASH_SIZE;
 605
 606         stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
 607
 608         if (!stat->hash)
 609                 return -ENOMEM;
 610
 611         if (!ftrace_profile_bits) {
 612                 size--;
 613
 614                 for (; size; size >>= 1)
 615                         ftrace_profile_bits++;
 616         }
 617
 618         /* Preallocate the function profiling pages */
 619         if (ftrace_profile_pages_init(stat) < 0) {
 620                 kfree(stat->hash);
 621                 stat->hash = NULL;
 622                 return -ENOMEM;
 623         }
 624
 625         return 0;
 626 }
 627
 628 static int ftrace_profile_init(void)
 629 {
 630         int cpu;
 631         int ret = 0;
 632
 633         for_each_possible_cpu(cpu) {
 634                 ret = ftrace_profile_init_cpu(cpu);
 635                 if (ret)
 636                         break;
 637         }
 638
 639         return ret;
 640 }
 641
 642 /* interrupts must be disabled */
 643 static struct ftrace_profile *
 644 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
 645 {
 646         struct ftrace_profile *rec;
 647         struct hlist_head *hhd;
 648         struct hlist_node *n;
 649         unsigned long key;
 650
 651         key = hash_long(ip, ftrace_profile_bits);
 652         hhd = &stat->hash[key];
 653
 654         if (hlist_empty(hhd))
 655                 return NULL;
 656
 657         hlist_for_each_entry_rcu(rec, n, hhd, node) {
 658                 if (rec->ip == ip)
 659                         return rec;
 660         }
 661
 662         return NULL;
 663 }
 664
 665 static void ftrace_add_profile(struct ftrace_profile_stat *stat,
 666                                struct ftrace_profile *rec)
 667 {
 668         unsigned long key;
 669
 670         key = hash_long(rec->ip, ftrace_profile_bits);
 671         hlist_add_head_rcu(&rec->node, &stat->hash[key]);
 672 }
 673
 674 /*
 675  * The memory is already allocated, this simply finds a new record to use.
 676  */
 677 static struct ftrace_profile *
 678 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
 679 {
 680         struct ftrace_profile *rec = NULL;
 681
 682         /* prevent recursion (from NMIs) */
 683         if (atomic_inc_return(&stat->disabled) != 1)
 684                 goto out;
 685
 686         /*
 687          * Try to find the function again since an NMI
 688          * could have added it
 689          */
 690         rec = ftrace_find_profiled_func(stat, ip);
 691         if (rec)
 692                 goto out;
 693
 694         if (stat->pages->index == PROFILES_PER_PAGE) {
 695                 if (!stat->pages->next)
 696                         goto out;
 697                 stat->pages = stat->pages->next;
 698         }
 699
 700         rec = &stat->pages->records[stat->pages->index++];
 701         rec->ip = ip;
 702         ftrace_add_profile(stat, rec);
 703
 704  out:
 705         atomic_dec(&stat->disabled);
 706
 707         return rec;
 708 }
 709
 710 static void
 711 function_profile_call(unsigned long ip, unsigned long parent_ip)
 712 {
 713         struct ftrace_profile_stat *stat;
 714         struct ftrace_profile *rec;
 715         unsigned long flags;
 716
 717         if (!ftrace_profile_enabled)
 718                 return;
 719
 720         local_irq_save(flags);
 721
 722         stat = &__get_cpu_var(ftrace_profile_stats);
 723         if (!stat->hash || !ftrace_profile_enabled)
 724                 goto out;
 725
 726         rec = ftrace_find_profiled_func(stat, ip);
 727         if (!rec) {
 728                 rec = ftrace_profile_alloc(stat, ip);
 729                 if (!rec)
 730                         goto out;
 731         }
 732
 733         rec->counter++;
 734  out:
 735         local_irq_restore(flags);
 736 }
 737
 738 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 739 static int profile_graph_entry(struct ftrace_graph_ent *trace)
 740 {
 741         function_profile_call(trace->func, 0);
 742         return 1;
 743 }
 744
 745 static void profile_graph_return(struct ftrace_graph_ret *trace)
 746 {
 747         struct ftrace_profile_stat *stat;
 748         unsigned long long calltime;
 749         struct ftrace_profile *rec;
 750         unsigned long flags;
 751
 752         local_irq_save(flags);
 753         stat = &__get_cpu_var(ftrace_profile_stats);
 754         if (!stat->hash || !ftrace_profile_enabled)
 755                 goto out;
 756
 757         /* If the calltime was zero'd ignore it */
 758         if (!trace->calltime)
 759                 goto out;
 760
 761         calltime = trace->rettime - trace->calltime;
 762
 763         if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
 764                 int index;
 765
 766                 index = trace->depth;
 767
 768                 /* Append this call time to the parent time to subtract */
 769                 if (index)
 770                         current->ret_stack[index - 1].subtime += calltime;
 771
 772                 if (current->ret_stack[index].subtime < calltime)
 773                         calltime -= current->ret_stack[index].subtime;
 774                 else
 775                         calltime = 0;
 776         }
 777
 778         rec = ftrace_find_profiled_func(stat, trace->func);
 779         if (rec) {
 780                 rec->time += calltime;
 781                 rec->time_squared += calltime * calltime;
 782         }
 783
 784  out:
 785         local_irq_restore(flags);
 786 }
 787
 788 static int register_ftrace_profiler(void)
 789 {
 790         return register_ftrace_graph(&profile_graph_return,
 791                                      &profile_graph_entry);
 792 }
 793
 794 static void unregister_ftrace_profiler(void)
 795 {
 796         unregister_ftrace_graph();
 797 }
 798 #else
 799 static struct ftrace_ops ftrace_profile_ops __read_mostly = {
 800         .func           = function_profile_call,
 801 };
 802
 803 static int register_ftrace_profiler(void)
 804 {
 805         return register_ftrace_function(&ftrace_profile_ops);
 806 }
 807
 808 static void unregister_ftrace_profiler(void)
 809 {
 810         unregister_ftrace_function(&ftrace_profile_ops);
 811 }
 812 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 813
 814 static ssize_t
 815 ftrace_profile_write(struct file *filp, const char __user *ubuf,
 816                      size_t cnt, loff_t *ppos)
 817 {
 818         unsigned long val;
 819         int ret;
 820
 821         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
 822         if (ret)
 823                 return ret;
 824
 825         val = !!val;
 826
 827         mutex_lock(&ftrace_profile_lock);
 828         if (ftrace_profile_enabled ^ val) {
 829                 if (val) {
 830                         ret = ftrace_profile_init();
 831                         if (ret < 0) {
 832                                 cnt = ret;
 833                                 goto out;
 834                         }
 835
 836                         ret = register_ftrace_profiler();
 837                         if (ret < 0) {
 838                                 cnt = ret;
 839                                 goto out;
 840                         }
 841                         ftrace_profile_enabled = 1;
 842                 } else {
 843                         ftrace_profile_enabled = 0;
 844                         /*
 845                          * unregister_ftrace_profiler calls stop_machine
 846                          * so this acts like an synchronize_sched.
 847                          */
 848                         unregister_ftrace_profiler();
 849                 }
 850         }
 851  out:
 852         mutex_unlock(&ftrace_profile_lock);
 853
 854         *ppos += cnt;
 855
 856         return cnt;
 857 }
 858
 859 static ssize_t
 860 ftrace_profile_read(struct file *filp, char __user *ubuf,
 861                      size_t cnt, loff_t *ppos)
 862 {
 863         char buf[64];           /* big enough to hold a number */
 864         int r;
 865
 866         r = sprintf(buf, "%u\n", ftrace_profile_enabled);
 867         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
 868 }
 869
 870 static const struct file_operations ftrace_profile_fops = {
 871         .open           = tracing_open_generic,
 872         .read           = ftrace_profile_read,
 873         .write          = ftrace_profile_write,
 874         .llseek         = default_llseek,
 875 };
 876
 877 /* used to initialize the real stat files */
 878 static struct tracer_stat function_stats __initdata = {
 879         .name           = "functions",
 880         .stat_start     = function_stat_start,
 881         .stat_next      = function_stat_next,
 882         .stat_cmp       = function_stat_cmp,
 883         .stat_headers   = function_stat_headers,
 884         .stat_show      = function_stat_show
 885 };
 886
 887 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
 888 {
 889         struct ftrace_profile_stat *stat;
 890         struct dentry *entry;
 891         char *name;
 892         int ret;
 893         int cpu;
 894
 895         for_each_possible_cpu(cpu) {
 896                 stat = &per_cpu(ftrace_profile_stats, cpu);
 897
 898                 /* allocate enough for function name + cpu number */
 899                 name = kmalloc(32, GFP_KERNEL);
 900                 if (!name) {
 901                         /*
 902                          * The files created are permanent, if something happens
 903                          * we still do not free memory.
 904                          */
 905                         WARN(1,
 906                              "Could not allocate stat file for cpu %d\n",
 907                              cpu);
 908                         return;
 909                 }
 910                 stat->stat = function_stats;
 911                 snprintf(name, 32, "function%d", cpu);
 912                 stat->stat.name = name;
 913                 ret = register_stat_tracer(&stat->stat);
 914                 if (ret) {
 915                         WARN(1,
 916                              "Could not register function stat for cpu %d\n",
 917                              cpu);
 918                         kfree(name);
 919                         return;
 920                 }
 921         }
 922
 923         entry = debugfs_create_file("function_profile_enabled", 0644,
 924                                     d_tracer, NULL, &ftrace_profile_fops);
 925         if (!entry)
 926                 pr_warning("Could not create debugfs "
 927                            "'function_profile_enabled' entry\n");
 928 }
 929
 930 #else /* CONFIG_FUNCTION_PROFILER */
 931 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
 932 {
 933 }
 934 #endif /* CONFIG_FUNCTION_PROFILER */
 935
 936 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
 937
 938 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 939 static int ftrace_graph_active;
 940 #else
 941 # define ftrace_graph_active 0
 942 #endif
 943
 944 static loff_t
 945 ftrace_filter_lseek(struct file *file, loff_t offset, int whence)
 946 {
 947         loff_t ret;
 948
 949         if (file->f_mode & FMODE_READ)
 950                 ret = seq_lseek(file, offset, whence);
 951         else
 952                 file->f_pos = ret = 1;
 953
 954         return ret;
 955 }
 956
 957 #ifdef CONFIG_DYNAMIC_FTRACE
 958
 959 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
 960 # error Dynamic ftrace depends on MCOUNT_RECORD
 961 #endif
 962
 963 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
 964
 965 struct ftrace_func_probe {
 966         struct hlist_node       node;
 967         struct ftrace_probe_ops *ops;
 968         unsigned long           flags;
 969         unsigned long           ip;
 970         void                    *data;
 971         struct rcu_head         rcu;
 972 };
 973
 974 enum {
 975         FTRACE_UPDATE_CALLS             = (1 << 0),
 976         FTRACE_DISABLE_CALLS            = (1 << 1),
 977         FTRACE_UPDATE_TRACE_FUNC        = (1 << 2),
 978         FTRACE_START_FUNC_RET           = (1 << 3),
 979         FTRACE_STOP_FUNC_RET            = (1 << 4),
 980 };
 981 struct ftrace_func_entry {
 982         struct hlist_node hlist;
 983         unsigned long ip;
 984 };
 985
 986 struct ftrace_hash {
 987         unsigned long           size_bits;
 988         struct hlist_head       *buckets;
 989         unsigned long           count;
 990         struct rcu_head         rcu;
 991 };
 992
 993 /*
 994  * We make these constant because no one should touch them,
 995  * but they are used as the default "empty hash", to avoid allocating
 996  * it all the time. These are in a read only section such that if
 997  * anyone does try to modify it, it will cause an exception.
 998  */
 999 static const struct hlist_head empty_buckets[1];
1000 static const struct ftrace_hash empty_hash = {
1001         .buckets = (struct hlist_head *)empty_buckets,
1002 };
1003 #define EMPTY_HASH      ((struct ftrace_hash *)&empty_hash)
1004
1005 static struct ftrace_ops global_ops = {
1006         .func                   = ftrace_stub,
1007         .notrace_hash           = EMPTY_HASH,
1008         .filter_hash            = EMPTY_HASH,
1009 };
1010
1011 static struct dyn_ftrace *ftrace_new_addrs;
1012
1013 static DEFINE_MUTEX(ftrace_regex_lock);
1014
1015 struct ftrace_page {
1016         struct ftrace_page      *next;
1017         int                     index;
1018         struct dyn_ftrace       records[];
1019 };
1020
1021 #define ENTRIES_PER_PAGE \
1022   ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
1023
1024 /* estimate from running different kernels */
1025 #define NR_TO_INIT              10000
1026
1027 static struct ftrace_page       *ftrace_pages_start;
1028 static struct ftrace_page       *ftrace_pages;
1029
1030 static struct dyn_ftrace *ftrace_free_records;
1031
1032 static bool ftrace_hash_empty(struct ftrace_hash *hash)
1033 {
1034         return !hash || !hash->count;
1035 }
1036
1037 static struct ftrace_func_entry *
1038 ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1039 {
1040         unsigned long key;
1041         struct ftrace_func_entry *entry;
1042         struct hlist_head *hhd;
1043         struct hlist_node *n;
1044
1045         if (ftrace_hash_empty(hash))
1046                 return NULL;
1047
1048         if (hash->size_bits > 0)
1049                 key = hash_long(ip, hash->size_bits);
1050         else
1051                 key = 0;
1052
1053         hhd = &hash->buckets[key];
1054
1055         hlist_for_each_entry_rcu(entry, n, hhd, hlist) {
1056                 if (entry->ip == ip)
1057                         return entry;
1058         }
1059         return NULL;
1060 }
1061
1062 static void __add_hash_entry(struct ftrace_hash *hash,
1063                              struct ftrace_func_entry *entry)
1064 {
1065         struct hlist_head *hhd;
1066         unsigned long key;
1067
1068         if (hash->size_bits)
1069                 key = hash_long(entry->ip, hash->size_bits);
1070         else
1071                 key = 0;
1072
1073         hhd = &hash->buckets[key];
1074         hlist_add_head(&entry->hlist, hhd);
1075         hash->count++;
1076 }
1077
1078 static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
1079 {
1080         struct ftrace_func_entry *entry;
1081
1082         entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1083         if (!entry)
1084                 return -ENOMEM;
1085
1086         entry->ip = ip;
1087         __add_hash_entry(hash, entry);
1088
1089         return 0;
1090 }
1091
1092 static void
1093 free_hash_entry(struct ftrace_hash *hash,
1094                   struct ftrace_func_entry *entry)
1095 {
1096         hlist_del(&entry->hlist);
1097         kfree(entry);
1098         hash->count--;
1099 }
1100
1101 static void
1102 remove_hash_entry(struct ftrace_hash *hash,
1103                   struct ftrace_func_entry *entry)
1104 {
1105         hlist_del(&entry->hlist);
1106         hash->count--;
1107 }
1108
1109 static void ftrace_hash_clear(struct ftrace_hash *hash)
1110 {
1111         struct hlist_head *hhd;
1112         struct hlist_node *tp, *tn;
1113         struct ftrace_func_entry *entry;
1114         int size = 1 << hash->size_bits;
1115         int i;
1116
1117         if (!hash->count)
1118                 return;
1119
1120         for (i = 0; i < size; i++) {
1121                 hhd = &hash->buckets[i];
1122                 hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist)
1123                         free_hash_entry(hash, entry);
1124         }
1125         FTRACE_WARN_ON(hash->count);
1126 }
1127
1128 static void free_ftrace_hash(struct ftrace_hash *hash)
1129 {
1130         if (!hash || hash == EMPTY_HASH)
1131                 return;
1132         ftrace_hash_clear(hash);
1133         kfree(hash->buckets);
1134         kfree(hash);
1135 }
1136
1137 static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
1138 {
1139         struct ftrace_hash *hash;
1140
1141         hash = container_of(rcu, struct ftrace_hash, rcu);
1142         free_ftrace_hash(hash);
1143 }
1144
1145 static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
1146 {
1147         if (!hash || hash == EMPTY_HASH)
1148                 return;
1149         call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
1150 }
1151
1152 static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
1153 {
1154         struct ftrace_hash *hash;
1155         int size;
1156
1157         hash = kzalloc(sizeof(*hash), GFP_KERNEL);
1158         if (!hash)
1159                 return NULL;
1160
1161         size = 1 << size_bits;
1162         hash->buckets = kzalloc(sizeof(*hash->buckets) * size, GFP_KERNEL);
1163
1164         if (!hash->buckets) {
1165                 kfree(hash);
1166                 return NULL;
1167         }
1168
1169         hash->size_bits = size_bits;
1170
1171         return hash;
1172 }
1173
1174 static struct ftrace_hash *
1175 alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
1176 {
1177         struct ftrace_func_entry *entry;
1178         struct ftrace_hash *new_hash;
1179         struct hlist_node *tp;
1180         int size;
1181         int ret;
1182         int i;
1183
1184         new_hash = alloc_ftrace_hash(size_bits);
1185         if (!new_hash)
1186                 return NULL;
1187
1188         /* Empty hash? */
1189         if (ftrace_hash_empty(hash))
1190                 return new_hash;
1191
1192         size = 1 << hash->size_bits;
1193         for (i = 0; i < size; i++) {
1194                 hlist_for_each_entry(entry, tp, &hash->buckets[i], hlist) {
1195                         ret = add_hash_entry(new_hash, entry->ip);
1196                         if (ret < 0)
1197                                 goto free_hash;
1198                 }
1199         }
1200
1201         FTRACE_WARN_ON(new_hash->count != hash->count);
1202
1203         return new_hash;
1204
1205  free_hash:
1206         free_ftrace_hash(new_hash);
1207         return NULL;
1208 }
1209
1210 static void
1211 ftrace_hash_rec_disable(struct ftrace_ops *ops, int filter_hash);
1212 static void
1213 ftrace_hash_rec_enable(struct ftrace_ops *ops, int filter_hash);
1214
1215 static int
1216 ftrace_hash_move(struct ftrace_ops *ops, int enable,
1217                  struct ftrace_hash **dst, struct ftrace_hash *src)
1218 {
1219         struct ftrace_func_entry *entry;
1220         struct hlist_node *tp, *tn;
1221         struct hlist_head *hhd;
1222         struct ftrace_hash *old_hash;
1223         struct ftrace_hash *new_hash;
1224         unsigned long key;
1225         int size = src->count;
1226         int bits = 0;
1227         int ret;
1228         int i;
1229
1230         /*
1231          * Remove the current set, update the hash and add
1232          * them back.
1233          */
1234         ftrace_hash_rec_disable(ops, enable);
1235
1236         /*
1237          * If the new source is empty, just free dst and assign it
1238          * the empty_hash.
1239          */
1240         if (!src->count) {
1241                 free_ftrace_hash_rcu(*dst);
1242                 rcu_assign_pointer(*dst, EMPTY_HASH);
1243                 /* still need to update the function records */
1244                 ret = 0;
1245                 goto out;
1246         }
1247
1248         /*
1249          * Make the hash size about 1/2 the # found
1250          */
1251         for (size /= 2; size; size >>= 1)
1252                 bits++;
1253
1254         /* Don't allocate too much */
1255         if (bits > FTRACE_HASH_MAX_BITS)
1256                 bits = FTRACE_HASH_MAX_BITS;
1257
1258         ret = -ENOMEM;
1259         new_hash = alloc_ftrace_hash(bits);
1260         if (!new_hash)
1261                 goto out;
1262
1263         size = 1 << src->size_bits;
1264         for (i = 0; i < size; i++) {
1265                 hhd = &src->buckets[i];
1266                 hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist) {
1267                         if (bits > 0)
1268                                 key = hash_long(entry->ip, bits);
1269                         else
1270                                 key = 0;
1271                         remove_hash_entry(src, entry);
1272                         __add_hash_entry(new_hash, entry);
1273                 }
1274         }
1275
1276         old_hash = *dst;
1277         rcu_assign_pointer(*dst, new_hash);
1278         free_ftrace_hash_rcu(old_hash);
1279
1280         ret = 0;
1281  out:
1282         /*
1283          * Enable regardless of ret:
1284          *  On success, we enable the new hash.
1285          *  On failure, we re-enable the original hash.
1286          */
1287         ftrace_hash_rec_enable(ops, enable);
1288
1289         return ret;
1290 }
1291
1292 /*
1293  * Test the hashes for this ops to see if we want to call
1294  * the ops->func or not.
1295  *
1296  * It's a match if the ip is in the ops->filter_hash or
1297  * the filter_hash does not exist or is empty,
1298  *  AND
1299  * the ip is not in the ops->notrace_hash.
1300  *
1301  * This needs to be called with preemption disabled as
1302  * the hashes are freed with call_rcu_sched().
1303  */
1304 static int
1305 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
1306 {
1307         struct ftrace_hash *filter_hash;
1308         struct ftrace_hash *notrace_hash;
1309         int ret;
1310
1311         filter_hash = rcu_dereference_raw(ops->filter_hash);
1312         notrace_hash = rcu_dereference_raw(ops->notrace_hash);
1313
1314         if ((ftrace_hash_empty(filter_hash) ||
1315              ftrace_lookup_ip(filter_hash, ip)) &&
1316             (ftrace_hash_empty(notrace_hash) ||
1317              !ftrace_lookup_ip(notrace_hash, ip)))
1318                 ret = 1;
1319         else
1320                 ret = 0;
1321
1322         return ret;
1323 }
1324
1325 /*
1326  * This is a double for. Do not use 'break' to break out of the loop,
1327  * you must use a goto.
1328  */
1329 #define do_for_each_ftrace_rec(pg, rec)                                 \
1330         for (pg = ftrace_pages_start; pg; pg = pg->next) {              \
1331                 int _____i;                                             \
1332                 for (_____i = 0; _____i < pg->index; _____i++) {        \
1333                         rec = &pg->records[_____i];
1334
1335 #define while_for_each_ftrace_rec()             \
1336                 }                               \
1337         }
1338
1339 static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
1340                                      int filter_hash,
1341                                      bool inc)
1342 {
1343         struct ftrace_hash *hash;
1344         struct ftrace_hash *other_hash;
1345         struct ftrace_page *pg;
1346         struct dyn_ftrace *rec;
1347         int count = 0;
1348         int all = 0;
1349
1350         /* Only update if the ops has been registered */
1351         if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1352                 return;
1353
1354         /*
1355          * In the filter_hash case:
1356          *   If the count is zero, we update all records.
1357          *   Otherwise we just update the items in the hash.
1358          *
1359          * In the notrace_hash case:
1360          *   We enable the update in the hash.
1361          *   As disabling notrace means enabling the tracing,
1362          *   and enabling notrace means disabling, the inc variable
1363          *   gets inversed.
1364          */
1365         if (filter_hash) {
1366                 hash = ops->filter_hash;
1367                 other_hash = ops->notrace_hash;
1368                 if (ftrace_hash_empty(hash))
1369                         all = 1;
1370         } else {
1371                 inc = !inc;
1372                 hash = ops->notrace_hash;
1373                 other_hash = ops->filter_hash;
1374                 /*
1375                  * If the notrace hash has no items,
1376                  * then there's nothing to do.
1377                  */
1378                 if (ftrace_hash_empty(hash))
1379                         return;
1380         }
1381
1382         do_for_each_ftrace_rec(pg, rec) {
1383                 int in_other_hash = 0;
1384                 int in_hash = 0;
1385                 int match = 0;
1386
1387                 if (all) {
1388                         /*
1389                          * Only the filter_hash affects all records.
1390                          * Update if the record is not in the notrace hash.
1391                          */
1392                         if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
1393                                 match = 1;
1394                 } else {
1395                         in_hash = !!ftrace_lookup_ip(hash, rec->ip);
1396                         in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
1397
1398                         /*
1399                          *
1400                          */
1401                         if (filter_hash && in_hash && !in_other_hash)
1402                                 match = 1;
1403                         else if (!filter_hash && in_hash &&
1404                                  (in_other_hash || ftrace_hash_empty(other_hash)))
1405                                 match = 1;
1406                 }
1407                 if (!match)
1408                         continue;
1409
1410                 if (inc) {
1411                         rec->flags++;
1412                         if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == FTRACE_REF_MAX))
1413                                 return;
1414                 } else {
1415                         if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == 0))
1416                                 return;
1417                         rec->flags--;
1418                 }
1419                 count++;
1420                 /* Shortcut, if we handled all records, we are done. */
1421                 if (!all && count == hash->count)
1422                         return;
1423         } while_for_each_ftrace_rec();
1424 }
1425
1426 static void ftrace_hash_rec_disable(struct ftrace_ops *ops,
1427                                     int filter_hash)
1428 {
1429         __ftrace_hash_rec_update(ops, filter_hash, 0);
1430 }
1431
1432 static void ftrace_hash_rec_enable(struct ftrace_ops *ops,
1433                                    int filter_hash)
1434 {
1435         __ftrace_hash_rec_update(ops, filter_hash, 1);
1436 }
1437
1438 static void ftrace_free_rec(struct dyn_ftrace *rec)
1439 {
1440         rec->freelist = ftrace_free_records;
1441         ftrace_free_records = rec;
1442         rec->flags |= FTRACE_FL_FREE;
1443 }
1444
1445 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
1446 {
1447         struct dyn_ftrace *rec;
1448
1449         /* First check for freed records */
1450         if (ftrace_free_records) {
1451                 rec = ftrace_free_records;
1452
1453                 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
1454                         FTRACE_WARN_ON_ONCE(1);
1455                         ftrace_free_records = NULL;
1456                         return NULL;
1457                 }
1458
1459                 ftrace_free_records = rec->freelist;
1460                 memset(rec, 0, sizeof(*rec));
1461                 return rec;
1462         }
1463
1464         if (ftrace_pages->index == ENTRIES_PER_PAGE) {
1465                 if (!ftrace_pages->next) {
1466                         /* allocate another page */
1467                         ftrace_pages->next =
1468                                 (void *)get_zeroed_page(GFP_KERNEL);
1469                         if (!ftrace_pages->next)
1470                                 return NULL;
1471                 }
1472                 ftrace_pages = ftrace_pages->next;
1473         }
1474
1475         return &ftrace_pages->records[ftrace_pages->index++];
1476 }
1477
1478 static struct dyn_ftrace *
1479 ftrace_record_ip(unsigned long ip)
1480 {
1481         struct dyn_ftrace *rec;
1482
1483         if (ftrace_disabled)
1484                 return NULL;
1485
1486         rec = ftrace_alloc_dyn_node(ip);
1487         if (!rec)
1488                 return NULL;
1489
1490         rec->ip = ip;
1491         rec->newlist = ftrace_new_addrs;
1492         ftrace_new_addrs = rec;
1493
1494         return rec;
1495 }
1496
1497 static void print_ip_ins(const char *fmt, unsigned char *p)
1498 {
1499         int i;
1500
1501         printk(KERN_CONT "%s", fmt);
1502
1503         for (i = 0; i < MCOUNT_INSN_SIZE; i++)
1504                 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
1505 }
1506
1507 static void ftrace_bug(int failed, unsigned long ip)
1508 {
1509         switch (failed) {
1510         case -EFAULT:
1511                 FTRACE_WARN_ON_ONCE(1);
1512                 pr_info("ftrace faulted on modifying ");
1513                 print_ip_sym(ip);
1514                 break;
1515         case -EINVAL:
1516                 FTRACE_WARN_ON_ONCE(1);
1517                 pr_info("ftrace failed to modify ");
1518                 print_ip_sym(ip);
1519                 print_ip_ins(" actual: ", (unsigned char *)ip);
1520                 printk(KERN_CONT "\n");
1521                 break;
1522         case -EPERM:
1523                 FTRACE_WARN_ON_ONCE(1);
1524                 pr_info("ftrace faulted on writing ");
1525                 print_ip_sym(ip);
1526                 break;
1527         default:
1528                 FTRACE_WARN_ON_ONCE(1);
1529                 pr_info("ftrace faulted on unknown error ");
1530                 print_ip_sym(ip);
1531         }
1532 }
1533
1534
1535 /* Return 1 if the address range is reserved for ftrace */
1536 int ftrace_text_reserved(void *start, void *end)
1537 {
1538         struct dyn_ftrace *rec;
1539         struct ftrace_page *pg;
1540
1541         do_for_each_ftrace_rec(pg, rec) {
1542                 if (rec->ip <= (unsigned long)end &&
1543                     rec->ip + MCOUNT_INSN_SIZE > (unsigned long)start)
1544                         return 1;
1545         } while_for_each_ftrace_rec();
1546         return 0;
1547 }
1548
1549
1550 static int
1551 __ftrace_replace_code(struct dyn_ftrace *rec, int update)
1552 {
1553         unsigned long ftrace_addr;
1554         unsigned long flag = 0UL;
1555
1556         ftrace_addr = (unsigned long)FTRACE_ADDR;
1557
1558         /*
1559          * If we are updating calls:
1560          *
1561          *   If the record has a ref count, then we need to enable it
1562          *   because someone is using it.
1563          *
1564          *   Otherwise we make sure its disabled.
1565          *
1566          * If we are disabling calls, then disable all records that
1567          * are enabled.
1568          */
1569         if (update && (rec->flags & ~FTRACE_FL_MASK))
1570                 flag = FTRACE_FL_ENABLED;
1571
1572         /* If the state of this record hasn't changed, then do nothing */
1573         if ((rec->flags & FTRACE_FL_ENABLED) == flag)
1574                 return 0;
1575
1576         if (flag) {
1577                 rec->flags |= FTRACE_FL_ENABLED;
1578                 return ftrace_make_call(rec, ftrace_addr);
1579         }
1580
1581         rec->flags &= ~FTRACE_FL_ENABLED;
1582         return ftrace_make_nop(NULL, rec, ftrace_addr);
1583 }
1584
1585 static void ftrace_replace_code(int update)
1586 {
1587         struct dyn_ftrace *rec;
1588         struct ftrace_page *pg;
1589         int failed;
1590
1591         if (unlikely(ftrace_disabled))
1592                 return;
1593
1594         do_for_each_ftrace_rec(pg, rec) {
1595                 /* Skip over free records */
1596                 if (rec->flags & FTRACE_FL_FREE)
1597                         continue;
1598
1599                 failed = __ftrace_replace_code(rec, update);
1600                 if (failed) {
1601                         ftrace_bug(failed, rec->ip);
1602                         /* Stop processing */
1603                         return;
1604                 }
1605         } while_for_each_ftrace_rec();
1606 }
1607
1608 static int
1609 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
1610 {
1611         unsigned long ip;
1612         int ret;
1613
1614         ip = rec->ip;
1615
1616         if (unlikely(ftrace_disabled))
1617                 return 0;
1618
1619         ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
1620         if (ret) {
1621                 ftrace_bug(ret, ip);
1622                 return 0;
1623         }
1624         return 1;
1625 }
1626
1627 /*
1628  * archs can override this function if they must do something
1629  * before the modifying code is performed.
1630  */
1631 int __weak ftrace_arch_code_modify_prepare(void)
1632 {
1633         return 0;
1634 }
1635
1636 /*
1637  * archs can override this function if they must do something
1638  * after the modifying code is performed.
1639  */
1640 int __weak ftrace_arch_code_modify_post_process(void)
1641 {
1642         return 0;
1643 }
1644
1645 static int __ftrace_modify_code(void *data)
1646 {
1647         int *command = data;
1648
1649         /*
1650          * Do not call function tracer while we update the code.
1651          * We are in stop machine, no worrying about races.
1652          */
1653         function_trace_stop++;
1654
1655         if (*command & FTRACE_UPDATE_CALLS)
1656                 ftrace_replace_code(1);
1657         else if (*command & FTRACE_DISABLE_CALLS)
1658                 ftrace_replace_code(0);
1659
1660         if (*command & FTRACE_UPDATE_TRACE_FUNC)
1661                 ftrace_update_ftrace_func(ftrace_trace_function);
1662
1663         if (*command & FTRACE_START_FUNC_RET)
1664                 ftrace_enable_ftrace_graph_caller();
1665         else if (*command & FTRACE_STOP_FUNC_RET)
1666                 ftrace_disable_ftrace_graph_caller();
1667
1668 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
1669         /*
1670          * For archs that call ftrace_test_stop_func(), we must
1671          * wait till after we update all the function callers
1672          * before we update the callback. This keeps different
1673          * ops that record different functions from corrupting
1674          * each other.
1675          */
1676         __ftrace_trace_function = __ftrace_trace_function_delay;
1677 #endif
1678         function_trace_stop--;
1679
1680         return 0;
1681 }
1682
1683 static void ftrace_run_update_code(int command)
1684 {
1685         int ret;
1686
1687         ret = ftrace_arch_code_modify_prepare();
1688         FTRACE_WARN_ON(ret);
1689         if (ret)
1690                 return;
1691
1692         stop_machine(__ftrace_modify_code, &command, NULL);
1693
1694         ret = ftrace_arch_code_modify_post_process();
1695         FTRACE_WARN_ON(ret);
1696 }
1697
1698 static ftrace_func_t saved_ftrace_func;
1699 static int ftrace_start_up;
1700 static int global_start_up;
1701
1702 static void ftrace_startup_enable(int command)
1703 {
1704         if (saved_ftrace_func != ftrace_trace_function) {
1705                 saved_ftrace_func = ftrace_trace_function;
1706                 command |= FTRACE_UPDATE_TRACE_FUNC;
1707         }
1708
1709         if (!command || !ftrace_enabled)
1710                 return;
1711
1712         ftrace_run_update_code(command);
1713 }
1714
1715 static int ftrace_startup(struct ftrace_ops *ops, int command)
1716 {
1717         bool hash_enable = true;
1718         int ret;
1719
1720         if (unlikely(ftrace_disabled))
1721                 return -ENODEV;
1722
1723         ret = __register_ftrace_function(ops);
1724         if (ret)
1725                 return ret;
1726
1727         ftrace_start_up++;
1728         command |= FTRACE_UPDATE_CALLS;
1729
1730         /* ops marked global share the filter hashes */
1731         if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
1732                 ops = &global_ops;
1733                 /* Don't update hash if global is already set */
1734                 if (global_start_up)
1735                         hash_enable = false;
1736                 global_start_up++;
1737         }
1738
1739         ops->flags |= FTRACE_OPS_FL_ENABLED;
1740         if (hash_enable)
1741                 ftrace_hash_rec_enable(ops, 1);
1742
1743         ftrace_startup_enable(command);
1744
1745         return 0;
1746 }
1747
1748 static int ftrace_shutdown(struct ftrace_ops *ops, int command)
1749 {
1750         bool hash_disable = true;
1751         int ret;
1752
1753         if (unlikely(ftrace_disabled))
1754                 return -ENODEV;
1755
1756         ret = __unregister_ftrace_function(ops);
1757         if (ret)
1758                 return ret;
1759
1760         ftrace_start_up--;
1761         /*
1762          * Just warn in case of unbalance, no need to kill ftrace, it's not
1763          * critical but the ftrace_call callers may be never nopped again after
1764          * further ftrace uses.
1765          */
1766         WARN_ON_ONCE(ftrace_start_up < 0);
1767
1768         if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
1769                 ops = &global_ops;
1770                 global_start_up--;
1771                 WARN_ON_ONCE(global_start_up < 0);
1772                 /* Don't update hash if global still has users */
1773                 if (global_start_up) {
1774                         WARN_ON_ONCE(!ftrace_start_up);
1775                         hash_disable = false;
1776                 }
1777         }
1778
1779         if (hash_disable)
1780                 ftrace_hash_rec_disable(ops, 1);
1781
1782         if (ops != &global_ops || !global_start_up)
1783                 ops->flags &= ~FTRACE_OPS_FL_ENABLED;
1784
1785         command |= FTRACE_UPDATE_CALLS;
1786
1787         if (saved_ftrace_func != ftrace_trace_function) {
1788                 saved_ftrace_func = ftrace_trace_function;
1789                 command |= FTRACE_UPDATE_TRACE_FUNC;
1790         }
1791
1792         if (!command || !ftrace_enabled)
1793                 return 0;
1794
1795         ftrace_run_update_code(command);
1796
1797         /*
1798          * Dynamic ops may be freed, we must make sure that all
1799          * callers are done before leaving this function.
1800          * The same goes for freeing the per_cpu data of the control
1801          * ops.
1802          *
1803          * Again, normal synchronize_sched() is not good enough.
1804          * We need to do a hard force of sched synchronization.
1805          * This is because we use preempt_disable() to do RCU, but
1806          * the function tracers can be called where RCU is not watching
1807          * (like before user_exit()). We can not rely on the RCU
1808          * infrastructure to do the synchronization, thus we must do it
1809          * ourselves.
1810          */
1811         if (ops->flags & (FTRACE_OPS_FL_DYNAMIC))
1812                 schedule_on_each_cpu(ftrace_sync);
1813
1814         return 0;
1815 }
1816
1817 static void ftrace_startup_sysctl(void)
1818 {
1819         int command;
1820
1821         if (unlikely(ftrace_disabled))
1822                 return;
1823
1824         /* Force update next time */
1825         saved_ftrace_func = NULL;
1826         /* ftrace_start_up is true if we want ftrace running */
1827         if (ftrace_start_up) {
1828                 command = FTRACE_UPDATE_CALLS;
1829                 if (ftrace_graph_active)
1830                         command |= FTRACE_START_FUNC_RET;
1831                 ftrace_startup_enable(command);
1832         }
1833 }
1834
1835 static void ftrace_shutdown_sysctl(void)
1836 {
1837         int command;
1838
1839         if (unlikely(ftrace_disabled))
1840                 return;
1841
1842         /* ftrace_start_up is true if ftrace is running */
1843         if (ftrace_start_up) {
1844                 command = FTRACE_DISABLE_CALLS;
1845                 if (ftrace_graph_active)
1846                         command |= FTRACE_STOP_FUNC_RET;
1847                 ftrace_run_update_code(command);
1848         }
1849 }
1850
1851 static cycle_t          ftrace_update_time;
1852 static unsigned long    ftrace_update_cnt;
1853 unsigned long           ftrace_update_tot_cnt;
1854
1855 static inline int ops_traces_mod(struct ftrace_ops *ops)
1856 {
1857         /*
1858          * Filter_hash being empty will default to trace module.
1859          * But notrace hash requires a test of individual module functions.
1860          */
1861         return ftrace_hash_empty(ops->filter_hash) &&
1862                 ftrace_hash_empty(ops->notrace_hash);
1863 }
1864
1865 /*
1866  * Check if the current ops references the record.
1867  *
1868  * If the ops traces all functions, then it was already accounted for.
1869  * If the ops does not trace the current record function, skip it.
1870  * If the ops ignores the function via notrace filter, skip it.
1871  */
1872 static inline bool
1873 ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec)
1874 {
1875         /* If ops isn't enabled, ignore it */
1876         if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1877                 return 0;
1878
1879         /* If ops traces all mods, we already accounted for it */
1880         if (ops_traces_mod(ops))
1881                 return 0;
1882
1883         /* The function must be in the filter */
1884         if (!ftrace_hash_empty(ops->filter_hash) &&
1885             !ftrace_lookup_ip(ops->filter_hash, rec->ip))
1886                 return 0;
1887
1888         /* If in notrace hash, we ignore it too */
1889         if (ftrace_lookup_ip(ops->notrace_hash, rec->ip))
1890                 return 0;
1891
1892         return 1;
1893 }
1894
1895 static int referenced_filters(struct dyn_ftrace *rec)
1896 {
1897         struct ftrace_ops *ops;
1898         int cnt = 0;
1899
1900         for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {
1901                 if (ops_references_rec(ops, rec))
1902                     cnt++;
1903         }
1904
1905         return cnt;
1906 }
1907
1908 static int ftrace_update_code(struct module *mod)
1909 {
1910         struct dyn_ftrace *p;
1911         cycle_t start, stop;
1912         unsigned long ref = 0;
1913         bool test = false;
1914
1915         /*
1916          * When adding a module, we need to check if tracers are
1917          * currently enabled and if they are set to trace all functions.
1918          * If they are, we need to enable the module functions as well
1919          * as update the reference counts for those function records.
1920          */
1921         if (mod) {
1922                 struct ftrace_ops *ops;
1923
1924                 for (ops = ftrace_ops_list;
1925                      ops != &ftrace_list_end; ops = ops->next) {
1926                         if (ops->flags & FTRACE_OPS_FL_ENABLED) {
1927                                 if (ops_traces_mod(ops))
1928                                         ref++;
1929                                 else
1930                                         test = true;
1931                         }
1932                 }
1933         }
1934
1935         start = ftrace_now(raw_smp_processor_id());
1936         ftrace_update_cnt = 0;
1937
1938         while (ftrace_new_addrs) {
1939                 int cnt = ref;
1940
1941                 /* If something went wrong, bail without enabling anything */
1942                 if (unlikely(ftrace_disabled))
1943                         return -1;
1944
1945                 p = ftrace_new_addrs;
1946                 ftrace_new_addrs = p->newlist;
1947                 if (test)
1948                         cnt += referenced_filters(p);
1949                 p->flags = cnt;
1950
1951                 /*
1952                  * Do the initial record conversion from mcount jump
1953                  * to the NOP instructions.
1954                  */
1955                 if (!ftrace_code_disable(mod, p)) {
1956                         ftrace_free_rec(p);
1957                         /* Game over */
1958                         break;
1959                 }
1960
1961                 ftrace_update_cnt++;
1962
1963                 /*
1964                  * If the tracing is enabled, go ahead and enable the record.
1965                  *
1966                  * The reason not to enable the record immediatelly is the
1967                  * inherent check of ftrace_make_nop/ftrace_make_call for
1968                  * correct previous instructions.  Making first the NOP
1969                  * conversion puts the module to the correct state, thus
1970                  * passing the ftrace_make_call check.
1971                  */
1972                 if (ftrace_start_up && cnt) {
1973                         int failed = __ftrace_replace_code(p, 1);
1974                         if (failed) {
1975                                 ftrace_bug(failed, p->ip);
1976                                 ftrace_free_rec(p);
1977                         }
1978                 }
1979         }
1980
1981         stop = ftrace_now(raw_smp_processor_id());
1982         ftrace_update_time = stop - start;
1983         ftrace_update_tot_cnt += ftrace_update_cnt;
1984
1985         return 0;
1986 }
1987
1988 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
1989 {
1990         struct ftrace_page *pg;
1991         int cnt;
1992         int i;
1993
1994         /* allocate a few pages */
1995         ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
1996         if (!ftrace_pages_start)
1997                 return -1;
1998
1999         /*
2000          * Allocate a few more pages.
2001          *
2002          * TODO: have some parser search vmlinux before
2003          *   final linking to find all calls to ftrace.
2004          *   Then we can:
2005          *    a) know how many pages to allocate.
2006          *     and/or
2007          *    b) set up the table then.
2008          *
2009          *  The dynamic code is still necessary for
2010          *  modules.
2011          */
2012
2013         pg = ftrace_pages = ftrace_pages_start;
2014
2015         cnt = num_to_init / ENTRIES_PER_PAGE;
2016         pr_info("ftrace: allocating %ld entries in %d pages\n",
2017                 num_to_init, cnt + 1);
2018
2019         for (i = 0; i < cnt; i++) {
2020                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
2021
2022                 /* If we fail, we'll try later anyway */
2023                 if (!pg->next)
2024                         break;
2025
2026                 pg = pg->next;
2027         }
2028
2029         return 0;
2030 }
2031
2032 enum {
2033         FTRACE_ITER_FILTER      = (1 << 0),
2034         FTRACE_ITER_NOTRACE     = (1 << 1),
2035         FTRACE_ITER_PRINTALL    = (1 << 2),
2036         FTRACE_ITER_HASH        = (1 << 3),
2037         FTRACE_ITER_ENABLED     = (1 << 4),
2038 };
2039
2040 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
2041
2042 struct ftrace_iterator {
2043         loff_t                          pos;
2044         loff_t                          func_pos;
2045         struct ftrace_page              *pg;
2046         struct dyn_ftrace               *func;
2047         struct ftrace_func_probe        *probe;
2048         struct trace_parser             parser;
2049         struct ftrace_hash              *hash;
2050         struct ftrace_ops               *ops;
2051         int                             hidx;
2052         int                             idx;
2053         unsigned                        flags;
2054 };
2055
2056 static void *
2057 t_hash_next(struct seq_file *m, loff_t *pos)
2058 {
2059         struct ftrace_iterator *iter = m->private;
2060         struct hlist_node *hnd = NULL;
2061         struct hlist_head *hhd;
2062
2063         (*pos)++;
2064         iter->pos = *pos;
2065
2066         if (iter->probe)
2067                 hnd = &iter->probe->node;
2068  retry:
2069         if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
2070                 return NULL;
2071
2072         hhd = &ftrace_func_hash[iter->hidx];
2073
2074         if (hlist_empty(hhd)) {
2075                 iter->hidx++;
2076                 hnd = NULL;
2077                 goto retry;
2078         }
2079
2080         if (!hnd)
2081                 hnd = hhd->first;
2082         else {
2083                 hnd = hnd->next;
2084                 if (!hnd) {
2085                         iter->hidx++;
2086                         goto retry;
2087                 }
2088         }
2089
2090         if (WARN_ON_ONCE(!hnd))
2091                 return NULL;
2092
2093         iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node);
2094
2095         return iter;
2096 }
2097
2098 static void *t_hash_start(struct seq_file *m, loff_t *pos)
2099 {
2100         struct ftrace_iterator *iter = m->private;
2101         void *p = NULL;
2102         loff_t l;
2103
2104         if (iter->func_pos > *pos)
2105                 return NULL;
2106
2107         iter->hidx = 0;
2108         for (l = 0; l <= (*pos - iter->func_pos); ) {
2109                 p = t_hash_next(m, &l);
2110                 if (!p)
2111                         break;
2112         }
2113         if (!p)
2114                 return NULL;
2115
2116         /* Only set this if we have an item */
2117         iter->flags |= FTRACE_ITER_HASH;
2118
2119         return iter;
2120 }
2121
2122 static int
2123 t_hash_show(struct seq_file *m, struct ftrace_iterator *iter)
2124 {
2125         struct ftrace_func_probe *rec;
2126
2127         rec = iter->probe;
2128         if (WARN_ON_ONCE(!rec))
2129                 return -EIO;
2130
2131         if (rec->ops->print)
2132                 return rec->ops->print(m, rec->ip, rec->ops, rec->data);
2133
2134         seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
2135
2136         if (rec->data)
2137                 seq_printf(m, ":%p", rec->data);
2138         seq_putc(m, '\n');
2139
2140         return 0;
2141 }
2142
2143 static void *
2144 t_next(struct seq_file *m, void *v, loff_t *pos)
2145 {
2146         struct ftrace_iterator *iter = m->private;
2147         struct ftrace_ops *ops = &global_ops;
2148         struct dyn_ftrace *rec = NULL;
2149
2150         if (unlikely(ftrace_disabled))
2151                 return NULL;
2152
2153         if (iter->flags & FTRACE_ITER_HASH)
2154                 return t_hash_next(m, pos);
2155
2156         (*pos)++;
2157         iter->pos = iter->func_pos = *pos;
2158
2159         if (iter->flags & FTRACE_ITER_PRINTALL)
2160                 return t_hash_start(m, pos);
2161
2162  retry:
2163         if (iter->idx >= iter->pg->index) {
2164                 if (iter->pg->next) {
2165                         iter->pg = iter->pg->next;
2166                         iter->idx = 0;
2167                         goto retry;
2168                 }
2169         } else {
2170                 rec = &iter->pg->records[iter->idx++];
2171                 if ((rec->flags & FTRACE_FL_FREE) ||
2172
2173                     ((iter->flags & FTRACE_ITER_FILTER) &&
2174                      !(ftrace_lookup_ip(ops->filter_hash, rec->ip))) ||
2175
2176                     ((iter->flags & FTRACE_ITER_NOTRACE) &&
2177                      !ftrace_lookup_ip(ops->notrace_hash, rec->ip)) ||
2178
2179                     ((iter->flags & FTRACE_ITER_ENABLED) &&
2180                      !(rec->flags & ~FTRACE_FL_MASK))) {
2181
2182                         rec = NULL;
2183                         goto retry;
2184                 }
2185         }
2186
2187         if (!rec)
2188                 return t_hash_start(m, pos);
2189
2190         iter->func = rec;
2191
2192         return iter;
2193 }
2194
2195 static void reset_iter_read(struct ftrace_iterator *iter)
2196 {
2197         iter->pos = 0;
2198         iter->func_pos = 0;
2199         iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_HASH);
2200 }
2201
2202 static void *t_start(struct seq_file *m, loff_t *pos)
2203 {
2204         struct ftrace_iterator *iter = m->private;
2205         struct ftrace_ops *ops = &global_ops;
2206         void *p = NULL;
2207         loff_t l;
2208
2209         mutex_lock(&ftrace_lock);
2210
2211         if (unlikely(ftrace_disabled))
2212                 return NULL;
2213
2214         /*
2215          * If an lseek was done, then reset and start from beginning.
2216          */
2217         if (*pos < iter->pos)
2218                 reset_iter_read(iter);
2219
2220         /*
2221          * For set_ftrace_filter reading, if we have the filter
2222          * off, we can short cut and just print out that all
2223          * functions are enabled.
2224          */
2225         if (iter->flags & FTRACE_ITER_FILTER &&
2226             ftrace_hash_empty(ops->filter_hash)) {
2227                 if (*pos > 0)
2228                         return t_hash_start(m, pos);
2229                 iter->flags |= FTRACE_ITER_PRINTALL;
2230                 /* reset in case of seek/pread */
2231                 iter->flags &= ~FTRACE_ITER_HASH;
2232                 return iter;
2233         }
2234
2235         if (iter->flags & FTRACE_ITER_HASH)
2236                 return t_hash_start(m, pos);
2237
2238         /*
2239          * Unfortunately, we need to restart at ftrace_pages_start
2240          * every time we let go of the ftrace_mutex. This is because
2241          * those pointers can change without the lock.
2242          */
2243         iter->pg = ftrace_pages_start;
2244         iter->idx = 0;
2245         for (l = 0; l <= *pos; ) {
2246                 p = t_next(m, p, &l);
2247                 if (!p)
2248                         break;
2249         }
2250
2251         if (!p) {
2252                 if (iter->flags & FTRACE_ITER_FILTER)
2253                         return t_hash_start(m, pos);
2254
2255                 return NULL;
2256         }
2257
2258         return iter;
2259 }
2260
2261 static void t_stop(struct seq_file *m, void *p)
2262 {
2263         mutex_unlock(&ftrace_lock);
2264 }
2265
2266 static int t_show(struct seq_file *m, void *v)
2267 {
2268         struct ftrace_iterator *iter = m->private;
2269         struct dyn_ftrace *rec;
2270
2271         if (iter->flags & FTRACE_ITER_HASH)
2272                 return t_hash_show(m, iter);
2273
2274         if (iter->flags & FTRACE_ITER_PRINTALL) {
2275                 seq_printf(m, "#### all functions enabled ####\n");
2276                 return 0;
2277         }
2278
2279         rec = iter->func;
2280
2281         if (!rec)
2282                 return 0;
2283
2284         seq_printf(m, "%ps", (void *)rec->ip);
2285         if (iter->flags & FTRACE_ITER_ENABLED)
2286                 seq_printf(m, " (%ld)",
2287                            rec->flags & ~FTRACE_FL_MASK);
2288         seq_printf(m, "\n");
2289
2290         return 0;
2291 }
2292
2293 static const struct seq_operations show_ftrace_seq_ops = {
2294         .start = t_start,
2295         .next = t_next,
2296         .stop = t_stop,
2297         .show = t_show,
2298 };
2299
2300 static int
2301 ftrace_avail_open(struct inode *inode, struct file *file)
2302 {
2303         struct ftrace_iterator *iter;
2304         int ret;
2305
2306         if (unlikely(ftrace_disabled))
2307                 return -ENODEV;
2308
2309         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2310         if (!iter)
2311                 return -ENOMEM;
2312
2313         iter->pg = ftrace_pages_start;
2314
2315         ret = seq_open(file, &show_ftrace_seq_ops);
2316         if (!ret) {
2317                 struct seq_file *m = file->private_data;
2318
2319                 m->private = iter;
2320         } else {
2321                 kfree(iter);
2322         }
2323
2324         return ret;
2325 }
2326
2327 static int
2328 ftrace_enabled_open(struct inode *inode, struct file *file)
2329 {
2330         struct ftrace_iterator *iter;
2331         int ret;
2332
2333         if (unlikely(ftrace_disabled))
2334                 return -ENODEV;
2335
2336         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2337         if (!iter)
2338                 return -ENOMEM;
2339
2340         iter->pg = ftrace_pages_start;
2341         iter->flags = FTRACE_ITER_ENABLED;
2342
2343         ret = seq_open(file, &show_ftrace_seq_ops);
2344         if (!ret) {
2345                 struct seq_file *m = file->private_data;
2346
2347                 m->private = iter;
2348         } else {
2349                 kfree(iter);
2350         }
2351
2352         return ret;
2353 }
2354
2355 static void ftrace_filter_reset(struct ftrace_hash *hash)
2356 {
2357         mutex_lock(&ftrace_lock);
2358         ftrace_hash_clear(hash);
2359         mutex_unlock(&ftrace_lock);
2360 }
2361
2362 static int
2363 ftrace_regex_open(struct ftrace_ops *ops, int flag,
2364                   struct inode *inode, struct file *file)
2365 {
2366         struct ftrace_iterator *iter;
2367         struct ftrace_hash *hash;
2368         int ret = 0;
2369
2370         if (unlikely(ftrace_disabled))
2371                 return -ENODEV;
2372
2373         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2374         if (!iter)
2375                 return -ENOMEM;
2376
2377         if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
2378                 kfree(iter);
2379                 return -ENOMEM;
2380         }
2381
2382         if (flag & FTRACE_ITER_NOTRACE)
2383                 hash = ops->notrace_hash;
2384         else
2385                 hash = ops->filter_hash;
2386
2387         iter->ops = ops;
2388         iter->flags = flag;
2389
2390         if (file->f_mode & FMODE_WRITE) {
2391                 mutex_lock(&ftrace_lock);
2392                 iter->hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, hash);
2393                 mutex_unlock(&ftrace_lock);
2394
2395                 if (!iter->hash) {
2396                         trace_parser_put(&iter->parser);
2397                         kfree(iter);
2398                         return -ENOMEM;
2399                 }
2400         }
2401
2402         mutex_lock(&ftrace_regex_lock);
2403
2404         if ((file->f_mode & FMODE_WRITE) &&
2405             (file->f_flags & O_TRUNC))
2406                 ftrace_filter_reset(iter->hash);
2407
2408         if (file->f_mode & FMODE_READ) {
2409                 iter->pg = ftrace_pages_start;
2410
2411                 ret = seq_open(file, &show_ftrace_seq_ops);
2412                 if (!ret) {
2413                         struct seq_file *m = file->private_data;
2414                         m->private = iter;
2415                 } else {
2416                         /* Failed */
2417                         free_ftrace_hash(iter->hash);
2418                         trace_parser_put(&iter->parser);
2419                         kfree(iter);
2420                 }
2421         } else
2422                 file->private_data = iter;
2423         mutex_unlock(&ftrace_regex_lock);
2424
2425         return ret;
2426 }
2427
2428 static int
2429 ftrace_filter_open(struct inode *inode, struct file *file)
2430 {
2431         return ftrace_regex_open(&global_ops, FTRACE_ITER_FILTER,
2432                                  inode, file);
2433 }
2434
2435 static int
2436 ftrace_notrace_open(struct inode *inode, struct file *file)
2437 {
2438         return ftrace_regex_open(&global_ops, FTRACE_ITER_NOTRACE,
2439                                  inode, file);
2440 }
2441
2442 static int ftrace_match(char *str, char *regex, int len, int type)
2443 {
2444         int matched = 0;
2445         int slen;
2446
2447         switch (type) {
2448         case MATCH_FULL:
2449                 if (strcmp(str, regex) == 0)
2450                         matched = 1;
2451                 break;
2452         case MATCH_FRONT_ONLY:
2453                 if (strncmp(str, regex, len) == 0)
2454                         matched = 1;
2455                 break;
2456         case MATCH_MIDDLE_ONLY:
2457                 if (strstr(str, regex))
2458                         matched = 1;
2459                 break;
2460         case MATCH_END_ONLY:
2461                 slen = strlen(str);
2462                 if (slen >= len && memcmp(str + slen - len, regex, len) == 0)
2463                         matched = 1;
2464                 break;
2465         }
2466
2467         return matched;
2468 }
2469
2470 static int
2471 enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int not)
2472 {
2473         struct ftrace_func_entry *entry;
2474         int ret = 0;
2475
2476         entry = ftrace_lookup_ip(hash, rec->ip);
2477         if (not) {
2478                 /* Do nothing if it doesn't exist */
2479                 if (!entry)
2480                         return 0;
2481
2482                 free_hash_entry(hash, entry);
2483         } else {
2484                 /* Do nothing if it exists */
2485                 if (entry)
2486                         return 0;
2487
2488                 ret = add_hash_entry(hash, rec->ip);
2489         }
2490         return ret;
2491 }
2492
2493 static int
2494 ftrace_match_record(struct dyn_ftrace *rec, char *mod,
2495                     char *regex, int len, int type)
2496 {
2497         char str[KSYM_SYMBOL_LEN];
2498         char *modname;
2499
2500         kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
2501
2502         if (mod) {
2503                 /* module lookup requires matching the module */
2504                 if (!modname || strcmp(modname, mod))
2505                         return 0;
2506
2507                 /* blank search means to match all funcs in the mod */
2508                 if (!len)
2509                         return 1;
2510         }
2511
2512         return ftrace_match(str, regex, len, type);
2513 }
2514
2515 static int
2516 match_records(struct ftrace_hash *hash, char *buff,
2517               int len, char *mod, int not)
2518 {
2519         unsigned search_len = 0;
2520         struct ftrace_page *pg;
2521         struct dyn_ftrace *rec;
2522         int type = MATCH_FULL;
2523         char *search = buff;
2524         int found = 0;
2525         int ret;
2526
2527         if (len) {
2528                 type = filter_parse_regex(buff, len, &search, &not);
2529                 search_len = strlen(search);
2530         }
2531
2532         mutex_lock(&ftrace_lock);
2533
2534         if (unlikely(ftrace_disabled))
2535                 goto out_unlock;
2536
2537         do_for_each_ftrace_rec(pg, rec) {
2538
2539                 if (ftrace_match_record(rec, mod, search, search_len, type)) {
2540                         ret = enter_record(hash, rec, not);
2541                         if (ret < 0) {
2542                                 found = ret;
2543                                 goto out_unlock;
2544                         }
2545                         found = 1;
2546                 }
2547         } while_for_each_ftrace_rec();
2548  out_unlock:
2549         mutex_unlock(&ftrace_lock);
2550
2551         return found;
2552 }
2553
2554 static int
2555 ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
2556 {
2557         return match_records(hash, buff, len, NULL, 0);
2558 }
2559
2560 static int
2561 ftrace_match_module_records(struct ftrace_hash *hash, char *buff, char *mod)
2562 {
2563         int not = 0;
2564
2565         /* blank or '*' mean the same */
2566         if (strcmp(buff, "*") == 0)
2567                 buff[0] = 0;
2568
2569         /* handle the case of 'dont filter this module' */
2570         if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
2571                 buff[0] = 0;
2572                 not = 1;
2573         }
2574
2575         return match_records(hash, buff, strlen(buff), mod, not);
2576 }
2577
2578 /*
2579  * We register the module command as a template to show others how
2580  * to register the a command as well.
2581  */
2582
2583 static int
2584 ftrace_mod_callback(struct ftrace_hash *hash,
2585                     char *func, char *cmd, char *param, int enable)
2586 {
2587         char *mod;
2588         int ret = -EINVAL;
2589
2590         /*
2591          * cmd == 'mod' because we only registered this func
2592          * for the 'mod' ftrace_func_command.
2593          * But if you register one func with multiple commands,
2594          * you can tell which command was used by the cmd
2595          * parameter.
2596          */
2597
2598         /* we must have a module name */
2599         if (!param)
2600                 return ret;
2601
2602         mod = strsep(&param, ":");
2603         if (!strlen(mod))
2604                 return ret;
2605
2606         ret = ftrace_match_module_records(hash, func, mod);
2607         if (!ret)
2608                 ret = -EINVAL;
2609         if (ret < 0)
2610                 return ret;
2611
2612         return 0;
2613 }
2614
2615 static struct ftrace_func_command ftrace_mod_cmd = {
2616         .name                   = "mod",
2617         .func                   = ftrace_mod_callback,
2618 };
2619
2620 static int __init ftrace_mod_cmd_init(void)
2621 {
2622         return register_ftrace_command(&ftrace_mod_cmd);
2623 }
2624 device_initcall(ftrace_mod_cmd_init);
2625
2626 static void
2627 function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
2628 {
2629         struct ftrace_func_probe *entry;
2630         struct hlist_head *hhd;
2631         struct hlist_node *n;
2632         unsigned long key;
2633
2634         key = hash_long(ip, FTRACE_HASH_BITS);
2635
2636         hhd = &ftrace_func_hash[key];
2637
2638         if (hlist_empty(hhd))
2639                 return;
2640
2641         /*
2642          * Disable preemption for these calls to prevent a RCU grace
2643          * period. This syncs the hash iteration and freeing of items
2644          * on the hash. rcu_read_lock is too dangerous here.
2645          */
2646         preempt_disable_notrace();
2647         hlist_for_each_entry_rcu(entry, n, hhd, node) {
2648                 if (entry->ip == ip)
2649                         entry->ops->func(ip, parent_ip, &entry->data);
2650         }
2651         preempt_enable_notrace();
2652 }
2653
2654 static struct ftrace_ops trace_probe_ops __read_mostly =
2655 {
2656         .func           = function_trace_probe_call,
2657 };
2658
2659 static int ftrace_probe_registered;
2660
2661 static void __enable_ftrace_function_probe(void)
2662 {
2663         int ret;
2664         int i;
2665
2666         if (ftrace_probe_registered)
2667                 return;
2668
2669         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2670                 struct hlist_head *hhd = &ftrace_func_hash[i];
2671                 if (hhd->first)
2672                         break;
2673         }
2674         /* Nothing registered? */
2675         if (i == FTRACE_FUNC_HASHSIZE)
2676                 return;
2677
2678         ret = ftrace_startup(&trace_probe_ops, 0);
2679
2680         ftrace_probe_registered = 1;
2681 }
2682
2683 static void __disable_ftrace_function_probe(void)
2684 {
2685         int i;
2686
2687         if (!ftrace_probe_registered)
2688                 return;
2689
2690         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2691                 struct hlist_head *hhd = &ftrace_func_hash[i];
2692                 if (hhd->first)
2693                         return;
2694         }
2695
2696         /* no more funcs left */
2697         ftrace_shutdown(&trace_probe_ops, 0);
2698
2699         ftrace_probe_registered = 0;
2700 }
2701
2702
2703 static void ftrace_free_entry_rcu(struct rcu_head *rhp)
2704 {
2705         struct ftrace_func_probe *entry =
2706                 container_of(rhp, struct ftrace_func_probe, rcu);
2707
2708         if (entry->ops->free)
2709                 entry->ops->free(&entry->data);
2710         kfree(entry);
2711 }
2712
2713
2714 int
2715 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2716                               void *data)
2717 {
2718         struct ftrace_func_probe *entry;
2719         struct ftrace_page *pg;
2720         struct dyn_ftrace *rec;
2721         int type, len, not;
2722         unsigned long key;
2723         int count = 0;
2724         char *search;
2725
2726         type = filter_parse_regex(glob, strlen(glob), &search, &not);
2727         len = strlen(search);
2728
2729         /* we do not support '!' for function probes */
2730         if (WARN_ON(not))
2731                 return -EINVAL;
2732
2733         mutex_lock(&ftrace_lock);
2734
2735         if (unlikely(ftrace_disabled))
2736                 goto out_unlock;
2737
2738         do_for_each_ftrace_rec(pg, rec) {
2739
2740                 if (!ftrace_match_record(rec, NULL, search, len, type))
2741                         continue;
2742
2743                 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
2744                 if (!entry) {
2745                         /* If we did not process any, then return error */
2746                         if (!count)
2747                                 count = -ENOMEM;
2748                         goto out_unlock;
2749                 }
2750
2751                 count++;
2752
2753                 entry->data = data;
2754
2755                 /*
2756                  * The caller might want to do something special
2757                  * for each function we find. We call the callback
2758                  * to give the caller an opportunity to do so.
2759                  */
2760                 if (ops->callback) {
2761                         if (ops->callback(rec->ip, &entry->data) < 0) {
2762                                 /* caller does not like this func */
2763                                 kfree(entry);
2764                                 continue;
2765                         }
2766                 }
2767
2768                 entry->ops = ops;
2769                 entry->ip = rec->ip;
2770
2771                 key = hash_long(entry->ip, FTRACE_HASH_BITS);
2772                 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
2773
2774         } while_for_each_ftrace_rec();
2775         __enable_ftrace_function_probe();
2776
2777  out_unlock:
2778         mutex_unlock(&ftrace_lock);
2779
2780         return count;
2781 }
2782
2783 enum {
2784         PROBE_TEST_FUNC         = 1,
2785         PROBE_TEST_DATA         = 2
2786 };
2787
2788 static void
2789 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2790                                   void *data, int flags)
2791 {
2792         struct ftrace_func_probe *entry;
2793         struct hlist_node *n, *tmp;
2794         char str[KSYM_SYMBOL_LEN];
2795         int type = MATCH_FULL;
2796         int i, len = 0;
2797         char *search;
2798
2799         if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
2800                 glob = NULL;
2801         else if (glob) {
2802                 int not;
2803
2804                 type = filter_parse_regex(glob, strlen(glob), &search, &not);
2805                 len = strlen(search);
2806
2807                 /* we do not support '!' for function probes */
2808                 if (WARN_ON(not))
2809                         return;
2810         }
2811
2812         mutex_lock(&ftrace_lock);
2813         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2814                 struct hlist_head *hhd = &ftrace_func_hash[i];
2815
2816                 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
2817
2818                         /* break up if statements for readability */
2819                         if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
2820                                 continue;
2821
2822                         if ((flags & PROBE_TEST_DATA) && entry->data != data)
2823                                 continue;
2824
2825                         /* do this last, since it is the most expensive */
2826                         if (glob) {
2827                                 kallsyms_lookup(entry->ip, NULL, NULL,
2828                                                 NULL, str);
2829                                 if (!ftrace_match(str, glob, len, type))
2830                                         continue;
2831                         }
2832
2833                         hlist_del_rcu(&entry->node);
2834                         call_rcu_sched(&entry->rcu, ftrace_free_entry_rcu);
2835                 }
2836         }
2837         __disable_ftrace_function_probe();
2838         mutex_unlock(&ftrace_lock);
2839 }
2840
2841 void
2842 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2843                                 void *data)
2844 {
2845         __unregister_ftrace_function_probe(glob, ops, data,
2846                                           PROBE_TEST_FUNC | PROBE_TEST_DATA);
2847 }
2848
2849 void
2850 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
2851 {
2852         __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
2853 }
2854
2855 void unregister_ftrace_function_probe_all(char *glob)
2856 {
2857         __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
2858 }
2859
2860 static LIST_HEAD(ftrace_commands);
2861 static DEFINE_MUTEX(ftrace_cmd_mutex);
2862
2863 int register_ftrace_command(struct ftrace_func_command *cmd)
2864 {
2865         struct ftrace_func_command *p;
2866         int ret = 0;
2867
2868         mutex_lock(&ftrace_cmd_mutex);
2869         list_for_each_entry(p, &ftrace_commands, list) {
2870                 if (strcmp(cmd->name, p->name) == 0) {
2871                         ret = -EBUSY;
2872                         goto out_unlock;
2873                 }
2874         }
2875         list_add(&cmd->list, &ftrace_commands);
2876  out_unlock:
2877         mutex_unlock(&ftrace_cmd_mutex);
2878
2879         return ret;
2880 }
2881
2882 int unregister_ftrace_command(struct ftrace_func_command *cmd)
2883 {
2884         struct ftrace_func_command *p, *n;
2885         int ret = -ENODEV;
2886
2887         mutex_lock(&ftrace_cmd_mutex);
2888         list_for_each_entry_safe(p, n, &ftrace_commands, list) {
2889                 if (strcmp(cmd->name, p->name) == 0) {
2890                         ret = 0;
2891                         list_del_init(&p->list);
2892                         goto out_unlock;
2893                 }
2894         }
2895  out_unlock:
2896         mutex_unlock(&ftrace_cmd_mutex);
2897
2898         return ret;
2899 }
2900
2901 static int ftrace_process_regex(struct ftrace_hash *hash,
2902                                 char *buff, int len, int enable)
2903 {
2904         char *func, *command, *next = buff;
2905         struct ftrace_func_command *p;
2906         int ret = -EINVAL;
2907
2908         func = strsep(&next, ":");
2909
2910         if (!next) {
2911                 ret = ftrace_match_records(hash, func, len);
2912                 if (!ret)
2913                         ret = -EINVAL;
2914                 if (ret < 0)
2915                         return ret;
2916                 return 0;
2917         }
2918
2919         /* command found */
2920
2921         command = strsep(&next, ":");
2922
2923         mutex_lock(&ftrace_cmd_mutex);
2924         list_for_each_entry(p, &ftrace_commands, list) {
2925                 if (strcmp(p->name, command) == 0) {
2926                         ret = p->func(hash, func, command, next, enable);
2927                         goto out_unlock;
2928                 }
2929         }
2930  out_unlock:
2931         mutex_unlock(&ftrace_cmd_mutex);
2932
2933         return ret;
2934 }
2935
2936 static ssize_t
2937 ftrace_regex_write(struct file *file, const char __user *ubuf,
2938                    size_t cnt, loff_t *ppos, int enable)
2939 {
2940         struct ftrace_iterator *iter;
2941         struct trace_parser *parser;
2942         ssize_t ret, read;
2943
2944         if (!cnt)
2945                 return 0;
2946
2947         mutex_lock(&ftrace_regex_lock);
2948
2949         ret = -ENODEV;
2950         if (unlikely(ftrace_disabled))
2951                 goto out_unlock;
2952
2953         if (file->f_mode & FMODE_READ) {
2954                 struct seq_file *m = file->private_data;
2955                 iter = m->private;
2956         } else
2957                 iter = file->private_data;
2958
2959         parser = &iter->parser;
2960         read = trace_get_user(parser, ubuf, cnt, ppos);
2961
2962         if (read >= 0 && trace_parser_loaded(parser) &&
2963             !trace_parser_cont(parser)) {
2964                 ret = ftrace_process_regex(iter->hash, parser->buffer,
2965                                            parser->idx, enable);
2966                 trace_parser_clear(parser);
2967                 if (ret)
2968                         goto out_unlock;
2969         }
2970
2971         ret = read;
2972 out_unlock:
2973         mutex_unlock(&ftrace_regex_lock);
2974
2975         return ret;
2976 }
2977
2978 static ssize_t
2979 ftrace_filter_write(struct file *file, const char __user *ubuf,
2980                     size_t cnt, loff_t *ppos)
2981 {
2982         return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
2983 }
2984
2985 static ssize_t
2986 ftrace_notrace_write(struct file *file, const char __user *ubuf,
2987                      size_t cnt, loff_t *ppos)
2988 {
2989         return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
2990 }
2991
2992 static int
2993 ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
2994                  int reset, int enable)
2995 {
2996         struct ftrace_hash **orig_hash;
2997         struct ftrace_hash *hash;
2998         int ret;
2999
3000         /* All global ops uses the global ops filters */
3001         if (ops->flags & FTRACE_OPS_FL_GLOBAL)
3002                 ops = &global_ops;
3003
3004         if (unlikely(ftrace_disabled))
3005                 return -ENODEV;
3006
3007         if (enable)
3008                 orig_hash = &ops->filter_hash;
3009         else
3010                 orig_hash = &ops->notrace_hash;
3011
3012         hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
3013         if (!hash)
3014                 return -ENOMEM;
3015
3016         mutex_lock(&ftrace_regex_lock);
3017         if (reset)
3018                 ftrace_filter_reset(hash);
3019         if (buf)
3020                 ftrace_match_records(hash, buf, len);
3021
3022         mutex_lock(&ftrace_lock);
3023         ret = ftrace_hash_move(ops, enable, orig_hash, hash);
3024         if (!ret && ops->flags & FTRACE_OPS_FL_ENABLED
3025             && ftrace_enabled)
3026                 ftrace_run_update_code(FTRACE_UPDATE_CALLS);
3027
3028         mutex_unlock(&ftrace_lock);
3029
3030         mutex_unlock(&ftrace_regex_lock);
3031
3032         free_ftrace_hash(hash);
3033         return ret;
3034 }
3035
3036 /**
3037  * ftrace_set_filter - set a function to filter on in ftrace
3038  * @ops - the ops to set the filter with
3039  * @buf - the string that holds the function filter text.
3040  * @len - the length of the string.
3041  * @reset - non zero to reset all filters before applying this filter.
3042  *
3043  * Filters denote which functions should be enabled when tracing is enabled.
3044  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
3045  */
3046 void ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
3047                        int len, int reset)
3048 {
3049         ftrace_set_regex(ops, buf, len, reset, 1);
3050 }
3051 EXPORT_SYMBOL_GPL(ftrace_set_filter);
3052
3053 /**
3054  * ftrace_set_notrace - set a function to not trace in ftrace
3055  * @ops - the ops to set the notrace filter with
3056  * @buf - the string that holds the function notrace text.
3057  * @len - the length of the string.
3058  * @reset - non zero to reset all filters before applying this filter.
3059  *
3060  * Notrace Filters denote which functions should not be enabled when tracing
3061  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
3062  * for tracing.
3063  */
3064 void ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
3065                         int len, int reset)
3066 {
3067         ftrace_set_regex(ops, buf, len, reset, 0);
3068 }
3069 EXPORT_SYMBOL_GPL(ftrace_set_notrace);
3070 /**
3071  * ftrace_set_filter - set a function to filter on in ftrace
3072  * @ops - the ops to set the filter with
3073  * @buf - the string that holds the function filter text.
3074  * @len - the length of the string.
3075  * @reset - non zero to reset all filters before applying this filter.
3076  *
3077  * Filters denote which functions should be enabled when tracing is enabled.
3078  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
3079  */
3080 void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
3081 {
3082         ftrace_set_regex(&global_ops, buf, len, reset, 1);
3083 }
3084 EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
3085
3086 /**
3087  * ftrace_set_notrace - set a function to not trace in ftrace
3088  * @ops - the ops to set the notrace filter with
3089  * @buf - the string that holds the function notrace text.
3090  * @len - the length of the string.
3091  * @reset - non zero to reset all filters before applying this filter.
3092  *
3093  * Notrace Filters denote which functions should not be enabled when tracing
3094  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
3095  * for tracing.
3096  */
3097 void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
3098 {
3099         ftrace_set_regex(&global_ops, buf, len, reset, 0);
3100 }
3101 EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
3102
3103 /*
3104  * command line interface to allow users to set filters on boot up.
3105  */
3106 #define FTRACE_FILTER_SIZE              COMMAND_LINE_SIZE
3107 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
3108 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
3109
3110 static int __init set_ftrace_notrace(char *str)
3111 {
3112         strncpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
3113         return 1;
3114 }
3115 __setup("ftrace_notrace=", set_ftrace_notrace);
3116
3117 static int __init set_ftrace_filter(char *str)
3118 {
3119         strncpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
3120         return 1;
3121 }
3122 __setup("ftrace_filter=", set_ftrace_filter);
3123
3124 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3125 static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
3126 static int ftrace_set_func(unsigned long *array, int *idx, char *buffer);
3127
3128 static int __init set_graph_function(char *str)
3129 {
3130         strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
3131         return 1;
3132 }
3133 __setup("ftrace_graph_filter=", set_graph_function);
3134
3135 static void __init set_ftrace_early_graph(char *buf)
3136 {
3137         int ret;
3138         char *func;
3139
3140         while (buf) {
3141                 func = strsep(&buf, ",");
3142                 /* we allow only one expression at a time */
3143                 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
3144                                       func);
3145                 if (ret)
3146                         printk(KERN_DEBUG "ftrace: function %s not "
3147                                           "traceable\n", func);
3148         }
3149 }
3150 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3151
3152 static void __init
3153 set_ftrace_early_filter(struct ftrace_ops *ops, char *buf, int enable)
3154 {
3155         char *func;
3156
3157         while (buf) {
3158                 func = strsep(&buf, ",");
3159                 ftrace_set_regex(ops, func, strlen(func), 0, enable);
3160         }
3161 }
3162
3163 static void __init set_ftrace_early_filters(void)
3164 {
3165         if (ftrace_filter_buf[0])
3166                 set_ftrace_early_filter(&global_ops, ftrace_filter_buf, 1);
3167         if (ftrace_notrace_buf[0])
3168                 set_ftrace_early_filter(&global_ops, ftrace_notrace_buf, 0);
3169 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3170         if (ftrace_graph_buf[0])
3171                 set_ftrace_early_graph(ftrace_graph_buf);
3172 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3173 }
3174
3175 static int
3176 ftrace_regex_release(struct inode *inode, struct file *file)
3177 {
3178         struct seq_file *m = (struct seq_file *)file->private_data;
3179         struct ftrace_iterator *iter;
3180         struct ftrace_hash **orig_hash;
3181         struct trace_parser *parser;
3182         int filter_hash;
3183         int ret;
3184
3185         mutex_lock(&ftrace_regex_lock);
3186         if (file->f_mode & FMODE_READ) {
3187                 iter = m->private;
3188
3189                 seq_release(inode, file);
3190         } else
3191                 iter = file->private_data;
3192
3193         parser = &iter->parser;
3194         if (trace_parser_loaded(parser)) {
3195                 parser->buffer[parser->idx] = 0;
3196                 ftrace_match_records(iter->hash, parser->buffer, parser->idx);
3197         }
3198
3199         trace_parser_put(parser);
3200
3201         if (file->f_mode & FMODE_WRITE) {
3202                 filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
3203
3204                 if (filter_hash)
3205                         orig_hash = &iter->ops->filter_hash;
3206                 else
3207                         orig_hash = &iter->ops->notrace_hash;
3208
3209                 mutex_lock(&ftrace_lock);
3210                 ret = ftrace_hash_move(iter->ops, filter_hash,
3211                                        orig_hash, iter->hash);
3212                 if (!ret && (iter->ops->flags & FTRACE_OPS_FL_ENABLED)
3213                     && ftrace_enabled)
3214                         ftrace_run_update_code(FTRACE_UPDATE_CALLS);
3215
3216                 mutex_unlock(&ftrace_lock);
3217         }
3218         free_ftrace_hash(iter->hash);
3219         kfree(iter);
3220
3221         mutex_unlock(&ftrace_regex_lock);
3222         return 0;
3223 }
3224
3225 static const struct file_operations ftrace_avail_fops = {
3226         .open = ftrace_avail_open,
3227         .read = seq_read,
3228         .llseek = seq_lseek,
3229         .release = seq_release_private,
3230 };
3231
3232 static const struct file_operations ftrace_enabled_fops = {
3233         .open = ftrace_enabled_open,
3234         .read = seq_read,
3235         .llseek = seq_lseek,
3236         .release = seq_release_private,
3237 };
3238
3239 static const struct file_operations ftrace_filter_fops = {
3240         .open = ftrace_filter_open,
3241         .read = seq_read,
3242         .write = ftrace_filter_write,
3243         .llseek = ftrace_filter_lseek,
3244         .release = ftrace_regex_release,
3245 };
3246
3247 static const struct file_operations ftrace_notrace_fops = {
3248         .open = ftrace_notrace_open,
3249         .read = seq_read,
3250         .write = ftrace_notrace_write,
3251         .llseek = ftrace_filter_lseek,
3252         .release = ftrace_regex_release,
3253 };
3254
3255 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3256
3257 static DEFINE_MUTEX(graph_lock);
3258
3259 int ftrace_graph_count;
3260 int ftrace_graph_filter_enabled;
3261 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
3262
3263 static void *
3264 __g_next(struct seq_file *m, loff_t *pos)
3265 {
3266         if (*pos >= ftrace_graph_count)
3267                 return NULL;
3268         return &ftrace_graph_funcs[*pos];
3269 }
3270
3271 static void *
3272 g_next(struct seq_file *m, void *v, loff_t *pos)
3273 {
3274         (*pos)++;
3275         return __g_next(m, pos);
3276 }
3277
3278 static void *g_start(struct seq_file *m, loff_t *pos)
3279 {
3280         mutex_lock(&graph_lock);
3281
3282         /* Nothing, tell g_show to print all functions are enabled */
3283         if (!ftrace_graph_filter_enabled && !*pos)
3284                 return (void *)1;
3285
3286         return __g_next(m, pos);
3287 }
3288
3289 static void g_stop(struct seq_file *m, void *p)
3290 {
3291         mutex_unlock(&graph_lock);
3292 }
3293
3294 static int g_show(struct seq_file *m, void *v)
3295 {
3296         unsigned long *ptr = v;
3297
3298         if (!ptr)
3299                 return 0;
3300
3301         if (ptr == (unsigned long *)1) {
3302                 seq_printf(m, "#### all functions enabled ####\n");
3303                 return 0;
3304         }
3305
3306         seq_printf(m, "%ps\n", (void *)*ptr);
3307
3308         return 0;
3309 }
3310
3311 static const struct seq_operations ftrace_graph_seq_ops = {
3312         .start = g_start,
3313         .next = g_next,
3314         .stop = g_stop,
3315         .show = g_show,
3316 };
3317
3318 static int
3319 ftrace_graph_open(struct inode *inode, struct file *file)
3320 {
3321         int ret = 0;
3322
3323         if (unlikely(ftrace_disabled))
3324                 return -ENODEV;
3325
3326         mutex_lock(&graph_lock);
3327         if ((file->f_mode & FMODE_WRITE) &&
3328             (file->f_flags & O_TRUNC)) {
3329                 ftrace_graph_filter_enabled = 0;
3330                 ftrace_graph_count = 0;
3331                 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
3332         }
3333         mutex_unlock(&graph_lock);
3334
3335         if (file->f_mode & FMODE_READ)
3336                 ret = seq_open(file, &ftrace_graph_seq_ops);
3337
3338         return ret;
3339 }
3340
3341 static int
3342 ftrace_graph_release(struct inode *inode, struct file *file)
3343 {
3344         if (file->f_mode & FMODE_READ)
3345                 seq_release(inode, file);
3346         return 0;
3347 }
3348
3349 static int
3350 ftrace_set_func(unsigned long *array, int *idx, char *buffer)
3351 {
3352         struct dyn_ftrace *rec;
3353         struct ftrace_page *pg;
3354         int search_len;
3355         int fail = 1;
3356         int type, not;
3357         char *search;
3358         bool exists;
3359         int i;
3360
3361         /* decode regex */
3362         type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
3363         if (!not && *idx >= FTRACE_GRAPH_MAX_FUNCS)
3364                 return -EBUSY;
3365
3366         search_len = strlen(search);
3367
3368         mutex_lock(&ftrace_lock);
3369
3370         if (unlikely(ftrace_disabled)) {
3371                 mutex_unlock(&ftrace_lock);
3372                 return -ENODEV;
3373         }
3374
3375         do_for_each_ftrace_rec(pg, rec) {
3376
3377                 if (rec->flags & FTRACE_FL_FREE)
3378                         continue;
3379
3380                 if (ftrace_match_record(rec, NULL, search, search_len, type)) {
3381                         /* if it is in the array */
3382                         exists = false;
3383                         for (i = 0; i < *idx; i++) {
3384                                 if (array[i] == rec->ip) {
3385                                         exists = true;
3386                                         break;
3387                                 }
3388                         }
3389
3390                         if (!not) {
3391                                 fail = 0;
3392                                 if (!exists) {
3393                                         array[(*idx)++] = rec->ip;
3394                                         if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
3395                                                 goto out;
3396                                 }
3397                         } else {
3398                                 if (exists) {
3399                                         array[i] = array[--(*idx)];
3400                                         array[*idx] = 0;
3401                                         fail = 0;
3402                                 }
3403                         }
3404                 }
3405         } while_for_each_ftrace_rec();
3406 out:
3407         mutex_unlock(&ftrace_lock);
3408
3409         if (fail)
3410                 return -EINVAL;
3411
3412         ftrace_graph_filter_enabled = !!(*idx);
3413
3414         return 0;
3415 }
3416
3417 static ssize_t
3418 ftrace_graph_write(struct file *file, const char __user *ubuf,
3419                    size_t cnt, loff_t *ppos)
3420 {
3421         struct trace_parser parser;
3422         ssize_t read, ret;
3423
3424         if (!cnt)
3425                 return 0;
3426
3427         mutex_lock(&graph_lock);
3428
3429         if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) {
3430                 ret = -ENOMEM;
3431                 goto out_unlock;
3432         }
3433
3434         read = trace_get_user(&parser, ubuf, cnt, ppos);
3435
3436         if (read >= 0 && trace_parser_loaded((&parser))) {
3437                 parser.buffer[parser.idx] = 0;
3438
3439                 /* we allow only one expression at a time */
3440                 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
3441                                         parser.buffer);
3442                 if (ret)
3443                         goto out_free;
3444         }
3445
3446         ret = read;
3447
3448 out_free:
3449         trace_parser_put(&parser);
3450 out_unlock:
3451         mutex_unlock(&graph_lock);
3452
3453         return ret;
3454 }
3455
3456 static const struct file_operations ftrace_graph_fops = {
3457         .open           = ftrace_graph_open,
3458         .read           = seq_read,
3459         .write          = ftrace_graph_write,
3460         .llseek         = ftrace_filter_lseek,
3461         .release        = ftrace_graph_release,
3462 };
3463 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3464
3465 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
3466 {
3467
3468         trace_create_file("available_filter_functions", 0444,
3469                         d_tracer, NULL, &ftrace_avail_fops);
3470
3471         trace_create_file("enabled_functions", 0444,
3472                         d_tracer, NULL, &ftrace_enabled_fops);
3473
3474         trace_create_file("set_ftrace_filter", 0644, d_tracer,
3475                         NULL, &ftrace_filter_fops);
3476
3477         trace_create_file("set_ftrace_notrace", 0644, d_tracer,
3478                                     NULL, &ftrace_notrace_fops);
3479
3480 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3481         trace_create_file("set_graph_function", 0444, d_tracer,
3482                                     NULL,
3483                                     &ftrace_graph_fops);
3484 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3485
3486         return 0;
3487 }
3488
3489 static int ftrace_process_locs(struct module *mod,
3490                                unsigned long *start,
3491                                unsigned long *end)
3492 {
3493         unsigned long *p;
3494         unsigned long addr;
3495         unsigned long flags = 0; /* Shut up gcc */
3496
3497         mutex_lock(&ftrace_lock);
3498         p = start;
3499         while (p < end) {
3500                 addr = ftrace_call_adjust(*p++);
3501                 /*
3502                  * Some architecture linkers will pad between
3503                  * the different mcount_loc sections of different
3504                  * object files to satisfy alignments.
3505                  * Skip any NULL pointers.
3506                  */
3507                 if (!addr)
3508                         continue;
3509                 ftrace_record_ip(addr);
3510         }
3511
3512         /*
3513          * We only need to disable interrupts on start up
3514          * because we are modifying code that an interrupt
3515          * may execute, and the modification is not atomic.
3516          * But for modules, nothing runs the code we modify
3517          * until we are finished with it, and there's no
3518          * reason to cause large interrupt latencies while we do it.
3519          */
3520         if (!mod)
3521                 local_irq_save(flags);
3522         ftrace_update_code(mod);
3523         if (!mod)
3524                 local_irq_restore(flags);
3525         mutex_unlock(&ftrace_lock);
3526
3527         return 0;
3528 }
3529
3530 #ifdef CONFIG_MODULES
3531 void ftrace_release_mod(struct module *mod)
3532 {
3533         struct dyn_ftrace *rec;
3534         struct ftrace_page *pg;
3535
3536         mutex_lock(&ftrace_lock);
3537
3538         if (ftrace_disabled)
3539                 goto out_unlock;
3540
3541         do_for_each_ftrace_rec(pg, rec) {
3542                 if (within_module_core(rec->ip, mod)) {
3543                         /*
3544                          * rec->ip is changed in ftrace_free_rec()
3545                          * It should not between s and e if record was freed.
3546                          */
3547                         FTRACE_WARN_ON(rec->flags & FTRACE_FL_FREE);
3548                         ftrace_free_rec(rec);
3549                 }
3550         } while_for_each_ftrace_rec();
3551  out_unlock:
3552         mutex_unlock(&ftrace_lock);
3553 }
3554
3555 static void ftrace_init_module(struct module *mod,
3556                                unsigned long *start, unsigned long *end)
3557 {
3558         if (ftrace_disabled || start == end)
3559                 return;
3560         ftrace_process_locs(mod, start, end);
3561 }
3562
3563 void ftrace_module_init(struct module *mod)
3564 {
3565         ftrace_init_module(mod, mod->ftrace_callsites,
3566                            mod->ftrace_callsites +
3567                            mod->num_ftrace_callsites);
3568 }
3569
3570 static int ftrace_module_notify_exit(struct notifier_block *self,
3571                                      unsigned long val, void *data)
3572 {
3573         struct module *mod = data;
3574
3575         if (val == MODULE_STATE_GOING)
3576                 ftrace_release_mod(mod);
3577
3578         return 0;
3579 }
3580 #else
3581 static int ftrace_module_notify_exit(struct notifier_block *self,
3582                                      unsigned long val, void *data)
3583 {
3584         return 0;
3585 }
3586 #endif /* CONFIG_MODULES */
3587
3588 struct notifier_block ftrace_module_exit_nb = {
3589         .notifier_call = ftrace_module_notify_exit,
3590         .priority = INT_MIN,    /* Run after anything that can remove kprobes */
3591 };
3592
3593 extern unsigned long __start_mcount_loc[];
3594 extern unsigned long __stop_mcount_loc[];
3595
3596 void __init ftrace_init(void)
3597 {
3598         unsigned long count, addr, flags;
3599         int ret;
3600
3601         /* Keep the ftrace pointer to the stub */
3602         addr = (unsigned long)ftrace_stub;
3603
3604         local_irq_save(flags);
3605         ftrace_dyn_arch_init(&addr);
3606         local_irq_restore(flags);
3607
3608         /* ftrace_dyn_arch_init places the return code in addr */
3609         if (addr)
3610                 goto failed;
3611
3612         count = __stop_mcount_loc - __start_mcount_loc;
3613
3614         ret = ftrace_dyn_table_alloc(count);
3615         if (ret)
3616                 goto failed;
3617
3618         last_ftrace_enabled = ftrace_enabled = 1;
3619
3620         ret = ftrace_process_locs(NULL,
3621                                   __start_mcount_loc,
3622                                   __stop_mcount_loc);
3623
3624         ret = register_module_notifier(&ftrace_module_exit_nb);
3625         if (ret)
3626                 pr_warning("Failed to register trace ftrace module exit notifier\n");
3627
3628         set_ftrace_early_filters();
3629
3630         return;
3631  failed:
3632         ftrace_disabled = 1;
3633 }
3634
3635 #else
3636
3637 static struct ftrace_ops global_ops = {
3638         .func                   = ftrace_stub,
3639 };
3640
3641 static int __init ftrace_nodyn_init(void)
3642 {
3643         ftrace_enabled = 1;
3644         return 0;
3645 }
3646 device_initcall(ftrace_nodyn_init);
3647
3648 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
3649 static inline void ftrace_startup_enable(int command) { }
3650 /* Keep as macros so we do not need to define the commands */
3651 # define ftrace_startup(ops, command)                                   \
3652         ({                                                              \
3653                 int ___ret = __register_ftrace_function(ops);           \
3654                 if (!___ret)                                            \
3655                         (ops)->flags |= FTRACE_OPS_FL_ENABLED;          \
3656                 ___ret;                                                 \
3657         })
3658 # define ftrace_shutdown(ops, command) __unregister_ftrace_function(ops)
3659
3660 # define ftrace_startup_sysctl()        do { } while (0)
3661 # define ftrace_shutdown_sysctl()       do { } while (0)
3662
3663 static inline int
3664 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
3665 {
3666         return 1;
3667 }
3668
3669 #endif /* CONFIG_DYNAMIC_FTRACE */
3670
3671 static void
3672 ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip)
3673 {
3674         struct ftrace_ops *op;
3675
3676         if (unlikely(trace_recursion_test(TRACE_INTERNAL_BIT)))
3677                 return;
3678
3679         trace_recursion_set(TRACE_INTERNAL_BIT);
3680         /*
3681          * Some of the ops may be dynamically allocated,
3682          * they must be freed after a synchronize_sched().
3683          */
3684         preempt_disable_notrace();
3685         op = rcu_dereference_raw(ftrace_ops_list);
3686         while (op != &ftrace_list_end) {
3687                 if (ftrace_ops_test(op, ip))
3688                         op->func(ip, parent_ip);
3689                 op = rcu_dereference_raw(op->next);
3690         };
3691         preempt_enable_notrace();
3692         trace_recursion_clear(TRACE_INTERNAL_BIT);
3693 }
3694
3695 static void clear_ftrace_swapper(void)
3696 {
3697         struct task_struct *p;
3698         int cpu;
3699
3700         get_online_cpus();
3701         for_each_online_cpu(cpu) {
3702                 p = idle_task(cpu);
3703                 clear_tsk_trace_trace(p);
3704         }
3705         put_online_cpus();
3706 }
3707
3708 static void set_ftrace_swapper(void)
3709 {
3710         struct task_struct *p;
3711         int cpu;
3712
3713         get_online_cpus();
3714         for_each_online_cpu(cpu) {
3715                 p = idle_task(cpu);
3716                 set_tsk_trace_trace(p);
3717         }
3718         put_online_cpus();
3719 }
3720
3721 static void clear_ftrace_pid(struct pid *pid)
3722 {
3723         struct task_struct *p;
3724
3725         rcu_read_lock();
3726         do_each_pid_task(pid, PIDTYPE_PID, p) {
3727                 clear_tsk_trace_trace(p);
3728         } while_each_pid_task(pid, PIDTYPE_PID, p);
3729         rcu_read_unlock();
3730
3731         put_pid(pid);
3732 }
3733
3734 static void set_ftrace_pid(struct pid *pid)
3735 {
3736         struct task_struct *p;
3737
3738         rcu_read_lock();
3739         do_each_pid_task(pid, PIDTYPE_PID, p) {
3740                 set_tsk_trace_trace(p);
3741         } while_each_pid_task(pid, PIDTYPE_PID, p);
3742         rcu_read_unlock();
3743 }
3744
3745 static void clear_ftrace_pid_task(struct pid *pid)
3746 {
3747         if (pid == ftrace_swapper_pid)
3748                 clear_ftrace_swapper();
3749         else
3750                 clear_ftrace_pid(pid);
3751 }
3752
3753 static void set_ftrace_pid_task(struct pid *pid)
3754 {
3755         if (pid == ftrace_swapper_pid)
3756                 set_ftrace_swapper();
3757         else
3758                 set_ftrace_pid(pid);
3759 }
3760
3761 static int ftrace_pid_add(int p)
3762 {
3763         struct pid *pid;
3764         struct ftrace_pid *fpid;
3765         int ret = -EINVAL;
3766
3767         mutex_lock(&ftrace_lock);
3768
3769         if (!p)
3770                 pid = ftrace_swapper_pid;
3771         else
3772                 pid = find_get_pid(p);
3773
3774         if (!pid)
3775                 goto out;
3776
3777         ret = 0;
3778
3779         list_for_each_entry(fpid, &ftrace_pids, list)
3780                 if (fpid->pid == pid)
3781                         goto out_put;
3782
3783         ret = -ENOMEM;
3784
3785         fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
3786         if (!fpid)
3787                 goto out_put;
3788
3789         list_add(&fpid->list, &ftrace_pids);
3790         fpid->pid = pid;
3791
3792         set_ftrace_pid_task(pid);
3793
3794         ftrace_update_pid_func();
3795         ftrace_startup_enable(0);
3796
3797         mutex_unlock(&ftrace_lock);
3798         return 0;
3799
3800 out_put:
3801         if (pid != ftrace_swapper_pid)
3802                 put_pid(pid);
3803
3804 out:
3805         mutex_unlock(&ftrace_lock);
3806         return ret;
3807 }
3808
3809 static void ftrace_pid_reset(void)
3810 {
3811         struct ftrace_pid *fpid, *safe;
3812
3813         mutex_lock(&ftrace_lock);
3814         list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
3815                 struct pid *pid = fpid->pid;
3816
3817                 clear_ftrace_pid_task(pid);
3818
3819                 list_del(&fpid->list);
3820                 kfree(fpid);
3821         }
3822
3823         ftrace_update_pid_func();
3824         ftrace_startup_enable(0);
3825
3826         mutex_unlock(&ftrace_lock);
3827 }
3828
3829 static void *fpid_start(struct seq_file *m, loff_t *pos)
3830 {
3831         mutex_lock(&ftrace_lock);
3832
3833         if (list_empty(&ftrace_pids) && (!*pos))
3834                 return (void *) 1;
3835
3836         return seq_list_start(&ftrace_pids, *pos);
3837 }
3838
3839 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
3840 {
3841         if (v == (void *)1)
3842                 return NULL;
3843
3844         return seq_list_next(v, &ftrace_pids, pos);
3845 }
3846
3847 static void fpid_stop(struct seq_file *m, void *p)
3848 {
3849         mutex_unlock(&ftrace_lock);
3850 }
3851
3852 static int fpid_show(struct seq_file *m, void *v)
3853 {
3854         const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);
3855
3856         if (v == (void *)1) {
3857                 seq_printf(m, "no pid\n");
3858                 return 0;
3859         }
3860
3861         if (fpid->pid == ftrace_swapper_pid)
3862                 seq_printf(m, "swapper tasks\n");
3863         else
3864                 seq_printf(m, "%u\n", pid_vnr(fpid->pid));
3865
3866         return 0;
3867 }
3868
3869 static const struct seq_operations ftrace_pid_sops = {
3870         .start = fpid_start,
3871         .next = fpid_next,
3872         .stop = fpid_stop,
3873         .show = fpid_show,
3874 };
3875
3876 static int
3877 ftrace_pid_open(struct inode *inode, struct file *file)
3878 {
3879         int ret = 0;
3880
3881         if ((file->f_mode & FMODE_WRITE) &&
3882             (file->f_flags & O_TRUNC))
3883                 ftrace_pid_reset();
3884
3885         if (file->f_mode & FMODE_READ)
3886                 ret = seq_open(file, &ftrace_pid_sops);
3887
3888         return ret;
3889 }
3890
3891 static ssize_t
3892 ftrace_pid_write(struct file *filp, const char __user *ubuf,
3893                    size_t cnt, loff_t *ppos)
3894 {
3895         char buf[64], *tmp;
3896         long val;
3897         int ret;
3898
3899         if (cnt >= sizeof(buf))
3900                 return -EINVAL;
3901
3902         if (copy_from_user(&buf, ubuf, cnt))
3903                 return -EFAULT;
3904
3905         buf[cnt] = 0;
3906
3907         /*
3908          * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
3909          * to clean the filter quietly.
3910          */
3911         tmp = strstrip(buf);
3912         if (strlen(tmp) == 0)
3913                 return 1;
3914
3915         ret = strict_strtol(tmp, 10, &val);
3916         if (ret < 0)
3917                 return ret;
3918
3919         ret = ftrace_pid_add(val);
3920
3921         return ret ? ret : cnt;
3922 }
3923
3924 static int
3925 ftrace_pid_release(struct inode *inode, struct file *file)
3926 {
3927         if (file->f_mode & FMODE_READ)
3928                 seq_release(inode, file);
3929
3930         return 0;
3931 }
3932
3933 static const struct file_operations ftrace_pid_fops = {
3934         .open           = ftrace_pid_open,
3935         .write          = ftrace_pid_write,
3936         .read           = seq_read,
3937         .llseek         = ftrace_filter_lseek,
3938         .release        = ftrace_pid_release,
3939 };
3940
3941 static __init int ftrace_init_debugfs(void)
3942 {
3943         struct dentry *d_tracer;
3944
3945         d_tracer = tracing_init_dentry();
3946         if (!d_tracer)
3947                 return 0;
3948
3949         ftrace_init_dyn_debugfs(d_tracer);
3950
3951         trace_create_file("set_ftrace_pid", 0644, d_tracer,
3952                             NULL, &ftrace_pid_fops);
3953
3954         ftrace_profile_debugfs(d_tracer);
3955
3956         return 0;
3957 }
3958 fs_initcall(ftrace_init_debugfs);
3959
3960 /**
3961  * ftrace_kill - kill ftrace
3962  *
3963  * This function should be used by panic code. It stops ftrace
3964  * but in a not so nice way. If you need to simply kill ftrace
3965  * from a non-atomic section, use ftrace_kill.
3966  */
3967 void ftrace_kill(void)
3968 {
3969         ftrace_disabled = 1;
3970         ftrace_enabled = 0;
3971         clear_ftrace_function();
3972 }
3973
3974 /**
3975  * Test if ftrace is dead or not.
3976  */
3977 int ftrace_is_dead(void)
3978 {
3979         return ftrace_disabled;
3980 }
3981
3982 /**
3983  * register_ftrace_function - register a function for profiling
3984  * @ops - ops structure that holds the function for profiling.
3985  *
3986  * Register a function to be called by all functions in the
3987  * kernel.
3988  *
3989  * Note: @ops->func and all the functions it calls must be labeled
3990  *       with "notrace", otherwise it will go into a
3991  *       recursive loop.
3992  */
3993 int register_ftrace_function(struct ftrace_ops *ops)
3994 {
3995         int ret = -1;
3996
3997         mutex_lock(&ftrace_lock);
3998
3999         ret = ftrace_startup(ops, 0);
4000
4001         mutex_unlock(&ftrace_lock);
4002         return ret;
4003 }
4004 EXPORT_SYMBOL_GPL(register_ftrace_function);
4005
4006 /**
4007  * unregister_ftrace_function - unregister a function for profiling.
4008  * @ops - ops structure that holds the function to unregister
4009  *
4010  * Unregister a function that was added to be called by ftrace profiling.
4011  */
4012 int unregister_ftrace_function(struct ftrace_ops *ops)
4013 {
4014         int ret;
4015
4016         mutex_lock(&ftrace_lock);
4017         ret = ftrace_shutdown(ops, 0);
4018         mutex_unlock(&ftrace_lock);
4019
4020         return ret;
4021 }
4022 EXPORT_SYMBOL_GPL(unregister_ftrace_function);
4023
4024 int
4025 ftrace_enable_sysctl(struct ctl_table *table, int write,
4026                      void __user *buffer, size_t *lenp,
4027                      loff_t *ppos)
4028 {
4029         int ret = -ENODEV;
4030
4031         mutex_lock(&ftrace_lock);
4032
4033         if (unlikely(ftrace_disabled))
4034                 goto out;
4035
4036         ret = proc_dointvec(table, write, buffer, lenp, ppos);
4037
4038         if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
4039                 goto out;
4040
4041         last_ftrace_enabled = !!ftrace_enabled;
4042
4043         if (ftrace_enabled) {
4044
4045                 /* we are starting ftrace again */
4046                 if (ftrace_ops_list != &ftrace_list_end)
4047                         update_ftrace_function();
4048
4049                 ftrace_startup_sysctl();
4050
4051         } else {
4052                 /* stopping ftrace calls (just send to ftrace_stub) */
4053                 ftrace_trace_function = ftrace_stub;
4054
4055                 ftrace_shutdown_sysctl();
4056         }
4057
4058  out:
4059         mutex_unlock(&ftrace_lock);
4060         return ret;
4061 }
4062
4063 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
4064
4065 static struct notifier_block ftrace_suspend_notifier;
4066
4067 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
4068 {
4069         return 0;
4070 }
4071
4072 /* The callbacks that hook a function */
4073 trace_func_graph_ret_t ftrace_graph_return =
4074                         (trace_func_graph_ret_t)ftrace_stub;
4075 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
4076 static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub;
4077
4078 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
4079 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
4080 {
4081         int i;
4082         int ret = 0;
4083         unsigned long flags;
4084         int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
4085         struct task_struct *g, *t;
4086
4087         for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
4088                 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
4089                                         * sizeof(struct ftrace_ret_stack),
4090                                         GFP_KERNEL);
4091                 if (!ret_stack_list[i]) {
4092                         start = 0;
4093                         end = i;
4094                         ret = -ENOMEM;
4095                         goto free;
4096                 }
4097         }
4098
4099         read_lock_irqsave(&tasklist_lock, flags);
4100         do_each_thread(g, t) {
4101                 if (start == end) {
4102                         ret = -EAGAIN;
4103                         goto unlock;
4104                 }
4105
4106                 if (t->ret_stack == NULL) {
4107                         atomic_set(&t->tracing_graph_pause, 0);
4108                         atomic_set(&t->trace_overrun, 0);
4109                         t->curr_ret_stack = -1;
4110                         /* Make sure the tasks see the -1 first: */
4111                         smp_wmb();
4112                         t->ret_stack = ret_stack_list[start++];
4113                 }
4114         } while_each_thread(g, t);
4115
4116 unlock:
4117         read_unlock_irqrestore(&tasklist_lock, flags);
4118 free:
4119         for (i = start; i < end; i++)
4120                 kfree(ret_stack_list[i]);
4121         return ret;
4122 }
4123
4124 static void
4125 ftrace_graph_probe_sched_switch(void *ignore,
4126                         struct task_struct *prev, struct task_struct *next)
4127 {
4128         unsigned long long timestamp;
4129         int index;
4130
4131         /*
4132          * Does the user want to count the time a function was asleep.
4133          * If so, do not update the time stamps.
4134          */
4135         if (trace_flags & TRACE_ITER_SLEEP_TIME)
4136                 return;
4137
4138         timestamp = trace_clock_local();
4139
4140         prev->ftrace_timestamp = timestamp;
4141
4142         /* only process tasks that we timestamped */
4143         if (!next->ftrace_timestamp)
4144                 return;
4145
4146         /*
4147          * Update all the counters in next to make up for the
4148          * time next was sleeping.
4149          */
4150         timestamp -= next->ftrace_timestamp;
4151
4152         for (index = next->curr_ret_stack; index >= 0; index--)
4153                 next->ret_stack[index].calltime += timestamp;
4154 }
4155
4156 /* Allocate a return stack for each task */
4157 static int start_graph_tracing(void)
4158 {
4159         struct ftrace_ret_stack **ret_stack_list;
4160         int ret, cpu;
4161
4162         ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
4163                                 sizeof(struct ftrace_ret_stack *),
4164                                 GFP_KERNEL);
4165
4166         if (!ret_stack_list)
4167                 return -ENOMEM;
4168
4169         /* The cpu_boot init_task->ret_stack will never be freed */
4170         for_each_online_cpu(cpu) {
4171                 if (!idle_task(cpu)->ret_stack)
4172                         ftrace_graph_init_idle_task(idle_task(cpu), cpu);
4173         }
4174
4175         do {
4176                 ret = alloc_retstack_tasklist(ret_stack_list);
4177         } while (ret == -EAGAIN);
4178
4179         if (!ret) {
4180                 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
4181                 if (ret)
4182                         pr_info("ftrace_graph: Couldn't activate tracepoint"
4183                                 " probe to kernel_sched_switch\n");
4184         }
4185
4186         kfree(ret_stack_list);
4187         return ret;
4188 }
4189
4190 /*
4191  * Hibernation protection.
4192  * The state of the current task is too much unstable during
4193  * suspend/restore to disk. We want to protect against that.
4194  */
4195 static int
4196 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
4197                                                         void *unused)
4198 {
4199         switch (state) {
4200         case PM_HIBERNATION_PREPARE:
4201                 pause_graph_tracing();
4202                 break;
4203
4204         case PM_POST_HIBERNATION:
4205                 unpause_graph_tracing();
4206                 break;
4207         }
4208         return NOTIFY_DONE;
4209 }
4210
4211 /* Just a place holder for function graph */
4212 static struct ftrace_ops fgraph_ops __read_mostly = {
4213         .func           = ftrace_stub,
4214         .flags          = FTRACE_OPS_FL_GLOBAL,
4215 };
4216
4217 static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace)
4218 {
4219         if (!ftrace_ops_test(&global_ops, trace->func))
4220                 return 0;
4221         return __ftrace_graph_entry(trace);
4222 }
4223
4224 /*
4225  * The function graph tracer should only trace the functions defined
4226  * by set_ftrace_filter and set_ftrace_notrace. If another function
4227  * tracer ops is registered, the graph tracer requires testing the
4228  * function against the global ops, and not just trace any function
4229  * that any ftrace_ops registered.
4230  */
4231 static void update_function_graph_func(void)
4232 {
4233         if (ftrace_ops_list == &ftrace_list_end ||
4234             (ftrace_ops_list == &global_ops &&
4235              global_ops.next == &ftrace_list_end))
4236                 ftrace_graph_entry = __ftrace_graph_entry;
4237         else
4238                 ftrace_graph_entry = ftrace_graph_entry_test;
4239 }
4240
4241 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
4242                         trace_func_graph_ent_t entryfunc)
4243 {
4244         int ret = 0;
4245
4246         mutex_lock(&ftrace_lock);
4247
4248         /* we currently allow only one tracer registered at a time */
4249         if (ftrace_graph_active) {
4250                 ret = -EBUSY;
4251                 goto out;
4252         }
4253
4254         ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
4255         register_pm_notifier(&ftrace_suspend_notifier);
4256
4257         ftrace_graph_active++;
4258         ret = start_graph_tracing();
4259         if (ret) {
4260                 ftrace_graph_active--;
4261                 goto out;
4262         }
4263
4264         ftrace_graph_return = retfunc;
4265
4266         /*
4267          * Update the indirect function to the entryfunc, and the
4268          * function that gets called to the entry_test first. Then
4269          * call the update fgraph entry function to determine if
4270          * the entryfunc should be called directly or not.
4271          */
4272         __ftrace_graph_entry = entryfunc;
4273         ftrace_graph_entry = ftrace_graph_entry_test;
4274         update_function_graph_func();
4275
4276         ret = ftrace_startup(&fgraph_ops, FTRACE_START_FUNC_RET);
4277
4278 out:
4279         mutex_unlock(&ftrace_lock);
4280         return ret;
4281 }
4282
4283 void unregister_ftrace_graph(void)
4284 {
4285         mutex_lock(&ftrace_lock);
4286
4287         if (unlikely(!ftrace_graph_active))
4288                 goto out;
4289
4290         ftrace_graph_active--;
4291         ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
4292         ftrace_graph_entry = ftrace_graph_entry_stub;
4293         __ftrace_graph_entry = ftrace_graph_entry_stub;
4294         ftrace_shutdown(&fgraph_ops, FTRACE_STOP_FUNC_RET);
4295         unregister_pm_notifier(&ftrace_suspend_notifier);
4296         unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
4297
4298  out:
4299         mutex_unlock(&ftrace_lock);
4300 }
4301
4302 static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);
4303
4304 static void
4305 graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
4306 {
4307         atomic_set(&t->tracing_graph_pause, 0);
4308         atomic_set(&t->trace_overrun, 0);
4309         t->ftrace_timestamp = 0;
4310         /* make curr_ret_stack visible before we add the ret_stack */
4311         smp_wmb();
4312         t->ret_stack = ret_stack;
4313 }
4314
4315 /*
4316  * Allocate a return stack for the idle task. May be the first
4317  * time through, or it may be done by CPU hotplug online.
4318  */
4319 void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
4320 {
4321         t->curr_ret_stack = -1;
4322         /*
4323          * The idle task has no parent, it either has its own
4324          * stack or no stack at all.
4325          */
4326         if (t->ret_stack)
4327                 WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
4328
4329         if (ftrace_graph_active) {
4330                 struct ftrace_ret_stack *ret_stack;
4331
4332                 ret_stack = per_cpu(idle_ret_stack, cpu);
4333                 if (!ret_stack) {
4334                         ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
4335                                             * sizeof(struct ftrace_ret_stack),
4336                                             GFP_KERNEL);
4337                         if (!ret_stack)
4338                                 return;
4339                         per_cpu(idle_ret_stack, cpu) = ret_stack;
4340                 }
4341                 graph_init_task(t, ret_stack);
4342         }
4343 }
4344
4345 /* Allocate a return stack for newly created task */
4346 void ftrace_graph_init_task(struct task_struct *t)
4347 {
4348         /* Make sure we do not use the parent ret_stack */
4349         t->ret_stack = NULL;
4350         t->curr_ret_stack = -1;
4351
4352         if (ftrace_graph_active) {
4353                 struct ftrace_ret_stack *ret_stack;
4354
4355                 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
4356                                 * sizeof(struct ftrace_ret_stack),
4357                                 GFP_KERNEL);
4358                 if (!ret_stack)
4359                         return;
4360                 graph_init_task(t, ret_stack);
4361         }
4362 }
4363
4364 void ftrace_graph_exit_task(struct task_struct *t)
4365 {
4366         struct ftrace_ret_stack *ret_stack = t->ret_stack;
4367
4368         t->ret_stack = NULL;
4369         /* NULL must become visible to IRQs before we free it: */
4370         barrier();
4371
4372         kfree(ret_stack);
4373 }
4374
4375 void ftrace_graph_stop(void)
4376 {
4377         ftrace_stop();
4378 }
4379 #endif