kernel/trace/ftrace.c

   1 /*
   2  * Infrastructure for profiling code inserted by 'gcc -pg'.
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Originally ported from the -rt patch by:
   8  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
   9  *
  10  * Based on code in the latency_tracer, that is:
  11  *
  12  *  Copyright (C) 2004-2006 Ingo Molnar
  13  *  Copyright (C) 2004 William Lee Irwin III
  14  */
  15
  16 #include <linux/stop_machine.h>
  17 #include <linux/clocksource.h>
  18 #include <linux/kallsyms.h>
  19 #include <linux/seq_file.h>
  20 #include <linux/suspend.h>
  21 #include <linux/debugfs.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/kthread.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/kprobes.h>
  26 #include <linux/ftrace.h>
  27 #include <linux/sysctl.h>
  28 #include <linux/ctype.h>
  29 #include <linux/list.h>
  30 #include <linux/hash.h>
  31
  32 #include <asm/ftrace.h>
  33
  34 #include "trace.h"
  35
  36 #define FTRACE_WARN_ON(cond)                    \
  37         do {                                    \
  38                 if (WARN_ON(cond))              \
  39                         ftrace_kill();          \
  40         } while (0)
  41
  42 #define FTRACE_WARN_ON_ONCE(cond)               \
  43         do {                                    \
  44                 if (WARN_ON_ONCE(cond))         \
  45                         ftrace_kill();          \
  46         } while (0)
  47
  48 /* hash bits for specific function selection */
  49 #define FTRACE_HASH_BITS 7
  50 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
  51
  52 /* ftrace_enabled is a method to turn ftrace on or off */
  53 int ftrace_enabled __read_mostly;
  54 static int last_ftrace_enabled;
  55
  56 /* Quick disabling of function tracer. */
  57 int function_trace_stop;
  58
  59 /*
  60  * ftrace_disabled is set when an anomaly is discovered.
  61  * ftrace_disabled is much stronger than ftrace_enabled.
  62  */
  63 static int ftrace_disabled __read_mostly;
  64
  65 static DEFINE_MUTEX(ftrace_lock);
  66
  67 static struct ftrace_ops ftrace_list_end __read_mostly =
  68 {
  69         .func = ftrace_stub,
  70 };
  71
  72 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
  73 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
  74 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
  75 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
  76
  77 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
  78 {
  79         struct ftrace_ops *op = ftrace_list;
  80
  81         /* in case someone actually ports this to alpha! */
  82         read_barrier_depends();
  83
  84         while (op != &ftrace_list_end) {
  85                 /* silly alpha */
  86                 read_barrier_depends();
  87                 op->func(ip, parent_ip);
  88                 op = op->next;
  89         };
  90 }
  91
  92 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
  93 {
  94         if (!test_tsk_trace_trace(current))
  95                 return;
  96
  97         ftrace_pid_function(ip, parent_ip);
  98 }
  99
 100 static void set_ftrace_pid_function(ftrace_func_t func)
 101 {
 102         /* do not set ftrace_pid_function to itself! */
 103         if (func != ftrace_pid_func)
 104                 ftrace_pid_function = func;
 105 }
 106
 107 /**
 108  * clear_ftrace_function - reset the ftrace function
 109  *
 110  * This NULLs the ftrace function and in essence stops
 111  * tracing.  There may be lag
 112  */
 113 void clear_ftrace_function(void)
 114 {
 115         ftrace_trace_function = ftrace_stub;
 116         __ftrace_trace_function = ftrace_stub;
 117         ftrace_pid_function = ftrace_stub;
 118 }
 119
 120 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 121 /*
 122  * For those archs that do not test ftrace_trace_stop in their
 123  * mcount call site, we need to do it from C.
 124  */
 125 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
 126 {
 127         if (function_trace_stop)
 128                 return;
 129
 130         __ftrace_trace_function(ip, parent_ip);
 131 }
 132 #endif
 133
 134 static int __register_ftrace_function(struct ftrace_ops *ops)
 135 {
 136         ops->next = ftrace_list;
 137         /*
 138          * We are entering ops into the ftrace_list but another
 139          * CPU might be walking that list. We need to make sure
 140          * the ops->next pointer is valid before another CPU sees
 141          * the ops pointer included into the ftrace_list.
 142          */
 143         smp_wmb();
 144         ftrace_list = ops;
 145
 146         if (ftrace_enabled) {
 147                 ftrace_func_t func;
 148
 149                 if (ops->next == &ftrace_list_end)
 150                         func = ops->func;
 151                 else
 152                         func = ftrace_list_func;
 153
 154                 if (ftrace_pid_trace) {
 155                         set_ftrace_pid_function(func);
 156                         func = ftrace_pid_func;
 157                 }
 158
 159                 /*
 160                  * For one func, simply call it directly.
 161                  * For more than one func, call the chain.
 162                  */
 163 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 164                 ftrace_trace_function = func;
 165 #else
 166                 __ftrace_trace_function = func;
 167                 ftrace_trace_function = ftrace_test_stop_func;
 168 #endif
 169         }
 170
 171         return 0;
 172 }
 173
 174 static int __unregister_ftrace_function(struct ftrace_ops *ops)
 175 {
 176         struct ftrace_ops **p;
 177
 178         /*
 179          * If we are removing the last function, then simply point
 180          * to the ftrace_stub.
 181          */
 182         if (ftrace_list == ops && ops->next == &ftrace_list_end) {
 183                 ftrace_trace_function = ftrace_stub;
 184                 ftrace_list = &ftrace_list_end;
 185                 return 0;
 186         }
 187
 188         for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
 189                 if (*p == ops)
 190                         break;
 191
 192         if (*p != ops)
 193                 return -1;
 194
 195         *p = (*p)->next;
 196
 197         if (ftrace_enabled) {
 198                 /* If we only have one func left, then call that directly */
 199                 if (ftrace_list->next == &ftrace_list_end) {
 200                         ftrace_func_t func = ftrace_list->func;
 201
 202                         if (ftrace_pid_trace) {
 203                                 set_ftrace_pid_function(func);
 204                                 func = ftrace_pid_func;
 205                         }
 206 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 207                         ftrace_trace_function = func;
 208 #else
 209                         __ftrace_trace_function = func;
 210 #endif
 211                 }
 212         }
 213
 214         return 0;
 215 }
 216
 217 static void ftrace_update_pid_func(void)
 218 {
 219         ftrace_func_t func;
 220
 221         mutex_lock(&ftrace_lock);
 222
 223         if (ftrace_trace_function == ftrace_stub)
 224                 goto out;
 225
 226         func = ftrace_trace_function;
 227
 228         if (ftrace_pid_trace) {
 229                 set_ftrace_pid_function(func);
 230                 func = ftrace_pid_func;
 231         } else {
 232                 if (func == ftrace_pid_func)
 233                         func = ftrace_pid_function;
 234         }
 235
 236 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 237         ftrace_trace_function = func;
 238 #else
 239         __ftrace_trace_function = func;
 240 #endif
 241
 242  out:
 243         mutex_unlock(&ftrace_lock);
 244 }
 245
 246 /* set when tracing only a pid */
 247 struct pid *ftrace_pid_trace;
 248 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
 249
 250 #ifdef CONFIG_DYNAMIC_FTRACE
 251
 252 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
 253 # error Dynamic ftrace depends on MCOUNT_RECORD
 254 #endif
 255
 256 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
 257
 258 struct ftrace_func_probe {
 259         struct hlist_node       node;
 260         struct ftrace_probe_ops *ops;
 261         unsigned long           flags;
 262         unsigned long           ip;
 263         void                    *data;
 264         struct rcu_head         rcu;
 265 };
 266
 267
 268 enum {
 269         FTRACE_ENABLE_CALLS             = (1 << 0),
 270         FTRACE_DISABLE_CALLS            = (1 << 1),
 271         FTRACE_UPDATE_TRACE_FUNC        = (1 << 2),
 272         FTRACE_ENABLE_MCOUNT            = (1 << 3),
 273         FTRACE_DISABLE_MCOUNT           = (1 << 4),
 274         FTRACE_START_FUNC_RET           = (1 << 5),
 275         FTRACE_STOP_FUNC_RET            = (1 << 6),
 276 };
 277
 278 static int ftrace_filtered;
 279
 280 static LIST_HEAD(ftrace_new_addrs);
 281
 282 static DEFINE_MUTEX(ftrace_regex_lock);
 283
 284 struct ftrace_page {
 285         struct ftrace_page      *next;
 286         int                     index;
 287         struct dyn_ftrace       records[];
 288 };
 289
 290 #define ENTRIES_PER_PAGE \
 291   ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
 292
 293 /* estimate from running different kernels */
 294 #define NR_TO_INIT              10000
 295
 296 static struct ftrace_page       *ftrace_pages_start;
 297 static struct ftrace_page       *ftrace_pages;
 298
 299 static struct dyn_ftrace *ftrace_free_records;
 300
 301 /*
 302  * This is a double for. Do not use 'break' to break out of the loop,
 303  * you must use a goto.
 304  */
 305 #define do_for_each_ftrace_rec(pg, rec)                                 \
 306         for (pg = ftrace_pages_start; pg; pg = pg->next) {              \
 307                 int _____i;                                             \
 308                 for (_____i = 0; _____i < pg->index; _____i++) {        \
 309                         rec = &pg->records[_____i];
 310
 311 #define while_for_each_ftrace_rec()             \
 312                 }                               \
 313         }
 314
 315 #ifdef CONFIG_KPROBES
 316
 317 static int frozen_record_count;
 318
 319 static inline void freeze_record(struct dyn_ftrace *rec)
 320 {
 321         if (!(rec->flags & FTRACE_FL_FROZEN)) {
 322                 rec->flags |= FTRACE_FL_FROZEN;
 323                 frozen_record_count++;
 324         }
 325 }
 326
 327 static inline void unfreeze_record(struct dyn_ftrace *rec)
 328 {
 329         if (rec->flags & FTRACE_FL_FROZEN) {
 330                 rec->flags &= ~FTRACE_FL_FROZEN;
 331                 frozen_record_count--;
 332         }
 333 }
 334
 335 static inline int record_frozen(struct dyn_ftrace *rec)
 336 {
 337         return rec->flags & FTRACE_FL_FROZEN;
 338 }
 339 #else
 340 # define freeze_record(rec)                     ({ 0; })
 341 # define unfreeze_record(rec)                   ({ 0; })
 342 # define record_frozen(rec)                     ({ 0; })
 343 #endif /* CONFIG_KPROBES */
 344
 345 static void ftrace_free_rec(struct dyn_ftrace *rec)
 346 {
 347         rec->ip = (unsigned long)ftrace_free_records;
 348         ftrace_free_records = rec;
 349         rec->flags |= FTRACE_FL_FREE;
 350 }
 351
 352 void ftrace_release(void *start, unsigned long size)
 353 {
 354         struct dyn_ftrace *rec;
 355         struct ftrace_page *pg;
 356         unsigned long s = (unsigned long)start;
 357         unsigned long e = s + size;
 358
 359         if (ftrace_disabled || !start)
 360                 return;
 361
 362         mutex_lock(&ftrace_lock);
 363         do_for_each_ftrace_rec(pg, rec) {
 364                 if ((rec->ip >= s) && (rec->ip < e))
 365                         ftrace_free_rec(rec);
 366         } while_for_each_ftrace_rec();
 367         mutex_unlock(&ftrace_lock);
 368 }
 369
 370 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
 371 {
 372         struct dyn_ftrace *rec;
 373
 374         /* First check for freed records */
 375         if (ftrace_free_records) {
 376                 rec = ftrace_free_records;
 377
 378                 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
 379                         FTRACE_WARN_ON_ONCE(1);
 380                         ftrace_free_records = NULL;
 381                         return NULL;
 382                 }
 383
 384                 ftrace_free_records = (void *)rec->ip;
 385                 memset(rec, 0, sizeof(*rec));
 386                 return rec;
 387         }
 388
 389         if (ftrace_pages->index == ENTRIES_PER_PAGE) {
 390                 if (!ftrace_pages->next) {
 391                         /* allocate another page */
 392                         ftrace_pages->next =
 393                                 (void *)get_zeroed_page(GFP_KERNEL);
 394                         if (!ftrace_pages->next)
 395                                 return NULL;
 396                 }
 397                 ftrace_pages = ftrace_pages->next;
 398         }
 399
 400         return &ftrace_pages->records[ftrace_pages->index++];
 401 }
 402
 403 static struct dyn_ftrace *
 404 ftrace_record_ip(unsigned long ip)
 405 {
 406         struct dyn_ftrace *rec;
 407
 408         if (ftrace_disabled)
 409                 return NULL;
 410
 411         rec = ftrace_alloc_dyn_node(ip);
 412         if (!rec)
 413                 return NULL;
 414
 415         rec->ip = ip;
 416
 417         list_add(&rec->list, &ftrace_new_addrs);
 418
 419         return rec;
 420 }
 421
 422 static void print_ip_ins(const char *fmt, unsigned char *p)
 423 {
 424         int i;
 425
 426         printk(KERN_CONT "%s", fmt);
 427
 428         for (i = 0; i < MCOUNT_INSN_SIZE; i++)
 429                 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
 430 }
 431
 432 static void ftrace_bug(int failed, unsigned long ip)
 433 {
 434         switch (failed) {
 435         case -EFAULT:
 436                 FTRACE_WARN_ON_ONCE(1);
 437                 pr_info("ftrace faulted on modifying ");
 438                 print_ip_sym(ip);
 439                 break;
 440         case -EINVAL:
 441                 FTRACE_WARN_ON_ONCE(1);
 442                 pr_info("ftrace failed to modify ");
 443                 print_ip_sym(ip);
 444                 print_ip_ins(" actual: ", (unsigned char *)ip);
 445                 printk(KERN_CONT "\n");
 446                 break;
 447         case -EPERM:
 448                 FTRACE_WARN_ON_ONCE(1);
 449                 pr_info("ftrace faulted on writing ");
 450                 print_ip_sym(ip);
 451                 break;
 452         default:
 453                 FTRACE_WARN_ON_ONCE(1);
 454                 pr_info("ftrace faulted on unknown error ");
 455                 print_ip_sym(ip);
 456         }
 457 }
 458
 459
 460 static int
 461 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
 462 {
 463         unsigned long ftrace_addr;
 464         unsigned long ip, fl;
 465
 466         ftrace_addr = (unsigned long)FTRACE_ADDR;
 467
 468         ip = rec->ip;
 469
 470         /*
 471          * If this record is not to be traced and
 472          * it is not enabled then do nothing.
 473          *
 474          * If this record is not to be traced and
 475          * it is enabled then disable it.
 476          *
 477          */
 478         if (rec->flags & FTRACE_FL_NOTRACE) {
 479                 if (rec->flags & FTRACE_FL_ENABLED)
 480                         rec->flags &= ~FTRACE_FL_ENABLED;
 481                 else
 482                         return 0;
 483
 484         } else if (ftrace_filtered && enable) {
 485                 /*
 486                  * Filtering is on:
 487                  */
 488
 489                 fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_ENABLED);
 490
 491                 /* Record is filtered and enabled, do nothing */
 492                 if (fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED))
 493                         return 0;
 494
 495                 /* Record is not filtered or enabled, do nothing */
 496                 if (!fl)
 497                         return 0;
 498
 499                 /* Record is not filtered but enabled, disable it */
 500                 if (fl == FTRACE_FL_ENABLED)
 501                         rec->flags &= ~FTRACE_FL_ENABLED;
 502                 else
 503                 /* Otherwise record is filtered but not enabled, enable it */
 504                         rec->flags |= FTRACE_FL_ENABLED;
 505         } else {
 506                 /* Disable or not filtered */
 507
 508                 if (enable) {
 509                         /* if record is enabled, do nothing */
 510                         if (rec->flags & FTRACE_FL_ENABLED)
 511                                 return 0;
 512
 513                         rec->flags |= FTRACE_FL_ENABLED;
 514
 515                 } else {
 516
 517                         /* if record is not enabled, do nothing */
 518                         if (!(rec->flags & FTRACE_FL_ENABLED))
 519                                 return 0;
 520
 521                         rec->flags &= ~FTRACE_FL_ENABLED;
 522                 }
 523         }
 524
 525         if (rec->flags & FTRACE_FL_ENABLED)
 526                 return ftrace_make_call(rec, ftrace_addr);
 527         else
 528                 return ftrace_make_nop(NULL, rec, ftrace_addr);
 529 }
 530
 531 static void ftrace_replace_code(int enable)
 532 {
 533         struct dyn_ftrace *rec;
 534         struct ftrace_page *pg;
 535         int failed;
 536
 537         do_for_each_ftrace_rec(pg, rec) {
 538                 /*
 539                  * Skip over free records and records that have
 540                  * failed.
 541                  */
 542                 if (rec->flags & FTRACE_FL_FREE ||
 543                     rec->flags & FTRACE_FL_FAILED)
 544                         continue;
 545
 546                 /* ignore updates to this record's mcount site */
 547                 if (get_kprobe((void *)rec->ip)) {
 548                         freeze_record(rec);
 549                         continue;
 550                 } else {
 551                         unfreeze_record(rec);
 552                 }
 553
 554                 failed = __ftrace_replace_code(rec, enable);
 555                 if (failed && (rec->flags & FTRACE_FL_CONVERTED)) {
 556                         rec->flags |= FTRACE_FL_FAILED;
 557                         if ((system_state == SYSTEM_BOOTING) ||
 558                             !core_kernel_text(rec->ip)) {
 559                                 ftrace_free_rec(rec);
 560                                 } else {
 561                                 ftrace_bug(failed, rec->ip);
 562                                         /* Stop processing */
 563                                         return;
 564                                 }
 565                 }
 566         } while_for_each_ftrace_rec();
 567 }
 568
 569 static int
 570 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
 571 {
 572         unsigned long ip;
 573         int ret;
 574
 575         ip = rec->ip;
 576
 577         ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
 578         if (ret) {
 579                 ftrace_bug(ret, ip);
 580                 rec->flags |= FTRACE_FL_FAILED;
 581                 return 0;
 582         }
 583         return 1;
 584 }
 585
 586 /*
 587  * archs can override this function if they must do something
 588  * before the modifying code is performed.
 589  */
 590 int __weak ftrace_arch_code_modify_prepare(void)
 591 {
 592         return 0;
 593 }
 594
 595 /*
 596  * archs can override this function if they must do something
 597  * after the modifying code is performed.
 598  */
 599 int __weak ftrace_arch_code_modify_post_process(void)
 600 {
 601         return 0;
 602 }
 603
 604 static int __ftrace_modify_code(void *data)
 605 {
 606         int *command = data;
 607
 608         if (*command & FTRACE_ENABLE_CALLS)
 609                 ftrace_replace_code(1);
 610         else if (*command & FTRACE_DISABLE_CALLS)
 611                 ftrace_replace_code(0);
 612
 613         if (*command & FTRACE_UPDATE_TRACE_FUNC)
 614                 ftrace_update_ftrace_func(ftrace_trace_function);
 615
 616         if (*command & FTRACE_START_FUNC_RET)
 617                 ftrace_enable_ftrace_graph_caller();
 618         else if (*command & FTRACE_STOP_FUNC_RET)
 619                 ftrace_disable_ftrace_graph_caller();
 620
 621         return 0;
 622 }
 623
 624 static void ftrace_run_update_code(int command)
 625 {
 626         int ret;
 627
 628         ret = ftrace_arch_code_modify_prepare();
 629         FTRACE_WARN_ON(ret);
 630         if (ret)
 631                 return;
 632
 633         stop_machine(__ftrace_modify_code, &command, NULL);
 634
 635         ret = ftrace_arch_code_modify_post_process();
 636         FTRACE_WARN_ON(ret);
 637 }
 638
 639 static ftrace_func_t saved_ftrace_func;
 640 static int ftrace_start_up;
 641
 642 static void ftrace_startup_enable(int command)
 643 {
 644         if (saved_ftrace_func != ftrace_trace_function) {
 645                 saved_ftrace_func = ftrace_trace_function;
 646                 command |= FTRACE_UPDATE_TRACE_FUNC;
 647         }
 648
 649         if (!command || !ftrace_enabled)
 650                 return;
 651
 652         ftrace_run_update_code(command);
 653 }
 654
 655 static void ftrace_startup(int command)
 656 {
 657         if (unlikely(ftrace_disabled))
 658                 return;
 659
 660         ftrace_start_up++;
 661         command |= FTRACE_ENABLE_CALLS;
 662
 663         ftrace_startup_enable(command);
 664 }
 665
 666 static void ftrace_shutdown(int command)
 667 {
 668         if (unlikely(ftrace_disabled))
 669                 return;
 670
 671         ftrace_start_up--;
 672         if (!ftrace_start_up)
 673                 command |= FTRACE_DISABLE_CALLS;
 674
 675         if (saved_ftrace_func != ftrace_trace_function) {
 676                 saved_ftrace_func = ftrace_trace_function;
 677                 command |= FTRACE_UPDATE_TRACE_FUNC;
 678         }
 679
 680         if (!command || !ftrace_enabled)
 681                 return;
 682
 683         ftrace_run_update_code(command);
 684 }
 685
 686 static void ftrace_startup_sysctl(void)
 687 {
 688         int command = FTRACE_ENABLE_MCOUNT;
 689
 690         if (unlikely(ftrace_disabled))
 691                 return;
 692
 693         /* Force update next time */
 694         saved_ftrace_func = NULL;
 695         /* ftrace_start_up is true if we want ftrace running */
 696         if (ftrace_start_up)
 697                 command |= FTRACE_ENABLE_CALLS;
 698
 699         ftrace_run_update_code(command);
 700 }
 701
 702 static void ftrace_shutdown_sysctl(void)
 703 {
 704         int command = FTRACE_DISABLE_MCOUNT;
 705
 706         if (unlikely(ftrace_disabled))
 707                 return;
 708
 709         /* ftrace_start_up is true if ftrace is running */
 710         if (ftrace_start_up)
 711                 command |= FTRACE_DISABLE_CALLS;
 712
 713         ftrace_run_update_code(command);
 714 }
 715
 716 static cycle_t          ftrace_update_time;
 717 static unsigned long    ftrace_update_cnt;
 718 unsigned long           ftrace_update_tot_cnt;
 719
 720 static int ftrace_update_code(struct module *mod)
 721 {
 722         struct dyn_ftrace *p, *t;
 723         cycle_t start, stop;
 724
 725         start = ftrace_now(raw_smp_processor_id());
 726         ftrace_update_cnt = 0;
 727
 728         list_for_each_entry_safe(p, t, &ftrace_new_addrs, list) {
 729
 730                 /* If something went wrong, bail without enabling anything */
 731                 if (unlikely(ftrace_disabled))
 732                         return -1;
 733
 734                 list_del_init(&p->list);
 735
 736                 /* convert record (i.e, patch mcount-call with NOP) */
 737                 if (ftrace_code_disable(mod, p)) {
 738                         p->flags |= FTRACE_FL_CONVERTED;
 739                         ftrace_update_cnt++;
 740                 } else
 741                         ftrace_free_rec(p);
 742         }
 743
 744         stop = ftrace_now(raw_smp_processor_id());
 745         ftrace_update_time = stop - start;
 746         ftrace_update_tot_cnt += ftrace_update_cnt;
 747
 748         return 0;
 749 }
 750
 751 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
 752 {
 753         struct ftrace_page *pg;
 754         int cnt;
 755         int i;
 756
 757         /* allocate a few pages */
 758         ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
 759         if (!ftrace_pages_start)
 760                 return -1;
 761
 762         /*
 763          * Allocate a few more pages.
 764          *
 765          * TODO: have some parser search vmlinux before
 766          *   final linking to find all calls to ftrace.
 767          *   Then we can:
 768          *    a) know how many pages to allocate.
 769          *     and/or
 770          *    b) set up the table then.
 771          *
 772          *  The dynamic code is still necessary for
 773          *  modules.
 774          */
 775
 776         pg = ftrace_pages = ftrace_pages_start;
 777
 778         cnt = num_to_init / ENTRIES_PER_PAGE;
 779         pr_info("ftrace: allocating %ld entries in %d pages\n",
 780                 num_to_init, cnt + 1);
 781
 782         for (i = 0; i < cnt; i++) {
 783                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
 784
 785                 /* If we fail, we'll try later anyway */
 786                 if (!pg->next)
 787                         break;
 788
 789                 pg = pg->next;
 790         }
 791
 792         return 0;
 793 }
 794
 795 enum {
 796         FTRACE_ITER_FILTER      = (1 << 0),
 797         FTRACE_ITER_CONT        = (1 << 1),
 798         FTRACE_ITER_NOTRACE     = (1 << 2),
 799         FTRACE_ITER_FAILURES    = (1 << 3),
 800         FTRACE_ITER_PRINTALL    = (1 << 4),
 801         FTRACE_ITER_HASH        = (1 << 5),
 802 };
 803
 804 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
 805
 806 struct ftrace_iterator {
 807         struct ftrace_page      *pg;
 808         int                     hidx;
 809         int                     idx;
 810         unsigned                flags;
 811         unsigned char           buffer[FTRACE_BUFF_MAX+1];
 812         unsigned                buffer_idx;
 813         unsigned                filtered;
 814 };
 815
 816 static void *
 817 t_hash_next(struct seq_file *m, void *v, loff_t *pos)
 818 {
 819         struct ftrace_iterator *iter = m->private;
 820         struct hlist_node *hnd = v;
 821         struct hlist_head *hhd;
 822
 823         WARN_ON(!(iter->flags & FTRACE_ITER_HASH));
 824
 825         (*pos)++;
 826
 827  retry:
 828         if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
 829                 return NULL;
 830
 831         hhd = &ftrace_func_hash[iter->hidx];
 832
 833         if (hlist_empty(hhd)) {
 834                 iter->hidx++;
 835                 hnd = NULL;
 836                 goto retry;
 837         }
 838
 839         if (!hnd)
 840                 hnd = hhd->first;
 841         else {
 842                 hnd = hnd->next;
 843                 if (!hnd) {
 844                         iter->hidx++;
 845                         goto retry;
 846                 }
 847         }
 848
 849         return hnd;
 850 }
 851
 852 static void *t_hash_start(struct seq_file *m, loff_t *pos)
 853 {
 854         struct ftrace_iterator *iter = m->private;
 855         void *p = NULL;
 856
 857         iter->flags |= FTRACE_ITER_HASH;
 858
 859         return t_hash_next(m, p, pos);
 860 }
 861
 862 static int t_hash_show(struct seq_file *m, void *v)
 863 {
 864         struct ftrace_func_probe *rec;
 865         struct hlist_node *hnd = v;
 866         char str[KSYM_SYMBOL_LEN];
 867
 868         rec = hlist_entry(hnd, struct ftrace_func_probe, node);
 869
 870         if (rec->ops->print)
 871                 return rec->ops->print(m, rec->ip, rec->ops, rec->data);
 872
 873         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
 874         seq_printf(m, "%s:", str);
 875
 876         kallsyms_lookup((unsigned long)rec->ops->func, NULL, NULL, NULL, str);
 877         seq_printf(m, "%s", str);
 878
 879         if (rec->data)
 880                 seq_printf(m, ":%p", rec->data);
 881         seq_putc(m, '\n');
 882
 883         return 0;
 884 }
 885
 886 static void *
 887 t_next(struct seq_file *m, void *v, loff_t *pos)
 888 {
 889         struct ftrace_iterator *iter = m->private;
 890         struct dyn_ftrace *rec = NULL;
 891
 892         if (iter->flags & FTRACE_ITER_HASH)
 893                 return t_hash_next(m, v, pos);
 894
 895         (*pos)++;
 896
 897         if (iter->flags & FTRACE_ITER_PRINTALL)
 898                 return NULL;
 899
 900  retry:
 901         if (iter->idx >= iter->pg->index) {
 902                 if (iter->pg->next) {
 903                         iter->pg = iter->pg->next;
 904                         iter->idx = 0;
 905                         goto retry;
 906                 } else {
 907                         iter->idx = -1;
 908                 }
 909         } else {
 910                 rec = &iter->pg->records[iter->idx++];
 911                 if ((rec->flags & FTRACE_FL_FREE) ||
 912
 913                     (!(iter->flags & FTRACE_ITER_FAILURES) &&
 914                      (rec->flags & FTRACE_FL_FAILED)) ||
 915
 916                     ((iter->flags & FTRACE_ITER_FAILURES) &&
 917                      !(rec->flags & FTRACE_FL_FAILED)) ||
 918
 919                     ((iter->flags & FTRACE_ITER_FILTER) &&
 920                      !(rec->flags & FTRACE_FL_FILTER)) ||
 921
 922                     ((iter->flags & FTRACE_ITER_NOTRACE) &&
 923                      !(rec->flags & FTRACE_FL_NOTRACE))) {
 924                         rec = NULL;
 925                         goto retry;
 926                 }
 927         }
 928
 929         return rec;
 930 }
 931
 932 static void *t_start(struct seq_file *m, loff_t *pos)
 933 {
 934         struct ftrace_iterator *iter = m->private;
 935         void *p = NULL;
 936
 937         mutex_lock(&ftrace_lock);
 938         /*
 939          * For set_ftrace_filter reading, if we have the filter
 940          * off, we can short cut and just print out that all
 941          * functions are enabled.
 942          */
 943         if (iter->flags & FTRACE_ITER_FILTER && !ftrace_filtered) {
 944                 if (*pos > 0)
 945                         return t_hash_start(m, pos);
 946                 iter->flags |= FTRACE_ITER_PRINTALL;
 947                 (*pos)++;
 948                 return iter;
 949         }
 950
 951         if (iter->flags & FTRACE_ITER_HASH)
 952                 return t_hash_start(m, pos);
 953
 954         if (*pos > 0) {
 955                 if (iter->idx < 0)
 956                         return p;
 957                 (*pos)--;
 958                 iter->idx--;
 959         }
 960
 961         p = t_next(m, p, pos);
 962
 963         if (!p)
 964                 return t_hash_start(m, pos);
 965
 966         return p;
 967 }
 968
 969 static void t_stop(struct seq_file *m, void *p)
 970 {
 971         mutex_unlock(&ftrace_lock);
 972 }
 973
 974 static int t_show(struct seq_file *m, void *v)
 975 {
 976         struct ftrace_iterator *iter = m->private;
 977         struct dyn_ftrace *rec = v;
 978         char str[KSYM_SYMBOL_LEN];
 979
 980         if (iter->flags & FTRACE_ITER_HASH)
 981                 return t_hash_show(m, v);
 982
 983         if (iter->flags & FTRACE_ITER_PRINTALL) {
 984                 seq_printf(m, "#### all functions enabled ####\n");
 985                 return 0;
 986         }
 987
 988         if (!rec)
 989                 return 0;
 990
 991         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
 992
 993         seq_printf(m, "%s\n", str);
 994
 995         return 0;
 996 }
 997
 998 static struct seq_operations show_ftrace_seq_ops = {
 999         .start = t_start,
1000         .next = t_next,
1001         .stop = t_stop,
1002         .show = t_show,
1003 };
1004
1005 static int
1006 ftrace_avail_open(struct inode *inode, struct file *file)
1007 {
1008         struct ftrace_iterator *iter;
1009         int ret;
1010
1011         if (unlikely(ftrace_disabled))
1012                 return -ENODEV;
1013
1014         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1015         if (!iter)
1016                 return -ENOMEM;
1017
1018         iter->pg = ftrace_pages_start;
1019
1020         ret = seq_open(file, &show_ftrace_seq_ops);
1021         if (!ret) {
1022                 struct seq_file *m = file->private_data;
1023
1024                 m->private = iter;
1025         } else {
1026                 kfree(iter);
1027         }
1028
1029         return ret;
1030 }
1031
1032 int ftrace_avail_release(struct inode *inode, struct file *file)
1033 {
1034         struct seq_file *m = (struct seq_file *)file->private_data;
1035         struct ftrace_iterator *iter = m->private;
1036
1037         seq_release(inode, file);
1038         kfree(iter);
1039
1040         return 0;
1041 }
1042
1043 static int
1044 ftrace_failures_open(struct inode *inode, struct file *file)
1045 {
1046         int ret;
1047         struct seq_file *m;
1048         struct ftrace_iterator *iter;
1049
1050         ret = ftrace_avail_open(inode, file);
1051         if (!ret) {
1052                 m = (struct seq_file *)file->private_data;
1053                 iter = (struct ftrace_iterator *)m->private;
1054                 iter->flags = FTRACE_ITER_FAILURES;
1055         }
1056
1057         return ret;
1058 }
1059
1060
1061 static void ftrace_filter_reset(int enable)
1062 {
1063         struct ftrace_page *pg;
1064         struct dyn_ftrace *rec;
1065         unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1066
1067         mutex_lock(&ftrace_lock);
1068         if (enable)
1069                 ftrace_filtered = 0;
1070         do_for_each_ftrace_rec(pg, rec) {
1071                 if (rec->flags & FTRACE_FL_FAILED)
1072                         continue;
1073                 rec->flags &= ~type;
1074         } while_for_each_ftrace_rec();
1075         mutex_unlock(&ftrace_lock);
1076 }
1077
1078 static int
1079 ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1080 {
1081         struct ftrace_iterator *iter;
1082         int ret = 0;
1083
1084         if (unlikely(ftrace_disabled))
1085                 return -ENODEV;
1086
1087         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1088         if (!iter)
1089                 return -ENOMEM;
1090
1091         mutex_lock(&ftrace_regex_lock);
1092         if ((file->f_mode & FMODE_WRITE) &&
1093             !(file->f_flags & O_APPEND))
1094                 ftrace_filter_reset(enable);
1095
1096         if (file->f_mode & FMODE_READ) {
1097                 iter->pg = ftrace_pages_start;
1098                 iter->flags = enable ? FTRACE_ITER_FILTER :
1099                         FTRACE_ITER_NOTRACE;
1100
1101                 ret = seq_open(file, &show_ftrace_seq_ops);
1102                 if (!ret) {
1103                         struct seq_file *m = file->private_data;
1104                         m->private = iter;
1105                 } else
1106                         kfree(iter);
1107         } else
1108                 file->private_data = iter;
1109         mutex_unlock(&ftrace_regex_lock);
1110
1111         return ret;
1112 }
1113
1114 static int
1115 ftrace_filter_open(struct inode *inode, struct file *file)
1116 {
1117         return ftrace_regex_open(inode, file, 1);
1118 }
1119
1120 static int
1121 ftrace_notrace_open(struct inode *inode, struct file *file)
1122 {
1123         return ftrace_regex_open(inode, file, 0);
1124 }
1125
1126 static ssize_t
1127 ftrace_regex_read(struct file *file, char __user *ubuf,
1128                        size_t cnt, loff_t *ppos)
1129 {
1130         if (file->f_mode & FMODE_READ)
1131                 return seq_read(file, ubuf, cnt, ppos);
1132         else
1133                 return -EPERM;
1134 }
1135
1136 static loff_t
1137 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1138 {
1139         loff_t ret;
1140
1141         if (file->f_mode & FMODE_READ)
1142                 ret = seq_lseek(file, offset, origin);
1143         else
1144                 file->f_pos = ret = 1;
1145
1146         return ret;
1147 }
1148
1149 enum {
1150         MATCH_FULL,
1151         MATCH_FRONT_ONLY,
1152         MATCH_MIDDLE_ONLY,
1153         MATCH_END_ONLY,
1154 };
1155
1156 /*
1157  * (static function - no need for kernel doc)
1158  *
1159  * Pass in a buffer containing a glob and this function will
1160  * set search to point to the search part of the buffer and
1161  * return the type of search it is (see enum above).
1162  * This does modify buff.
1163  *
1164  * Returns enum type.
1165  *  search returns the pointer to use for comparison.
1166  *  not returns 1 if buff started with a '!'
1167  *     0 otherwise.
1168  */
1169 static int
1170 ftrace_setup_glob(char *buff, int len, char **search, int *not)
1171 {
1172         int type = MATCH_FULL;
1173         int i;
1174
1175         if (buff[0] == '!') {
1176                 *not = 1;
1177                 buff++;
1178                 len--;
1179         } else
1180                 *not = 0;
1181
1182         *search = buff;
1183
1184         for (i = 0; i < len; i++) {
1185                 if (buff[i] == '*') {
1186                         if (!i) {
1187                                 *search = buff + 1;
1188                                 type = MATCH_END_ONLY;
1189                         } else {
1190                                 if (type == MATCH_END_ONLY)
1191                                         type = MATCH_MIDDLE_ONLY;
1192                                 else
1193                                         type = MATCH_FRONT_ONLY;
1194                                 buff[i] = 0;
1195                                 break;
1196                         }
1197                 }
1198         }
1199
1200         return type;
1201 }
1202
1203 static int ftrace_match(char *str, char *regex, int len, int type)
1204 {
1205         int matched = 0;
1206         char *ptr;
1207
1208         switch (type) {
1209         case MATCH_FULL:
1210                 if (strcmp(str, regex) == 0)
1211                         matched = 1;
1212                 break;
1213         case MATCH_FRONT_ONLY:
1214                 if (strncmp(str, regex, len) == 0)
1215                         matched = 1;
1216                 break;
1217         case MATCH_MIDDLE_ONLY:
1218                 if (strstr(str, regex))
1219                         matched = 1;
1220                 break;
1221         case MATCH_END_ONLY:
1222                 ptr = strstr(str, regex);
1223                 if (ptr && (ptr[len] == 0))
1224                         matched = 1;
1225                 break;
1226         }
1227
1228         return matched;
1229 }
1230
1231 static int
1232 ftrace_match_record(struct dyn_ftrace *rec, char *regex, int len, int type)
1233 {
1234         char str[KSYM_SYMBOL_LEN];
1235
1236         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1237         return ftrace_match(str, regex, len, type);
1238 }
1239
1240 static void ftrace_match_records(char *buff, int len, int enable)
1241 {
1242         unsigned int search_len;
1243         struct ftrace_page *pg;
1244         struct dyn_ftrace *rec;
1245         unsigned long flag;
1246         char *search;
1247         int type;
1248         int not;
1249
1250         flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1251         type = ftrace_setup_glob(buff, len, &search, &not);
1252
1253         search_len = strlen(search);
1254
1255         mutex_lock(&ftrace_lock);
1256         do_for_each_ftrace_rec(pg, rec) {
1257
1258                 if (rec->flags & FTRACE_FL_FAILED)
1259                         continue;
1260
1261                 if (ftrace_match_record(rec, search, search_len, type)) {
1262                         if (not)
1263                                 rec->flags &= ~flag;
1264                         else
1265                                 rec->flags |= flag;
1266                 }
1267                 /*
1268                  * Only enable filtering if we have a function that
1269                  * is filtered on.
1270                  */
1271                 if (enable && (rec->flags & FTRACE_FL_FILTER))
1272                         ftrace_filtered = 1;
1273         } while_for_each_ftrace_rec();
1274         mutex_unlock(&ftrace_lock);
1275 }
1276
1277 static int
1278 ftrace_match_module_record(struct dyn_ftrace *rec, char *mod,
1279                            char *regex, int len, int type)
1280 {
1281         char str[KSYM_SYMBOL_LEN];
1282         char *modname;
1283
1284         kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
1285
1286         if (!modname || strcmp(modname, mod))
1287                 return 0;
1288
1289         /* blank search means to match all funcs in the mod */
1290         if (len)
1291                 return ftrace_match(str, regex, len, type);
1292         else
1293                 return 1;
1294 }
1295
1296 static void ftrace_match_module_records(char *buff, char *mod, int enable)
1297 {
1298         unsigned search_len = 0;
1299         struct ftrace_page *pg;
1300         struct dyn_ftrace *rec;
1301         int type = MATCH_FULL;
1302         char *search = buff;
1303         unsigned long flag;
1304         int not = 0;
1305
1306         flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1307
1308         /* blank or '*' mean the same */
1309         if (strcmp(buff, "*") == 0)
1310                 buff[0] = 0;
1311
1312         /* handle the case of 'dont filter this module' */
1313         if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
1314                 buff[0] = 0;
1315                 not = 1;
1316         }
1317
1318         if (strlen(buff)) {
1319                 type = ftrace_setup_glob(buff, strlen(buff), &search, &not);
1320                 search_len = strlen(search);
1321         }
1322
1323         mutex_lock(&ftrace_lock);
1324         do_for_each_ftrace_rec(pg, rec) {
1325
1326                 if (rec->flags & FTRACE_FL_FAILED)
1327                         continue;
1328
1329                 if (ftrace_match_module_record(rec, mod,
1330                                                search, search_len, type)) {
1331                         if (not)
1332                                 rec->flags &= ~flag;
1333                         else
1334                                 rec->flags |= flag;
1335                 }
1336                 if (enable && (rec->flags & FTRACE_FL_FILTER))
1337                         ftrace_filtered = 1;
1338
1339         } while_for_each_ftrace_rec();
1340         mutex_unlock(&ftrace_lock);
1341 }
1342
1343 /*
1344  * We register the module command as a template to show others how
1345  * to register the a command as well.
1346  */
1347
1348 static int
1349 ftrace_mod_callback(char *func, char *cmd, char *param, int enable)
1350 {
1351         char *mod;
1352
1353         /*
1354          * cmd == 'mod' because we only registered this func
1355          * for the 'mod' ftrace_func_command.
1356          * But if you register one func with multiple commands,
1357          * you can tell which command was used by the cmd
1358          * parameter.
1359          */
1360
1361         /* we must have a module name */
1362         if (!param)
1363                 return -EINVAL;
1364
1365         mod = strsep(&param, ":");
1366         if (!strlen(mod))
1367                 return -EINVAL;
1368
1369         ftrace_match_module_records(func, mod, enable);
1370         return 0;
1371 }
1372
1373 static struct ftrace_func_command ftrace_mod_cmd = {
1374         .name                   = "mod",
1375         .func                   = ftrace_mod_callback,
1376 };
1377
1378 static int __init ftrace_mod_cmd_init(void)
1379 {
1380         return register_ftrace_command(&ftrace_mod_cmd);
1381 }
1382 device_initcall(ftrace_mod_cmd_init);
1383
1384 static void
1385 function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
1386 {
1387         struct ftrace_func_probe *entry;
1388         struct hlist_head *hhd;
1389         struct hlist_node *n;
1390         unsigned long key;
1391         int resched;
1392
1393         key = hash_long(ip, FTRACE_HASH_BITS);
1394
1395         hhd = &ftrace_func_hash[key];
1396
1397         if (hlist_empty(hhd))
1398                 return;
1399
1400         /*
1401          * Disable preemption for these calls to prevent a RCU grace
1402          * period. This syncs the hash iteration and freeing of items
1403          * on the hash. rcu_read_lock is too dangerous here.
1404          */
1405         resched = ftrace_preempt_disable();
1406         hlist_for_each_entry_rcu(entry, n, hhd, node) {
1407                 if (entry->ip == ip)
1408                         entry->ops->func(ip, parent_ip, &entry->data);
1409         }
1410         ftrace_preempt_enable(resched);
1411 }
1412
1413 static struct ftrace_ops trace_probe_ops __read_mostly =
1414 {
1415         .func = function_trace_probe_call,
1416 };
1417
1418 static int ftrace_probe_registered;
1419
1420 static void __enable_ftrace_function_probe(void)
1421 {
1422         int i;
1423
1424         if (ftrace_probe_registered)
1425                 return;
1426
1427         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1428                 struct hlist_head *hhd = &ftrace_func_hash[i];
1429                 if (hhd->first)
1430                         break;
1431         }
1432         /* Nothing registered? */
1433         if (i == FTRACE_FUNC_HASHSIZE)
1434                 return;
1435
1436         __register_ftrace_function(&trace_probe_ops);
1437         ftrace_startup(0);
1438         ftrace_probe_registered = 1;
1439 }
1440
1441 static void __disable_ftrace_function_probe(void)
1442 {
1443         int i;
1444
1445         if (!ftrace_probe_registered)
1446                 return;
1447
1448         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1449                 struct hlist_head *hhd = &ftrace_func_hash[i];
1450                 if (hhd->first)
1451                         return;
1452         }
1453
1454         /* no more funcs left */
1455         __unregister_ftrace_function(&trace_probe_ops);
1456         ftrace_shutdown(0);
1457         ftrace_probe_registered = 0;
1458 }
1459
1460
1461 static void ftrace_free_entry_rcu(struct rcu_head *rhp)
1462 {
1463         struct ftrace_func_probe *entry =
1464                 container_of(rhp, struct ftrace_func_probe, rcu);
1465
1466         if (entry->ops->free)
1467                 entry->ops->free(&entry->data);
1468         kfree(entry);
1469 }
1470
1471
1472 int
1473 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1474                               void *data)
1475 {
1476         struct ftrace_func_probe *entry;
1477         struct ftrace_page *pg;
1478         struct dyn_ftrace *rec;
1479         int type, len, not;
1480         unsigned long key;
1481         int count = 0;
1482         char *search;
1483
1484         type = ftrace_setup_glob(glob, strlen(glob), &search, &not);
1485         len = strlen(search);
1486
1487         /* we do not support '!' for function probes */
1488         if (WARN_ON(not))
1489                 return -EINVAL;
1490
1491         mutex_lock(&ftrace_lock);
1492         do_for_each_ftrace_rec(pg, rec) {
1493
1494                 if (rec->flags & FTRACE_FL_FAILED)
1495                         continue;
1496
1497                 if (!ftrace_match_record(rec, search, len, type))
1498                         continue;
1499
1500                 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1501                 if (!entry) {
1502                         /* If we did not process any, then return error */
1503                         if (!count)
1504                                 count = -ENOMEM;
1505                         goto out_unlock;
1506                 }
1507
1508                 count++;
1509
1510                 entry->data = data;
1511
1512                 /*
1513                  * The caller might want to do something special
1514                  * for each function we find. We call the callback
1515                  * to give the caller an opportunity to do so.
1516                  */
1517                 if (ops->callback) {
1518                         if (ops->callback(rec->ip, &entry->data) < 0) {
1519                                 /* caller does not like this func */
1520                                 kfree(entry);
1521                                 continue;
1522                         }
1523                 }
1524
1525                 entry->ops = ops;
1526                 entry->ip = rec->ip;
1527
1528                 key = hash_long(entry->ip, FTRACE_HASH_BITS);
1529                 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
1530
1531         } while_for_each_ftrace_rec();
1532         __enable_ftrace_function_probe();
1533
1534  out_unlock:
1535         mutex_unlock(&ftrace_lock);
1536
1537         return count;
1538 }
1539
1540 enum {
1541         PROBE_TEST_FUNC         = 1,
1542         PROBE_TEST_DATA         = 2
1543 };
1544
1545 static void
1546 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1547                                   void *data, int flags)
1548 {
1549         struct ftrace_func_probe *entry;
1550         struct hlist_node *n, *tmp;
1551         char str[KSYM_SYMBOL_LEN];
1552         int type = MATCH_FULL;
1553         int i, len = 0;
1554         char *search;
1555
1556         if (glob && (strcmp(glob, "*") || !strlen(glob)))
1557                 glob = NULL;
1558         else {
1559                 int not;
1560
1561                 type = ftrace_setup_glob(glob, strlen(glob), &search, &not);
1562                 len = strlen(search);
1563
1564                 /* we do not support '!' for function probes */
1565                 if (WARN_ON(not))
1566                         return;
1567         }
1568
1569         mutex_lock(&ftrace_lock);
1570         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1571                 struct hlist_head *hhd = &ftrace_func_hash[i];
1572
1573                 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
1574
1575                         /* break up if statements for readability */
1576                         if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
1577                                 continue;
1578
1579                         if ((flags & PROBE_TEST_DATA) && entry->data != data)
1580                                 continue;
1581
1582                         /* do this last, since it is the most expensive */
1583                         if (glob) {
1584                                 kallsyms_lookup(entry->ip, NULL, NULL,
1585                                                 NULL, str);
1586                                 if (!ftrace_match(str, glob, len, type))
1587                                         continue;
1588                         }
1589
1590                         hlist_del(&entry->node);
1591                         call_rcu(&entry->rcu, ftrace_free_entry_rcu);
1592                 }
1593         }
1594         __disable_ftrace_function_probe();
1595         mutex_unlock(&ftrace_lock);
1596 }
1597
1598 void
1599 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1600                                 void *data)
1601 {
1602         __unregister_ftrace_function_probe(glob, ops, data,
1603                                           PROBE_TEST_FUNC | PROBE_TEST_DATA);
1604 }
1605
1606 void
1607 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
1608 {
1609         __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
1610 }
1611
1612 void unregister_ftrace_function_probe_all(char *glob)
1613 {
1614         __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
1615 }
1616
1617 static LIST_HEAD(ftrace_commands);
1618 static DEFINE_MUTEX(ftrace_cmd_mutex);
1619
1620 int register_ftrace_command(struct ftrace_func_command *cmd)
1621 {
1622         struct ftrace_func_command *p;
1623         int ret = 0;
1624
1625         mutex_lock(&ftrace_cmd_mutex);
1626         list_for_each_entry(p, &ftrace_commands, list) {
1627                 if (strcmp(cmd->name, p->name) == 0) {
1628                         ret = -EBUSY;
1629                         goto out_unlock;
1630                 }
1631         }
1632         list_add(&cmd->list, &ftrace_commands);
1633  out_unlock:
1634         mutex_unlock(&ftrace_cmd_mutex);
1635
1636         return ret;
1637 }
1638
1639 int unregister_ftrace_command(struct ftrace_func_command *cmd)
1640 {
1641         struct ftrace_func_command *p, *n;
1642         int ret = -ENODEV;
1643
1644         mutex_lock(&ftrace_cmd_mutex);
1645         list_for_each_entry_safe(p, n, &ftrace_commands, list) {
1646                 if (strcmp(cmd->name, p->name) == 0) {
1647                         ret = 0;
1648                         list_del_init(&p->list);
1649                         goto out_unlock;
1650                 }
1651         }
1652  out_unlock:
1653         mutex_unlock(&ftrace_cmd_mutex);
1654
1655         return ret;
1656 }
1657
1658 static int ftrace_process_regex(char *buff, int len, int enable)
1659 {
1660         char *func, *command, *next = buff;
1661         struct ftrace_func_command *p;
1662         int ret = -EINVAL;
1663
1664         func = strsep(&next, ":");
1665
1666         if (!next) {
1667                 ftrace_match_records(func, len, enable);
1668                 return 0;
1669         }
1670
1671         /* command found */
1672
1673         command = strsep(&next, ":");
1674
1675         mutex_lock(&ftrace_cmd_mutex);
1676         list_for_each_entry(p, &ftrace_commands, list) {
1677                 if (strcmp(p->name, command) == 0) {
1678                         ret = p->func(func, command, next, enable);
1679                         goto out_unlock;
1680                 }
1681         }
1682  out_unlock:
1683         mutex_unlock(&ftrace_cmd_mutex);
1684
1685         return ret;
1686 }
1687
1688 static ssize_t
1689 ftrace_regex_write(struct file *file, const char __user *ubuf,
1690                    size_t cnt, loff_t *ppos, int enable)
1691 {
1692         struct ftrace_iterator *iter;
1693         char ch;
1694         size_t read = 0;
1695         ssize_t ret;
1696
1697         if (!cnt || cnt < 0)
1698                 return 0;
1699
1700         mutex_lock(&ftrace_regex_lock);
1701
1702         if (file->f_mode & FMODE_READ) {
1703                 struct seq_file *m = file->private_data;
1704                 iter = m->private;
1705         } else
1706                 iter = file->private_data;
1707
1708         if (!*ppos) {
1709                 iter->flags &= ~FTRACE_ITER_CONT;
1710                 iter->buffer_idx = 0;
1711         }
1712
1713         ret = get_user(ch, ubuf++);
1714         if (ret)
1715                 goto out;
1716         read++;
1717         cnt--;
1718
1719         if (!(iter->flags & ~FTRACE_ITER_CONT)) {
1720                 /* skip white space */
1721                 while (cnt && isspace(ch)) {
1722                         ret = get_user(ch, ubuf++);
1723                         if (ret)
1724                                 goto out;
1725                         read++;
1726                         cnt--;
1727                 }
1728
1729                 if (isspace(ch)) {
1730                         file->f_pos += read;
1731                         ret = read;
1732                         goto out;
1733                 }
1734
1735                 iter->buffer_idx = 0;
1736         }
1737
1738         while (cnt && !isspace(ch)) {
1739                 if (iter->buffer_idx < FTRACE_BUFF_MAX)
1740                         iter->buffer[iter->buffer_idx++] = ch;
1741                 else {
1742                         ret = -EINVAL;
1743                         goto out;
1744                 }
1745                 ret = get_user(ch, ubuf++);
1746                 if (ret)
1747                         goto out;
1748                 read++;
1749                 cnt--;
1750         }
1751
1752         if (isspace(ch)) {
1753                 iter->filtered++;
1754                 iter->buffer[iter->buffer_idx] = 0;
1755                 ret = ftrace_process_regex(iter->buffer,
1756                                            iter->buffer_idx, enable);
1757                 if (ret)
1758                         goto out;
1759                 iter->buffer_idx = 0;
1760         } else
1761                 iter->flags |= FTRACE_ITER_CONT;
1762
1763
1764         file->f_pos += read;
1765
1766         ret = read;
1767  out:
1768         mutex_unlock(&ftrace_regex_lock);
1769
1770         return ret;
1771 }
1772
1773 static ssize_t
1774 ftrace_filter_write(struct file *file, const char __user *ubuf,
1775                     size_t cnt, loff_t *ppos)
1776 {
1777         return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
1778 }
1779
1780 static ssize_t
1781 ftrace_notrace_write(struct file *file, const char __user *ubuf,
1782                      size_t cnt, loff_t *ppos)
1783 {
1784         return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
1785 }
1786
1787 static void
1788 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
1789 {
1790         if (unlikely(ftrace_disabled))
1791                 return;
1792
1793         mutex_lock(&ftrace_regex_lock);
1794         if (reset)
1795                 ftrace_filter_reset(enable);
1796         if (buf)
1797                 ftrace_match_records(buf, len, enable);
1798         mutex_unlock(&ftrace_regex_lock);
1799 }
1800
1801 /**
1802  * ftrace_set_filter - set a function to filter on in ftrace
1803  * @buf - the string that holds the function filter text.
1804  * @len - the length of the string.
1805  * @reset - non zero to reset all filters before applying this filter.
1806  *
1807  * Filters denote which functions should be enabled when tracing is enabled.
1808  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
1809  */
1810 void ftrace_set_filter(unsigned char *buf, int len, int reset)
1811 {
1812         ftrace_set_regex(buf, len, reset, 1);
1813 }
1814
1815 /**
1816  * ftrace_set_notrace - set a function to not trace in ftrace
1817  * @buf - the string that holds the function notrace text.
1818  * @len - the length of the string.
1819  * @reset - non zero to reset all filters before applying this filter.
1820  *
1821  * Notrace Filters denote which functions should not be enabled when tracing
1822  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
1823  * for tracing.
1824  */
1825 void ftrace_set_notrace(unsigned char *buf, int len, int reset)
1826 {
1827         ftrace_set_regex(buf, len, reset, 0);
1828 }
1829
1830 static int
1831 ftrace_regex_release(struct inode *inode, struct file *file, int enable)
1832 {
1833         struct seq_file *m = (struct seq_file *)file->private_data;
1834         struct ftrace_iterator *iter;
1835
1836         mutex_lock(&ftrace_regex_lock);
1837         if (file->f_mode & FMODE_READ) {
1838                 iter = m->private;
1839
1840                 seq_release(inode, file);
1841         } else
1842                 iter = file->private_data;
1843
1844         if (iter->buffer_idx) {
1845                 iter->filtered++;
1846                 iter->buffer[iter->buffer_idx] = 0;
1847                 ftrace_match_records(iter->buffer, iter->buffer_idx, enable);
1848         }
1849
1850         mutex_lock(&ftrace_lock);
1851         if (ftrace_start_up && ftrace_enabled)
1852                 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
1853         mutex_unlock(&ftrace_lock);
1854
1855         kfree(iter);
1856         mutex_unlock(&ftrace_regex_lock);
1857         return 0;
1858 }
1859
1860 static int
1861 ftrace_filter_release(struct inode *inode, struct file *file)
1862 {
1863         return ftrace_regex_release(inode, file, 1);
1864 }
1865
1866 static int
1867 ftrace_notrace_release(struct inode *inode, struct file *file)
1868 {
1869         return ftrace_regex_release(inode, file, 0);
1870 }
1871
1872 static struct file_operations ftrace_avail_fops = {
1873         .open = ftrace_avail_open,
1874         .read = seq_read,
1875         .llseek = seq_lseek,
1876         .release = ftrace_avail_release,
1877 };
1878
1879 static struct file_operations ftrace_failures_fops = {
1880         .open = ftrace_failures_open,
1881         .read = seq_read,
1882         .llseek = seq_lseek,
1883         .release = ftrace_avail_release,
1884 };
1885
1886 static struct file_operations ftrace_filter_fops = {
1887         .open = ftrace_filter_open,
1888         .read = ftrace_regex_read,
1889         .write = ftrace_filter_write,
1890         .llseek = ftrace_regex_lseek,
1891         .release = ftrace_filter_release,
1892 };
1893
1894 static struct file_operations ftrace_notrace_fops = {
1895         .open = ftrace_notrace_open,
1896         .read = ftrace_regex_read,
1897         .write = ftrace_notrace_write,
1898         .llseek = ftrace_regex_lseek,
1899         .release = ftrace_notrace_release,
1900 };
1901
1902 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1903
1904 static DEFINE_MUTEX(graph_lock);
1905
1906 int ftrace_graph_count;
1907 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
1908
1909 static void *
1910 g_next(struct seq_file *m, void *v, loff_t *pos)
1911 {
1912         unsigned long *array = m->private;
1913         int index = *pos;
1914
1915         (*pos)++;
1916
1917         if (index >= ftrace_graph_count)
1918                 return NULL;
1919
1920         return &array[index];
1921 }
1922
1923 static void *g_start(struct seq_file *m, loff_t *pos)
1924 {
1925         void *p = NULL;
1926
1927         mutex_lock(&graph_lock);
1928
1929         /* Nothing, tell g_show to print all functions are enabled */
1930         if (!ftrace_graph_count && !*pos)
1931                 return (void *)1;
1932
1933         p = g_next(m, p, pos);
1934
1935         return p;
1936 }
1937
1938 static void g_stop(struct seq_file *m, void *p)
1939 {
1940         mutex_unlock(&graph_lock);
1941 }
1942
1943 static int g_show(struct seq_file *m, void *v)
1944 {
1945         unsigned long *ptr = v;
1946         char str[KSYM_SYMBOL_LEN];
1947
1948         if (!ptr)
1949                 return 0;
1950
1951         if (ptr == (unsigned long *)1) {
1952                 seq_printf(m, "#### all functions enabled ####\n");
1953                 return 0;
1954         }
1955
1956         kallsyms_lookup(*ptr, NULL, NULL, NULL, str);
1957
1958         seq_printf(m, "%s\n", str);
1959
1960         return 0;
1961 }
1962
1963 static struct seq_operations ftrace_graph_seq_ops = {
1964         .start = g_start,
1965         .next = g_next,
1966         .stop = g_stop,
1967         .show = g_show,
1968 };
1969
1970 static int
1971 ftrace_graph_open(struct inode *inode, struct file *file)
1972 {
1973         int ret = 0;
1974
1975         if (unlikely(ftrace_disabled))
1976                 return -ENODEV;
1977
1978         mutex_lock(&graph_lock);
1979         if ((file->f_mode & FMODE_WRITE) &&
1980             !(file->f_flags & O_APPEND)) {
1981                 ftrace_graph_count = 0;
1982                 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
1983         }
1984
1985         if (file->f_mode & FMODE_READ) {
1986                 ret = seq_open(file, &ftrace_graph_seq_ops);
1987                 if (!ret) {
1988                         struct seq_file *m = file->private_data;
1989                         m->private = ftrace_graph_funcs;
1990                 }
1991         } else
1992                 file->private_data = ftrace_graph_funcs;
1993         mutex_unlock(&graph_lock);
1994
1995         return ret;
1996 }
1997
1998 static ssize_t
1999 ftrace_graph_read(struct file *file, char __user *ubuf,
2000                        size_t cnt, loff_t *ppos)
2001 {
2002         if (file->f_mode & FMODE_READ)
2003                 return seq_read(file, ubuf, cnt, ppos);
2004         else
2005                 return -EPERM;
2006 }
2007
2008 static int
2009 ftrace_set_func(unsigned long *array, int *idx, char *buffer)
2010 {
2011         struct dyn_ftrace *rec;
2012         struct ftrace_page *pg;
2013         int search_len;
2014         int found = 0;
2015         int type, not;
2016         char *search;
2017         bool exists;
2018         int i;
2019
2020         if (ftrace_disabled)
2021                 return -ENODEV;
2022
2023         /* decode regex */
2024         type = ftrace_setup_glob(buffer, strlen(buffer), &search, &not);
2025         if (not)
2026                 return -EINVAL;
2027
2028         search_len = strlen(search);
2029
2030         mutex_lock(&ftrace_lock);
2031         do_for_each_ftrace_rec(pg, rec) {
2032
2033                 if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
2034                         break;
2035
2036                 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE))
2037                         continue;
2038
2039                 if (ftrace_match_record(rec, search, search_len, type)) {
2040                         /* ensure it is not already in the array */
2041                         exists = false;
2042                         for (i = 0; i < *idx; i++)
2043                                 if (array[i] == rec->ip) {
2044                                         exists = true;
2045                                         break;
2046                                 }
2047                         if (!exists) {
2048                                 array[(*idx)++] = rec->ip;
2049                                 found = 1;
2050                         }
2051                 }
2052         } while_for_each_ftrace_rec();
2053
2054         mutex_unlock(&ftrace_lock);
2055
2056         return found ? 0 : -EINVAL;
2057 }
2058
2059 static ssize_t
2060 ftrace_graph_write(struct file *file, const char __user *ubuf,
2061                    size_t cnt, loff_t *ppos)
2062 {
2063         unsigned char buffer[FTRACE_BUFF_MAX+1];
2064         unsigned long *array;
2065         size_t read = 0;
2066         ssize_t ret;
2067         int index = 0;
2068         char ch;
2069
2070         if (!cnt || cnt < 0)
2071                 return 0;
2072
2073         mutex_lock(&graph_lock);
2074
2075         if (ftrace_graph_count >= FTRACE_GRAPH_MAX_FUNCS) {
2076                 ret = -EBUSY;
2077                 goto out;
2078         }
2079
2080         if (file->f_mode & FMODE_READ) {
2081                 struct seq_file *m = file->private_data;
2082                 array = m->private;
2083         } else
2084                 array = file->private_data;
2085
2086         ret = get_user(ch, ubuf++);
2087         if (ret)
2088                 goto out;
2089         read++;
2090         cnt--;
2091
2092         /* skip white space */
2093         while (cnt && isspace(ch)) {
2094                 ret = get_user(ch, ubuf++);
2095                 if (ret)
2096                         goto out;
2097                 read++;
2098                 cnt--;
2099         }
2100
2101         if (isspace(ch)) {
2102                 *ppos += read;
2103                 ret = read;
2104                 goto out;
2105         }
2106
2107         while (cnt && !isspace(ch)) {
2108                 if (index < FTRACE_BUFF_MAX)
2109                         buffer[index++] = ch;
2110                 else {
2111                         ret = -EINVAL;
2112                         goto out;
2113                 }
2114                 ret = get_user(ch, ubuf++);
2115                 if (ret)
2116                         goto out;
2117                 read++;
2118                 cnt--;
2119         }
2120         buffer[index] = 0;
2121
2122         /* we allow only one expression at a time */
2123         ret = ftrace_set_func(array, &ftrace_graph_count, buffer);
2124         if (ret)
2125                 goto out;
2126
2127         file->f_pos += read;
2128
2129         ret = read;
2130  out:
2131         mutex_unlock(&graph_lock);
2132
2133         return ret;
2134 }
2135
2136 static const struct file_operations ftrace_graph_fops = {
2137         .open = ftrace_graph_open,
2138         .read = ftrace_graph_read,
2139         .write = ftrace_graph_write,
2140 };
2141 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2142
2143 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
2144 {
2145         struct dentry *entry;
2146
2147         entry = debugfs_create_file("available_filter_functions", 0444,
2148                                     d_tracer, NULL, &ftrace_avail_fops);
2149         if (!entry)
2150                 pr_warning("Could not create debugfs "
2151                            "'available_filter_functions' entry\n");
2152
2153         entry = debugfs_create_file("failures", 0444,
2154                                     d_tracer, NULL, &ftrace_failures_fops);
2155         if (!entry)
2156                 pr_warning("Could not create debugfs 'failures' entry\n");
2157
2158         entry = debugfs_create_file("set_ftrace_filter", 0644, d_tracer,
2159                                     NULL, &ftrace_filter_fops);
2160         if (!entry)
2161                 pr_warning("Could not create debugfs "
2162                            "'set_ftrace_filter' entry\n");
2163
2164         entry = debugfs_create_file("set_ftrace_notrace", 0644, d_tracer,
2165                                     NULL, &ftrace_notrace_fops);
2166         if (!entry)
2167                 pr_warning("Could not create debugfs "
2168                            "'set_ftrace_notrace' entry\n");
2169
2170 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2171         entry = debugfs_create_file("set_graph_function", 0444, d_tracer,
2172                                     NULL,
2173                                     &ftrace_graph_fops);
2174         if (!entry)
2175                 pr_warning("Could not create debugfs "
2176                            "'set_graph_function' entry\n");
2177 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2178
2179         return 0;
2180 }
2181
2182 static int ftrace_convert_nops(struct module *mod,
2183                                unsigned long *start,
2184                                unsigned long *end)
2185 {
2186         unsigned long *p;
2187         unsigned long addr;
2188         unsigned long flags;
2189
2190         mutex_lock(&ftrace_lock);
2191         p = start;
2192         while (p < end) {
2193                 addr = ftrace_call_adjust(*p++);
2194                 /*
2195                  * Some architecture linkers will pad between
2196                  * the different mcount_loc sections of different
2197                  * object files to satisfy alignments.
2198                  * Skip any NULL pointers.
2199                  */
2200                 if (!addr)
2201                         continue;
2202                 ftrace_record_ip(addr);
2203         }
2204
2205         /* disable interrupts to prevent kstop machine */
2206         local_irq_save(flags);
2207         ftrace_update_code(mod);
2208         local_irq_restore(flags);
2209         mutex_unlock(&ftrace_lock);
2210
2211         return 0;
2212 }
2213
2214 void ftrace_init_module(struct module *mod,
2215                         unsigned long *start, unsigned long *end)
2216 {
2217         if (ftrace_disabled || start == end)
2218                 return;
2219         ftrace_convert_nops(mod, start, end);
2220 }
2221
2222 extern unsigned long __start_mcount_loc[];
2223 extern unsigned long __stop_mcount_loc[];
2224
2225 void __init ftrace_init(void)
2226 {
2227         unsigned long count, addr, flags;
2228         int ret;
2229
2230         /* Keep the ftrace pointer to the stub */
2231         addr = (unsigned long)ftrace_stub;
2232
2233         local_irq_save(flags);
2234         ftrace_dyn_arch_init(&addr);
2235         local_irq_restore(flags);
2236
2237         /* ftrace_dyn_arch_init places the return code in addr */
2238         if (addr)
2239                 goto failed;
2240
2241         count = __stop_mcount_loc - __start_mcount_loc;
2242
2243         ret = ftrace_dyn_table_alloc(count);
2244         if (ret)
2245                 goto failed;
2246
2247         last_ftrace_enabled = ftrace_enabled = 1;
2248
2249         ret = ftrace_convert_nops(NULL,
2250                                   __start_mcount_loc,
2251                                   __stop_mcount_loc);
2252
2253         return;
2254  failed:
2255         ftrace_disabled = 1;
2256 }
2257
2258 #else
2259
2260 static int __init ftrace_nodyn_init(void)
2261 {
2262         ftrace_enabled = 1;
2263         return 0;
2264 }
2265 device_initcall(ftrace_nodyn_init);
2266
2267 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
2268 static inline void ftrace_startup_enable(int command) { }
2269 /* Keep as macros so we do not need to define the commands */
2270 # define ftrace_startup(command)        do { } while (0)
2271 # define ftrace_shutdown(command)       do { } while (0)
2272 # define ftrace_startup_sysctl()        do { } while (0)
2273 # define ftrace_shutdown_sysctl()       do { } while (0)
2274 #endif /* CONFIG_DYNAMIC_FTRACE */
2275
2276 static ssize_t
2277 ftrace_pid_read(struct file *file, char __user *ubuf,
2278                        size_t cnt, loff_t *ppos)
2279 {
2280         char buf[64];
2281         int r;
2282
2283         if (ftrace_pid_trace == ftrace_swapper_pid)
2284                 r = sprintf(buf, "swapper tasks\n");
2285         else if (ftrace_pid_trace)
2286                 r = sprintf(buf, "%u\n", pid_nr(ftrace_pid_trace));
2287         else
2288                 r = sprintf(buf, "no pid\n");
2289
2290         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2291 }
2292
2293 static void clear_ftrace_swapper(void)
2294 {
2295         struct task_struct *p;
2296         int cpu;
2297
2298         get_online_cpus();
2299         for_each_online_cpu(cpu) {
2300                 p = idle_task(cpu);
2301                 clear_tsk_trace_trace(p);
2302         }
2303         put_online_cpus();
2304 }
2305
2306 static void set_ftrace_swapper(void)
2307 {
2308         struct task_struct *p;
2309         int cpu;
2310
2311         get_online_cpus();
2312         for_each_online_cpu(cpu) {
2313                 p = idle_task(cpu);
2314                 set_tsk_trace_trace(p);
2315         }
2316         put_online_cpus();
2317 }
2318
2319 static void clear_ftrace_pid(struct pid *pid)
2320 {
2321         struct task_struct *p;
2322
2323         rcu_read_lock();
2324         do_each_pid_task(pid, PIDTYPE_PID, p) {
2325                 clear_tsk_trace_trace(p);
2326         } while_each_pid_task(pid, PIDTYPE_PID, p);
2327         rcu_read_unlock();
2328
2329         put_pid(pid);
2330 }
2331
2332 static void set_ftrace_pid(struct pid *pid)
2333 {
2334         struct task_struct *p;
2335
2336         rcu_read_lock();
2337         do_each_pid_task(pid, PIDTYPE_PID, p) {
2338                 set_tsk_trace_trace(p);
2339         } while_each_pid_task(pid, PIDTYPE_PID, p);
2340         rcu_read_unlock();
2341 }
2342
2343 static void clear_ftrace_pid_task(struct pid **pid)
2344 {
2345         if (*pid == ftrace_swapper_pid)
2346                 clear_ftrace_swapper();
2347         else
2348                 clear_ftrace_pid(*pid);
2349
2350         *pid = NULL;
2351 }
2352
2353 static void set_ftrace_pid_task(struct pid *pid)
2354 {
2355         if (pid == ftrace_swapper_pid)
2356                 set_ftrace_swapper();
2357         else
2358                 set_ftrace_pid(pid);
2359 }
2360
2361 static ssize_t
2362 ftrace_pid_write(struct file *filp, const char __user *ubuf,
2363                    size_t cnt, loff_t *ppos)
2364 {
2365         struct pid *pid;
2366         char buf[64];
2367         long val;
2368         int ret;
2369
2370         if (cnt >= sizeof(buf))
2371                 return -EINVAL;
2372
2373         if (copy_from_user(&buf, ubuf, cnt))
2374                 return -EFAULT;
2375
2376         buf[cnt] = 0;
2377
2378         ret = strict_strtol(buf, 10, &val);
2379         if (ret < 0)
2380                 return ret;
2381
2382         mutex_lock(&ftrace_lock);
2383         if (val < 0) {
2384                 /* disable pid tracing */
2385                 if (!ftrace_pid_trace)
2386                         goto out;
2387
2388                 clear_ftrace_pid_task(&ftrace_pid_trace);
2389
2390         } else {
2391                 /* swapper task is special */
2392                 if (!val) {
2393                         pid = ftrace_swapper_pid;
2394                         if (pid == ftrace_pid_trace)
2395                                 goto out;
2396                 } else {
2397                         pid = find_get_pid(val);
2398
2399                         if (pid == ftrace_pid_trace) {
2400                                 put_pid(pid);
2401                                 goto out;
2402                         }
2403                 }
2404
2405                 if (ftrace_pid_trace)
2406                         clear_ftrace_pid_task(&ftrace_pid_trace);
2407
2408                 if (!pid)
2409                         goto out;
2410
2411                 ftrace_pid_trace = pid;
2412
2413                 set_ftrace_pid_task(ftrace_pid_trace);
2414         }
2415
2416         /* update the function call */
2417         ftrace_update_pid_func();
2418         ftrace_startup_enable(0);
2419
2420  out:
2421         mutex_unlock(&ftrace_lock);
2422
2423         return cnt;
2424 }
2425
2426 static struct file_operations ftrace_pid_fops = {
2427         .read = ftrace_pid_read,
2428         .write = ftrace_pid_write,
2429 };
2430
2431 static __init int ftrace_init_debugfs(void)
2432 {
2433         struct dentry *d_tracer;
2434         struct dentry *entry;
2435
2436         d_tracer = tracing_init_dentry();
2437         if (!d_tracer)
2438                 return 0;
2439
2440         ftrace_init_dyn_debugfs(d_tracer);
2441
2442         entry = debugfs_create_file("set_ftrace_pid", 0644, d_tracer,
2443                                     NULL, &ftrace_pid_fops);
2444         if (!entry)
2445                 pr_warning("Could not create debugfs "
2446                            "'set_ftrace_pid' entry\n");
2447         return 0;
2448 }
2449 fs_initcall(ftrace_init_debugfs);
2450
2451 /**
2452  * ftrace_kill - kill ftrace
2453  *
2454  * This function should be used by panic code. It stops ftrace
2455  * but in a not so nice way. If you need to simply kill ftrace
2456  * from a non-atomic section, use ftrace_kill.
2457  */
2458 void ftrace_kill(void)
2459 {
2460         ftrace_disabled = 1;
2461         ftrace_enabled = 0;
2462         clear_ftrace_function();
2463 }
2464
2465 /**
2466  * register_ftrace_function - register a function for profiling
2467  * @ops - ops structure that holds the function for profiling.
2468  *
2469  * Register a function to be called by all functions in the
2470  * kernel.
2471  *
2472  * Note: @ops->func and all the functions it calls must be labeled
2473  *       with "notrace", otherwise it will go into a
2474  *       recursive loop.
2475  */
2476 int register_ftrace_function(struct ftrace_ops *ops)
2477 {
2478         int ret;
2479
2480         if (unlikely(ftrace_disabled))
2481                 return -1;
2482
2483         mutex_lock(&ftrace_lock);
2484
2485         ret = __register_ftrace_function(ops);
2486         ftrace_startup(0);
2487
2488         mutex_unlock(&ftrace_lock);
2489         return ret;
2490 }
2491
2492 /**
2493  * unregister_ftrace_function - unregister a function for profiling.
2494  * @ops - ops structure that holds the function to unregister
2495  *
2496  * Unregister a function that was added to be called by ftrace profiling.
2497  */
2498 int unregister_ftrace_function(struct ftrace_ops *ops)
2499 {
2500         int ret;
2501
2502         mutex_lock(&ftrace_lock);
2503         ret = __unregister_ftrace_function(ops);
2504         ftrace_shutdown(0);
2505         mutex_unlock(&ftrace_lock);
2506
2507         return ret;
2508 }
2509
2510 int
2511 ftrace_enable_sysctl(struct ctl_table *table, int write,
2512                      struct file *file, void __user *buffer, size_t *lenp,
2513                      loff_t *ppos)
2514 {
2515         int ret;
2516
2517         if (unlikely(ftrace_disabled))
2518                 return -ENODEV;
2519
2520         mutex_lock(&ftrace_lock);
2521
2522         ret  = proc_dointvec(table, write, file, buffer, lenp, ppos);
2523
2524         if (ret || !write || (last_ftrace_enabled == ftrace_enabled))
2525                 goto out;
2526
2527         last_ftrace_enabled = ftrace_enabled;
2528
2529         if (ftrace_enabled) {
2530
2531                 ftrace_startup_sysctl();
2532
2533                 /* we are starting ftrace again */
2534                 if (ftrace_list != &ftrace_list_end) {
2535                         if (ftrace_list->next == &ftrace_list_end)
2536                                 ftrace_trace_function = ftrace_list->func;
2537                         else
2538                                 ftrace_trace_function = ftrace_list_func;
2539                 }
2540
2541         } else {
2542                 /* stopping ftrace calls (just send to ftrace_stub) */
2543                 ftrace_trace_function = ftrace_stub;
2544
2545                 ftrace_shutdown_sysctl();
2546         }
2547
2548  out:
2549         mutex_unlock(&ftrace_lock);
2550         return ret;
2551 }
2552
2553 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2554
2555 static atomic_t ftrace_graph_active;
2556 static struct notifier_block ftrace_suspend_notifier;
2557
2558 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
2559 {
2560         return 0;
2561 }
2562
2563 /* The callbacks that hook a function */
2564 trace_func_graph_ret_t ftrace_graph_return =
2565                         (trace_func_graph_ret_t)ftrace_stub;
2566 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
2567
2568 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
2569 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
2570 {
2571         int i;
2572         int ret = 0;
2573         unsigned long flags;
2574         int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
2575         struct task_struct *g, *t;
2576
2577         for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
2578                 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
2579                                         * sizeof(struct ftrace_ret_stack),
2580                                         GFP_KERNEL);
2581                 if (!ret_stack_list[i]) {
2582                         start = 0;
2583                         end = i;
2584                         ret = -ENOMEM;
2585                         goto free;
2586                 }
2587         }
2588
2589         read_lock_irqsave(&tasklist_lock, flags);
2590         do_each_thread(g, t) {
2591                 if (start == end) {
2592                         ret = -EAGAIN;
2593                         goto unlock;
2594                 }
2595
2596                 if (t->ret_stack == NULL) {
2597                         t->curr_ret_stack = -1;
2598                         /* Make sure IRQs see the -1 first: */
2599                         barrier();
2600                         t->ret_stack = ret_stack_list[start++];
2601                         atomic_set(&t->tracing_graph_pause, 0);
2602                         atomic_set(&t->trace_overrun, 0);
2603                 }
2604         } while_each_thread(g, t);
2605
2606 unlock:
2607         read_unlock_irqrestore(&tasklist_lock, flags);
2608 free:
2609         for (i = start; i < end; i++)
2610                 kfree(ret_stack_list[i]);
2611         return ret;
2612 }
2613
2614 /* Allocate a return stack for each task */
2615 static int start_graph_tracing(void)
2616 {
2617         struct ftrace_ret_stack **ret_stack_list;
2618         int ret, cpu;
2619
2620         ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
2621                                 sizeof(struct ftrace_ret_stack *),
2622                                 GFP_KERNEL);
2623
2624         if (!ret_stack_list)
2625                 return -ENOMEM;
2626
2627         /* The cpu_boot init_task->ret_stack will never be freed */
2628         for_each_online_cpu(cpu)
2629                 ftrace_graph_init_task(idle_task(cpu));
2630
2631         do {
2632                 ret = alloc_retstack_tasklist(ret_stack_list);
2633         } while (ret == -EAGAIN);
2634
2635         kfree(ret_stack_list);
2636         return ret;
2637 }
2638
2639 /*
2640  * Hibernation protection.
2641  * The state of the current task is too much unstable during
2642  * suspend/restore to disk. We want to protect against that.
2643  */
2644 static int
2645 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
2646                                                         void *unused)
2647 {
2648         switch (state) {
2649         case PM_HIBERNATION_PREPARE:
2650                 pause_graph_tracing();
2651                 break;
2652
2653         case PM_POST_HIBERNATION:
2654                 unpause_graph_tracing();
2655                 break;
2656         }
2657         return NOTIFY_DONE;
2658 }
2659
2660 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
2661                         trace_func_graph_ent_t entryfunc)
2662 {
2663         int ret = 0;
2664
2665         mutex_lock(&ftrace_lock);
2666
2667         ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
2668         register_pm_notifier(&ftrace_suspend_notifier);
2669
2670         atomic_inc(&ftrace_graph_active);
2671         ret = start_graph_tracing();
2672         if (ret) {
2673                 atomic_dec(&ftrace_graph_active);
2674                 goto out;
2675         }
2676
2677         ftrace_graph_return = retfunc;
2678         ftrace_graph_entry = entryfunc;
2679
2680         ftrace_startup(FTRACE_START_FUNC_RET);
2681
2682 out:
2683         mutex_unlock(&ftrace_lock);
2684         return ret;
2685 }
2686
2687 void unregister_ftrace_graph(void)
2688 {
2689         mutex_lock(&ftrace_lock);
2690
2691         atomic_dec(&ftrace_graph_active);
2692         ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
2693         ftrace_graph_entry = ftrace_graph_entry_stub;
2694         ftrace_shutdown(FTRACE_STOP_FUNC_RET);
2695         unregister_pm_notifier(&ftrace_suspend_notifier);
2696
2697         mutex_unlock(&ftrace_lock);
2698 }
2699
2700 /* Allocate a return stack for newly created task */
2701 void ftrace_graph_init_task(struct task_struct *t)
2702 {
2703         if (atomic_read(&ftrace_graph_active)) {
2704                 t->ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
2705                                 * sizeof(struct ftrace_ret_stack),
2706                                 GFP_KERNEL);
2707                 if (!t->ret_stack)
2708                         return;
2709                 t->curr_ret_stack = -1;
2710                 atomic_set(&t->tracing_graph_pause, 0);
2711                 atomic_set(&t->trace_overrun, 0);
2712         } else
2713                 t->ret_stack = NULL;
2714 }
2715
2716 void ftrace_graph_exit_task(struct task_struct *t)
2717 {
2718         struct ftrace_ret_stack *ret_stack = t->ret_stack;
2719
2720         t->ret_stack = NULL;
2721         /* NULL must become visible to IRQs before we free it: */
2722         barrier();
2723
2724         kfree(ret_stack);
2725 }
2726
2727 void ftrace_graph_stop(void)
2728 {
2729         ftrace_stop();
2730 }
2731 #endif
2732