kernel/trace/ftrace.c

   1 /*
   2  * Infrastructure for profiling code inserted by 'gcc -pg'.
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Originally ported from the -rt patch by:
   8  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
   9  *
  10  * Based on code in the latency_tracer, that is:
  11  *
  12  *  Copyright (C) 2004-2006 Ingo Molnar
  13  *  Copyright (C) 2004 William Lee Irwin III
  14  */
  15
  16 #include <linux/stop_machine.h>
  17 #include <linux/clocksource.h>
  18 #include <linux/kallsyms.h>
  19 #include <linux/seq_file.h>
  20 #include <linux/debugfs.h>
  21 #include <linux/hardirq.h>
  22 #include <linux/kthread.h>
  23 #include <linux/uaccess.h>
  24 #include <linux/kprobes.h>
  25 #include <linux/ftrace.h>
  26 #include <linux/sysctl.h>
  27 #include <linux/ctype.h>
  28 #include <linux/hash.h>
  29 #include <linux/list.h>
  30
  31 #include <asm/ftrace.h>
  32
  33 #include "trace.h"
  34
  35 #define FTRACE_WARN_ON(cond)                    \
  36         do {                                    \
  37                 if (WARN_ON(cond))              \
  38                         ftrace_kill();          \
  39         } while (0)
  40
  41 #define FTRACE_WARN_ON_ONCE(cond)               \
  42         do {                                    \
  43                 if (WARN_ON_ONCE(cond))         \
  44                         ftrace_kill();          \
  45         } while (0)
  46
  47 /* ftrace_enabled is a method to turn ftrace on or off */
  48 int ftrace_enabled __read_mostly;
  49 static int last_ftrace_enabled;
  50
  51 /*
  52  * ftrace_disabled is set when an anomaly is discovered.
  53  * ftrace_disabled is much stronger than ftrace_enabled.
  54  */
  55 static int ftrace_disabled __read_mostly;
  56
  57 static DEFINE_SPINLOCK(ftrace_lock);
  58 static DEFINE_MUTEX(ftrace_sysctl_lock);
  59
  60 static struct ftrace_ops ftrace_list_end __read_mostly =
  61 {
  62         .func = ftrace_stub,
  63 };
  64
  65 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
  66 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
  67
  68 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
  69 {
  70         struct ftrace_ops *op = ftrace_list;
  71
  72         /* in case someone actually ports this to alpha! */
  73         read_barrier_depends();
  74
  75         while (op != &ftrace_list_end) {
  76                 /* silly alpha */
  77                 read_barrier_depends();
  78                 op->func(ip, parent_ip);
  79                 op = op->next;
  80         };
  81 }
  82
  83 /**
  84  * clear_ftrace_function - reset the ftrace function
  85  *
  86  * This NULLs the ftrace function and in essence stops
  87  * tracing.  There may be lag
  88  */
  89 void clear_ftrace_function(void)
  90 {
  91         ftrace_trace_function = ftrace_stub;
  92 }
  93
  94 static int __register_ftrace_function(struct ftrace_ops *ops)
  95 {
  96         /* should not be called from interrupt context */
  97         spin_lock(&ftrace_lock);
  98
  99         ops->next = ftrace_list;
 100         /*
 101          * We are entering ops into the ftrace_list but another
 102          * CPU might be walking that list. We need to make sure
 103          * the ops->next pointer is valid before another CPU sees
 104          * the ops pointer included into the ftrace_list.
 105          */
 106         smp_wmb();
 107         ftrace_list = ops;
 108
 109         if (ftrace_enabled) {
 110                 /*
 111                  * For one func, simply call it directly.
 112                  * For more than one func, call the chain.
 113                  */
 114                 if (ops->next == &ftrace_list_end)
 115                         ftrace_trace_function = ops->func;
 116                 else
 117                         ftrace_trace_function = ftrace_list_func;
 118         }
 119
 120         spin_unlock(&ftrace_lock);
 121
 122         return 0;
 123 }
 124
 125 static int __unregister_ftrace_function(struct ftrace_ops *ops)
 126 {
 127         struct ftrace_ops **p;
 128         int ret = 0;
 129
 130         /* should not be called from interrupt context */
 131         spin_lock(&ftrace_lock);
 132
 133         /*
 134          * If we are removing the last function, then simply point
 135          * to the ftrace_stub.
 136          */
 137         if (ftrace_list == ops && ops->next == &ftrace_list_end) {
 138                 ftrace_trace_function = ftrace_stub;
 139                 ftrace_list = &ftrace_list_end;
 140                 goto out;
 141         }
 142
 143         for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
 144                 if (*p == ops)
 145                         break;
 146
 147         if (*p != ops) {
 148                 ret = -1;
 149                 goto out;
 150         }
 151
 152         *p = (*p)->next;
 153
 154         if (ftrace_enabled) {
 155                 /* If we only have one func left, then call that directly */
 156                 if (ftrace_list == &ftrace_list_end ||
 157                     ftrace_list->next == &ftrace_list_end)
 158                         ftrace_trace_function = ftrace_list->func;
 159         }
 160
 161  out:
 162         spin_unlock(&ftrace_lock);
 163
 164         return ret;
 165 }
 166
 167 #ifdef CONFIG_DYNAMIC_FTRACE
 168 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
 169 # error Dynamic ftrace depends on MCOUNT_RECORD
 170 #endif
 171
 172 /*
 173  * Since MCOUNT_ADDR may point to mcount itself, we do not want
 174  * to get it confused by reading a reference in the code as we
 175  * are parsing on objcopy output of text. Use a variable for
 176  * it instead.
 177  */
 178 static unsigned long mcount_addr = MCOUNT_ADDR;
 179
 180 enum {
 181         FTRACE_ENABLE_CALLS             = (1 << 0),
 182         FTRACE_DISABLE_CALLS            = (1 << 1),
 183         FTRACE_UPDATE_TRACE_FUNC        = (1 << 2),
 184         FTRACE_ENABLE_MCOUNT            = (1 << 3),
 185         FTRACE_DISABLE_MCOUNT           = (1 << 4),
 186 };
 187
 188 static int ftrace_filtered;
 189 static int tracing_on;
 190 static int frozen_record_count;
 191
 192 static struct hlist_head ftrace_hash[FTRACE_HASHSIZE];
 193
 194 static DEFINE_PER_CPU(int, ftrace_shutdown_disable_cpu);
 195
 196 static DEFINE_MUTEX(ftrace_regex_lock);
 197
 198 struct ftrace_page {
 199         struct ftrace_page      *next;
 200         unsigned long           index;
 201         struct dyn_ftrace       records[];
 202 };
 203
 204 #define ENTRIES_PER_PAGE \
 205   ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
 206
 207 /* estimate from running different kernels */
 208 #define NR_TO_INIT              10000
 209
 210 static struct ftrace_page       *ftrace_pages_start;
 211 static struct ftrace_page       *ftrace_pages;
 212
 213 static int ftrace_record_suspend;
 214
 215 static struct dyn_ftrace *ftrace_free_records;
 216
 217
 218 #ifdef CONFIG_KPROBES
 219 static inline void freeze_record(struct dyn_ftrace *rec)
 220 {
 221         if (!(rec->flags & FTRACE_FL_FROZEN)) {
 222                 rec->flags |= FTRACE_FL_FROZEN;
 223                 frozen_record_count++;
 224         }
 225 }
 226
 227 static inline void unfreeze_record(struct dyn_ftrace *rec)
 228 {
 229         if (rec->flags & FTRACE_FL_FROZEN) {
 230                 rec->flags &= ~FTRACE_FL_FROZEN;
 231                 frozen_record_count--;
 232         }
 233 }
 234
 235 static inline int record_frozen(struct dyn_ftrace *rec)
 236 {
 237         return rec->flags & FTRACE_FL_FROZEN;
 238 }
 239 #else
 240 # define freeze_record(rec)                     ({ 0; })
 241 # define unfreeze_record(rec)                   ({ 0; })
 242 # define record_frozen(rec)                     ({ 0; })
 243 #endif /* CONFIG_KPROBES */
 244
 245 int skip_trace(unsigned long ip)
 246 {
 247         unsigned long fl;
 248         struct dyn_ftrace *rec;
 249         struct hlist_node *t;
 250         struct hlist_head *head;
 251
 252         if (frozen_record_count == 0)
 253                 return 0;
 254
 255         head = &ftrace_hash[hash_long(ip, FTRACE_HASHBITS)];
 256         hlist_for_each_entry_rcu(rec, t, head, node) {
 257                 if (rec->ip == ip) {
 258                         if (record_frozen(rec)) {
 259                                 if (rec->flags & FTRACE_FL_FAILED)
 260                                         return 1;
 261
 262                                 if (!(rec->flags & FTRACE_FL_CONVERTED))
 263                                         return 1;
 264
 265                                 if (!tracing_on || !ftrace_enabled)
 266                                         return 1;
 267
 268                                 if (ftrace_filtered) {
 269                                         fl = rec->flags & (FTRACE_FL_FILTER |
 270                                                            FTRACE_FL_NOTRACE);
 271                                         if (!fl || (fl & FTRACE_FL_NOTRACE))
 272                                                 return 1;
 273                                 }
 274                         }
 275                         break;
 276                 }
 277         }
 278
 279         return 0;
 280 }
 281
 282 static inline int
 283 ftrace_ip_in_hash(unsigned long ip, unsigned long key)
 284 {
 285         struct dyn_ftrace *p;
 286         struct hlist_node *t;
 287         int found = 0;
 288
 289         hlist_for_each_entry_rcu(p, t, &ftrace_hash[key], node) {
 290                 if (p->ip == ip) {
 291                         found = 1;
 292                         break;
 293                 }
 294         }
 295
 296         return found;
 297 }
 298
 299 static inline void
 300 ftrace_add_hash(struct dyn_ftrace *node, unsigned long key)
 301 {
 302         hlist_add_head_rcu(&node->node, &ftrace_hash[key]);
 303 }
 304
 305 /* called from kstop_machine */
 306 static inline void ftrace_del_hash(struct dyn_ftrace *node)
 307 {
 308         hlist_del(&node->node);
 309 }
 310
 311 static void ftrace_free_rec(struct dyn_ftrace *rec)
 312 {
 313         rec->ip = (unsigned long)ftrace_free_records;
 314         ftrace_free_records = rec;
 315         rec->flags |= FTRACE_FL_FREE;
 316 }
 317
 318 void ftrace_release(void *start, unsigned long size)
 319 {
 320         struct dyn_ftrace *rec;
 321         struct ftrace_page *pg;
 322         unsigned long s = (unsigned long)start;
 323         unsigned long e = s + size;
 324         int i;
 325
 326         if (ftrace_disabled || !start)
 327                 return;
 328
 329         /* should not be called from interrupt context */
 330         spin_lock(&ftrace_lock);
 331
 332         for (pg = ftrace_pages_start; pg; pg = pg->next) {
 333                 for (i = 0; i < pg->index; i++) {
 334                         rec = &pg->records[i];
 335
 336                         if ((rec->ip >= s) && (rec->ip < e))
 337                                 ftrace_free_rec(rec);
 338                 }
 339         }
 340         spin_unlock(&ftrace_lock);
 341
 342         ftrace_release_hash(s, e);
 343 }
 344
 345 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
 346 {
 347         struct dyn_ftrace *rec;
 348
 349         /* First check for freed records */
 350         if (ftrace_free_records) {
 351                 rec = ftrace_free_records;
 352
 353                 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
 354                         FTRACE_WARN_ON_ONCE(1);
 355                         ftrace_free_records = NULL;
 356                         return NULL;
 357                 }
 358
 359                 ftrace_free_records = (void *)rec->ip;
 360                 memset(rec, 0, sizeof(*rec));
 361                 return rec;
 362         }
 363
 364         if (ftrace_pages->index == ENTRIES_PER_PAGE) {
 365                 if (!ftrace_pages->next)
 366                         return NULL;
 367                 ftrace_pages = ftrace_pages->next;
 368         }
 369
 370         return &ftrace_pages->records[ftrace_pages->index++];
 371 }
 372
 373 static void
 374 ftrace_record_ip(unsigned long ip)
 375 {
 376         struct dyn_ftrace *node;
 377         unsigned long key;
 378         int resched;
 379         int cpu;
 380
 381         if (!ftrace_enabled || ftrace_disabled)
 382                 return;
 383
 384         resched = need_resched();
 385         preempt_disable_notrace();
 386
 387         /*
 388          * We simply need to protect against recursion.
 389          * Use the the raw version of smp_processor_id and not
 390          * __get_cpu_var which can call debug hooks that can
 391          * cause a recursive crash here.
 392          */
 393         cpu = raw_smp_processor_id();
 394         per_cpu(ftrace_shutdown_disable_cpu, cpu)++;
 395         if (per_cpu(ftrace_shutdown_disable_cpu, cpu) != 1)
 396                 goto out;
 397
 398         if (unlikely(ftrace_record_suspend))
 399                 goto out;
 400
 401         key = hash_long(ip, FTRACE_HASHBITS);
 402
 403         FTRACE_WARN_ON_ONCE(key >= FTRACE_HASHSIZE);
 404
 405         if (ftrace_ip_in_hash(ip, key))
 406                 goto out;
 407
 408         /* This ip may have hit the hash before the lock */
 409         if (ftrace_ip_in_hash(ip, key))
 410                 goto out;
 411
 412         node = ftrace_alloc_dyn_node(ip);
 413         if (!node)
 414                 goto out;
 415
 416         node->ip = ip;
 417
 418         ftrace_add_hash(node, key);
 419
 420  out:
 421         per_cpu(ftrace_shutdown_disable_cpu, cpu)--;
 422
 423         /* prevent recursion with scheduler */
 424         if (resched)
 425                 preempt_enable_no_resched_notrace();
 426         else
 427                 preempt_enable_notrace();
 428 }
 429
 430 #define FTRACE_ADDR ((long)(ftrace_caller))
 431
 432 static int
 433 __ftrace_replace_code(struct dyn_ftrace *rec,
 434                       unsigned char *old, unsigned char *new, int enable)
 435 {
 436         unsigned long ip, fl;
 437
 438         ip = rec->ip;
 439
 440         if (ftrace_filtered && enable) {
 441                 /*
 442                  * If filtering is on:
 443                  *
 444                  * If this record is set to be filtered and
 445                  * is enabled then do nothing.
 446                  *
 447                  * If this record is set to be filtered and
 448                  * it is not enabled, enable it.
 449                  *
 450                  * If this record is not set to be filtered
 451                  * and it is not enabled do nothing.
 452                  *
 453                  * If this record is set not to trace then
 454                  * do nothing.
 455                  *
 456                  * If this record is set not to trace and
 457                  * it is enabled then disable it.
 458                  *
 459                  * If this record is not set to be filtered and
 460                  * it is enabled, disable it.
 461                  */
 462
 463                 fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_NOTRACE |
 464                                    FTRACE_FL_ENABLED);
 465
 466                 if ((fl ==  (FTRACE_FL_FILTER | FTRACE_FL_ENABLED)) ||
 467                     (fl ==  (FTRACE_FL_FILTER | FTRACE_FL_NOTRACE)) ||
 468                     !fl || (fl == FTRACE_FL_NOTRACE))
 469                         return 0;
 470
 471                 /*
 472                  * If it is enabled disable it,
 473                  * otherwise enable it!
 474                  */
 475                 if (fl & FTRACE_FL_ENABLED) {
 476                         /* swap new and old */
 477                         new = old;
 478                         old = ftrace_call_replace(ip, FTRACE_ADDR);
 479                         rec->flags &= ~FTRACE_FL_ENABLED;
 480                 } else {
 481                         new = ftrace_call_replace(ip, FTRACE_ADDR);
 482                         rec->flags |= FTRACE_FL_ENABLED;
 483                 }
 484         } else {
 485
 486                 if (enable) {
 487                         /*
 488                          * If this record is set not to trace and is
 489                          * not enabled, do nothing.
 490                          */
 491                         fl = rec->flags & (FTRACE_FL_NOTRACE | FTRACE_FL_ENABLED);
 492                         if (fl == FTRACE_FL_NOTRACE)
 493                                 return 0;
 494
 495                         new = ftrace_call_replace(ip, FTRACE_ADDR);
 496                 } else
 497                         old = ftrace_call_replace(ip, FTRACE_ADDR);
 498
 499                 if (enable) {
 500                         if (rec->flags & FTRACE_FL_ENABLED)
 501                                 return 0;
 502                         rec->flags |= FTRACE_FL_ENABLED;
 503                 } else {
 504                         if (!(rec->flags & FTRACE_FL_ENABLED))
 505                                 return 0;
 506                         rec->flags &= ~FTRACE_FL_ENABLED;
 507                 }
 508         }
 509
 510         return ftrace_modify_code(ip, old, new);
 511 }
 512
 513 static void ftrace_replace_code(int enable)
 514 {
 515         int i, failed;
 516         unsigned char *new = NULL, *old = NULL;
 517         struct dyn_ftrace *rec;
 518         struct ftrace_page *pg;
 519
 520         if (enable)
 521                 old = ftrace_nop_replace();
 522         else
 523                 new = ftrace_nop_replace();
 524
 525         for (pg = ftrace_pages_start; pg; pg = pg->next) {
 526                 for (i = 0; i < pg->index; i++) {
 527                         rec = &pg->records[i];
 528
 529                         /* don't modify code that has already faulted */
 530                         if (rec->flags & FTRACE_FL_FAILED)
 531                                 continue;
 532
 533                         /* ignore updates to this record's mcount site */
 534                         if (get_kprobe((void *)rec->ip)) {
 535                                 freeze_record(rec);
 536                                 continue;
 537                         } else {
 538                                 unfreeze_record(rec);
 539                         }
 540
 541                         failed = __ftrace_replace_code(rec, old, new, enable);
 542                         if (failed && (rec->flags & FTRACE_FL_CONVERTED)) {
 543                                 rec->flags |= FTRACE_FL_FAILED;
 544                                 if ((system_state == SYSTEM_BOOTING) ||
 545                                     !core_kernel_text(rec->ip)) {
 546                                         ftrace_del_hash(rec);
 547                                         ftrace_free_rec(rec);
 548                                 }
 549                         }
 550                 }
 551         }
 552 }
 553
 554 static void ftrace_shutdown_replenish(void)
 555 {
 556         if (ftrace_pages->next)
 557                 return;
 558
 559         /* allocate another page */
 560         ftrace_pages->next = (void *)get_zeroed_page(GFP_KERNEL);
 561 }
 562
 563 static void print_ip_ins(const char *fmt, unsigned char *p)
 564 {
 565         int i;
 566
 567         printk(KERN_CONT "%s", fmt);
 568
 569         for (i = 0; i < MCOUNT_INSN_SIZE; i++)
 570                 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
 571 }
 572
 573 static int
 574 ftrace_code_disable(struct dyn_ftrace *rec)
 575 {
 576         unsigned long ip;
 577         unsigned char *nop, *call;
 578         int ret;
 579
 580         ip = rec->ip;
 581
 582         nop = ftrace_nop_replace();
 583         call = ftrace_call_replace(ip, mcount_addr);
 584
 585         ret = ftrace_modify_code(ip, call, nop);
 586         if (ret) {
 587                 switch (ret) {
 588                 case -EFAULT:
 589                         FTRACE_WARN_ON_ONCE(1);
 590                         pr_info("ftrace faulted on modifying ");
 591                         print_ip_sym(ip);
 592                         break;
 593                 case -EINVAL:
 594                         FTRACE_WARN_ON_ONCE(1);
 595                         pr_info("ftrace failed to modify ");
 596                         print_ip_sym(ip);
 597                         print_ip_ins(" expected: ", call);
 598                         print_ip_ins(" actual: ", (unsigned char *)ip);
 599                         print_ip_ins(" replace: ", nop);
 600                         printk(KERN_CONT "\n");
 601                         break;
 602                 case -EPERM:
 603                         FTRACE_WARN_ON_ONCE(1);
 604                         pr_info("ftrace faulted on writing ");
 605                         print_ip_sym(ip);
 606                         break;
 607                 default:
 608                         FTRACE_WARN_ON_ONCE(1);
 609                         pr_info("ftrace faulted on unknown error ");
 610                         print_ip_sym(ip);
 611                 }
 612
 613                 rec->flags |= FTRACE_FL_FAILED;
 614                 return 0;
 615         }
 616         return 1;
 617 }
 618
 619 static int ftrace_update_code(void *ignore);
 620
 621 static int __ftrace_modify_code(void *data)
 622 {
 623         int *command = data;
 624
 625         if (*command & FTRACE_ENABLE_CALLS) {
 626                 /*
 627                  * Update any recorded ips now that we have the
 628                  * machine stopped
 629                  */
 630                 ftrace_update_code(NULL);
 631                 ftrace_replace_code(1);
 632                 tracing_on = 1;
 633         } else if (*command & FTRACE_DISABLE_CALLS) {
 634                 ftrace_replace_code(0);
 635                 tracing_on = 0;
 636         }
 637
 638         if (*command & FTRACE_UPDATE_TRACE_FUNC)
 639                 ftrace_update_ftrace_func(ftrace_trace_function);
 640
 641         return 0;
 642 }
 643
 644 static void ftrace_run_update_code(int command)
 645 {
 646         stop_machine(__ftrace_modify_code, &command, NULL);
 647 }
 648
 649 static ftrace_func_t saved_ftrace_func;
 650 static int ftrace_start;
 651 static DEFINE_MUTEX(ftrace_start_lock);
 652
 653 static void ftrace_startup(void)
 654 {
 655         int command = 0;
 656
 657         if (unlikely(ftrace_disabled))
 658                 return;
 659
 660         mutex_lock(&ftrace_start_lock);
 661         ftrace_start++;
 662         if (ftrace_start == 1)
 663                 command |= FTRACE_ENABLE_CALLS;
 664
 665         if (saved_ftrace_func != ftrace_trace_function) {
 666                 saved_ftrace_func = ftrace_trace_function;
 667                 command |= FTRACE_UPDATE_TRACE_FUNC;
 668         }
 669
 670         if (!command || !ftrace_enabled)
 671                 goto out;
 672
 673         ftrace_run_update_code(command);
 674  out:
 675         mutex_unlock(&ftrace_start_lock);
 676 }
 677
 678 static void ftrace_shutdown(void)
 679 {
 680         int command = 0;
 681
 682         if (unlikely(ftrace_disabled))
 683                 return;
 684
 685         mutex_lock(&ftrace_start_lock);
 686         ftrace_start--;
 687         if (!ftrace_start)
 688                 command |= FTRACE_DISABLE_CALLS;
 689
 690         if (saved_ftrace_func != ftrace_trace_function) {
 691                 saved_ftrace_func = ftrace_trace_function;
 692                 command |= FTRACE_UPDATE_TRACE_FUNC;
 693         }
 694
 695         if (!command || !ftrace_enabled)
 696                 goto out;
 697
 698         ftrace_run_update_code(command);
 699  out:
 700         mutex_unlock(&ftrace_start_lock);
 701 }
 702
 703 static void ftrace_startup_sysctl(void)
 704 {
 705         int command = FTRACE_ENABLE_MCOUNT;
 706
 707         if (unlikely(ftrace_disabled))
 708                 return;
 709
 710         mutex_lock(&ftrace_start_lock);
 711         /* Force update next time */
 712         saved_ftrace_func = NULL;
 713         /* ftrace_start is true if we want ftrace running */
 714         if (ftrace_start)
 715                 command |= FTRACE_ENABLE_CALLS;
 716
 717         ftrace_run_update_code(command);
 718         mutex_unlock(&ftrace_start_lock);
 719 }
 720
 721 static void ftrace_shutdown_sysctl(void)
 722 {
 723         int command = FTRACE_DISABLE_MCOUNT;
 724
 725         if (unlikely(ftrace_disabled))
 726                 return;
 727
 728         mutex_lock(&ftrace_start_lock);
 729         /* ftrace_start is true if ftrace is running */
 730         if (ftrace_start)
 731                 command |= FTRACE_DISABLE_CALLS;
 732
 733         ftrace_run_update_code(command);
 734         mutex_unlock(&ftrace_start_lock);
 735 }
 736
 737 static cycle_t          ftrace_update_time;
 738 static unsigned long    ftrace_update_cnt;
 739 unsigned long           ftrace_update_tot_cnt;
 740
 741 static int ftrace_update_code(void *ignore)
 742 {
 743         int i, save_ftrace_enabled;
 744         cycle_t start, stop;
 745         struct dyn_ftrace *p;
 746         struct hlist_node *t, *n;
 747         struct hlist_head *head, temp_list;
 748
 749         /* Don't be recording funcs now */
 750         ftrace_record_suspend++;
 751         save_ftrace_enabled = ftrace_enabled;
 752         ftrace_enabled = 0;
 753
 754         start = ftrace_now(raw_smp_processor_id());
 755         ftrace_update_cnt = 0;
 756
 757         /* No locks needed, the machine is stopped! */
 758         for (i = 0; i < FTRACE_HASHSIZE; i++) {
 759                 INIT_HLIST_HEAD(&temp_list);
 760                 head = &ftrace_hash[i];
 761
 762                 /* all CPUS are stopped, we are safe to modify code */
 763                 hlist_for_each_entry_safe(p, t, n, head, node) {
 764                         /* Skip over failed records which have not been
 765                          * freed. */
 766                         if (p->flags & FTRACE_FL_FAILED)
 767                                 continue;
 768
 769                         /* Unconverted records are always at the head of the
 770                          * hash bucket. Once we encounter a converted record,
 771                          * simply skip over to the next bucket. Saves ftraced
 772                          * some processor cycles (ftrace does its bid for
 773                          * global warming :-p ). */
 774                         if (p->flags & (FTRACE_FL_CONVERTED))
 775                                 break;
 776
 777                         /* Ignore updates to this record's mcount site.
 778                          * Reintroduce this record at the head of this
 779                          * bucket to attempt to "convert" it again if
 780                          * the kprobe on it is unregistered before the
 781                          * next run. */
 782                         if (get_kprobe((void *)p->ip)) {
 783                                 ftrace_del_hash(p);
 784                                 INIT_HLIST_NODE(&p->node);
 785                                 hlist_add_head(&p->node, &temp_list);
 786                                 freeze_record(p);
 787                                 continue;
 788                         } else {
 789                                 unfreeze_record(p);
 790                         }
 791
 792                         /* convert record (i.e, patch mcount-call with NOP) */
 793                         if (ftrace_code_disable(p)) {
 794                                 p->flags |= FTRACE_FL_CONVERTED;
 795                                 ftrace_update_cnt++;
 796                         } else {
 797                                 if ((system_state == SYSTEM_BOOTING) ||
 798                                     !core_kernel_text(p->ip)) {
 799                                         ftrace_del_hash(p);
 800                                         ftrace_free_rec(p);
 801                                 }
 802                         }
 803                 }
 804
 805                 hlist_for_each_entry_safe(p, t, n, &temp_list, node) {
 806                         hlist_del(&p->node);
 807                         INIT_HLIST_NODE(&p->node);
 808                         hlist_add_head(&p->node, head);
 809                 }
 810         }
 811
 812         stop = ftrace_now(raw_smp_processor_id());
 813         ftrace_update_time = stop - start;
 814         ftrace_update_tot_cnt += ftrace_update_cnt;
 815
 816         ftrace_enabled = save_ftrace_enabled;
 817         ftrace_record_suspend--;
 818
 819         return 0;
 820 }
 821
 822 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
 823 {
 824         struct ftrace_page *pg;
 825         int cnt;
 826         int i;
 827
 828         /* allocate a few pages */
 829         ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
 830         if (!ftrace_pages_start)
 831                 return -1;
 832
 833         /*
 834          * Allocate a few more pages.
 835          *
 836          * TODO: have some parser search vmlinux before
 837          *   final linking to find all calls to ftrace.
 838          *   Then we can:
 839          *    a) know how many pages to allocate.
 840          *     and/or
 841          *    b) set up the table then.
 842          *
 843          *  The dynamic code is still necessary for
 844          *  modules.
 845          */
 846
 847         pg = ftrace_pages = ftrace_pages_start;
 848
 849         cnt = num_to_init / ENTRIES_PER_PAGE;
 850         pr_info("ftrace: allocating %ld hash entries in %d pages\n",
 851                 num_to_init, cnt);
 852
 853         for (i = 0; i < cnt; i++) {
 854                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
 855
 856                 /* If we fail, we'll try later anyway */
 857                 if (!pg->next)
 858                         break;
 859
 860                 pg = pg->next;
 861         }
 862
 863         return 0;
 864 }
 865
 866 enum {
 867         FTRACE_ITER_FILTER      = (1 << 0),
 868         FTRACE_ITER_CONT        = (1 << 1),
 869         FTRACE_ITER_NOTRACE     = (1 << 2),
 870         FTRACE_ITER_FAILURES    = (1 << 3),
 871 };
 872
 873 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
 874
 875 struct ftrace_iterator {
 876         loff_t                  pos;
 877         struct ftrace_page      *pg;
 878         unsigned                idx;
 879         unsigned                flags;
 880         unsigned char           buffer[FTRACE_BUFF_MAX+1];
 881         unsigned                buffer_idx;
 882         unsigned                filtered;
 883 };
 884
 885 static void *
 886 t_next(struct seq_file *m, void *v, loff_t *pos)
 887 {
 888         struct ftrace_iterator *iter = m->private;
 889         struct dyn_ftrace *rec = NULL;
 890
 891         (*pos)++;
 892
 893         /* should not be called from interrupt context */
 894         spin_lock(&ftrace_lock);
 895  retry:
 896         if (iter->idx >= iter->pg->index) {
 897                 if (iter->pg->next) {
 898                         iter->pg = iter->pg->next;
 899                         iter->idx = 0;
 900                         goto retry;
 901                 }
 902         } else {
 903                 rec = &iter->pg->records[iter->idx++];
 904                 if ((rec->flags & FTRACE_FL_FREE) ||
 905
 906                     (!(iter->flags & FTRACE_ITER_FAILURES) &&
 907                      (rec->flags & FTRACE_FL_FAILED)) ||
 908
 909                     ((iter->flags & FTRACE_ITER_FAILURES) &&
 910                      !(rec->flags & FTRACE_FL_FAILED)) ||
 911
 912                     ((iter->flags & FTRACE_ITER_NOTRACE) &&
 913                      !(rec->flags & FTRACE_FL_NOTRACE))) {
 914                         rec = NULL;
 915                         goto retry;
 916                 }
 917         }
 918         spin_unlock(&ftrace_lock);
 919
 920         iter->pos = *pos;
 921
 922         return rec;
 923 }
 924
 925 static void *t_start(struct seq_file *m, loff_t *pos)
 926 {
 927         struct ftrace_iterator *iter = m->private;
 928         void *p = NULL;
 929         loff_t l = -1;
 930
 931         if (*pos != iter->pos) {
 932                 for (p = t_next(m, p, &l); p && l < *pos; p = t_next(m, p, &l))
 933                         ;
 934         } else {
 935                 l = *pos;
 936                 p = t_next(m, p, &l);
 937         }
 938
 939         return p;
 940 }
 941
 942 static void t_stop(struct seq_file *m, void *p)
 943 {
 944 }
 945
 946 static int t_show(struct seq_file *m, void *v)
 947 {
 948         struct dyn_ftrace *rec = v;
 949         char str[KSYM_SYMBOL_LEN];
 950
 951         if (!rec)
 952                 return 0;
 953
 954         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
 955
 956         seq_printf(m, "%s\n", str);
 957
 958         return 0;
 959 }
 960
 961 static struct seq_operations show_ftrace_seq_ops = {
 962         .start = t_start,
 963         .next = t_next,
 964         .stop = t_stop,
 965         .show = t_show,
 966 };
 967
 968 static int
 969 ftrace_avail_open(struct inode *inode, struct file *file)
 970 {
 971         struct ftrace_iterator *iter;
 972         int ret;
 973
 974         if (unlikely(ftrace_disabled))
 975                 return -ENODEV;
 976
 977         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
 978         if (!iter)
 979                 return -ENOMEM;
 980
 981         iter->pg = ftrace_pages_start;
 982         iter->pos = -1;
 983
 984         ret = seq_open(file, &show_ftrace_seq_ops);
 985         if (!ret) {
 986                 struct seq_file *m = file->private_data;
 987
 988                 m->private = iter;
 989         } else {
 990                 kfree(iter);
 991         }
 992
 993         return ret;
 994 }
 995
 996 int ftrace_avail_release(struct inode *inode, struct file *file)
 997 {
 998         struct seq_file *m = (struct seq_file *)file->private_data;
 999         struct ftrace_iterator *iter = m->private;
1000
1001         seq_release(inode, file);
1002         kfree(iter);
1003
1004         return 0;
1005 }
1006
1007 static int
1008 ftrace_failures_open(struct inode *inode, struct file *file)
1009 {
1010         int ret;
1011         struct seq_file *m;
1012         struct ftrace_iterator *iter;
1013
1014         ret = ftrace_avail_open(inode, file);
1015         if (!ret) {
1016                 m = (struct seq_file *)file->private_data;
1017                 iter = (struct ftrace_iterator *)m->private;
1018                 iter->flags = FTRACE_ITER_FAILURES;
1019         }
1020
1021         return ret;
1022 }
1023
1024
1025 static void ftrace_filter_reset(int enable)
1026 {
1027         struct ftrace_page *pg;
1028         struct dyn_ftrace *rec;
1029         unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1030         unsigned i;
1031
1032         /* should not be called from interrupt context */
1033         spin_lock(&ftrace_lock);
1034         if (enable)
1035                 ftrace_filtered = 0;
1036         pg = ftrace_pages_start;
1037         while (pg) {
1038                 for (i = 0; i < pg->index; i++) {
1039                         rec = &pg->records[i];
1040                         if (rec->flags & FTRACE_FL_FAILED)
1041                                 continue;
1042                         rec->flags &= ~type;
1043                 }
1044                 pg = pg->next;
1045         }
1046         spin_unlock(&ftrace_lock);
1047 }
1048
1049 static int
1050 ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1051 {
1052         struct ftrace_iterator *iter;
1053         int ret = 0;
1054
1055         if (unlikely(ftrace_disabled))
1056                 return -ENODEV;
1057
1058         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1059         if (!iter)
1060                 return -ENOMEM;
1061
1062         mutex_lock(&ftrace_regex_lock);
1063         if ((file->f_mode & FMODE_WRITE) &&
1064             !(file->f_flags & O_APPEND))
1065                 ftrace_filter_reset(enable);
1066
1067         if (file->f_mode & FMODE_READ) {
1068                 iter->pg = ftrace_pages_start;
1069                 iter->pos = -1;
1070                 iter->flags = enable ? FTRACE_ITER_FILTER :
1071                         FTRACE_ITER_NOTRACE;
1072
1073                 ret = seq_open(file, &show_ftrace_seq_ops);
1074                 if (!ret) {
1075                         struct seq_file *m = file->private_data;
1076                         m->private = iter;
1077                 } else
1078                         kfree(iter);
1079         } else
1080                 file->private_data = iter;
1081         mutex_unlock(&ftrace_regex_lock);
1082
1083         return ret;
1084 }
1085
1086 static int
1087 ftrace_filter_open(struct inode *inode, struct file *file)
1088 {
1089         return ftrace_regex_open(inode, file, 1);
1090 }
1091
1092 static int
1093 ftrace_notrace_open(struct inode *inode, struct file *file)
1094 {
1095         return ftrace_regex_open(inode, file, 0);
1096 }
1097
1098 static ssize_t
1099 ftrace_regex_read(struct file *file, char __user *ubuf,
1100                        size_t cnt, loff_t *ppos)
1101 {
1102         if (file->f_mode & FMODE_READ)
1103                 return seq_read(file, ubuf, cnt, ppos);
1104         else
1105                 return -EPERM;
1106 }
1107
1108 static loff_t
1109 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1110 {
1111         loff_t ret;
1112
1113         if (file->f_mode & FMODE_READ)
1114                 ret = seq_lseek(file, offset, origin);
1115         else
1116                 file->f_pos = ret = 1;
1117
1118         return ret;
1119 }
1120
1121 enum {
1122         MATCH_FULL,
1123         MATCH_FRONT_ONLY,
1124         MATCH_MIDDLE_ONLY,
1125         MATCH_END_ONLY,
1126 };
1127
1128 static void
1129 ftrace_match(unsigned char *buff, int len, int enable)
1130 {
1131         char str[KSYM_SYMBOL_LEN];
1132         char *search = NULL;
1133         struct ftrace_page *pg;
1134         struct dyn_ftrace *rec;
1135         int type = MATCH_FULL;
1136         unsigned long flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1137         unsigned i, match = 0, search_len = 0;
1138
1139         for (i = 0; i < len; i++) {
1140                 if (buff[i] == '*') {
1141                         if (!i) {
1142                                 search = buff + i + 1;
1143                                 type = MATCH_END_ONLY;
1144                                 search_len = len - (i + 1);
1145                         } else {
1146                                 if (type == MATCH_END_ONLY) {
1147                                         type = MATCH_MIDDLE_ONLY;
1148                                 } else {
1149                                         match = i;
1150                                         type = MATCH_FRONT_ONLY;
1151                                 }
1152                                 buff[i] = 0;
1153                                 break;
1154                         }
1155                 }
1156         }
1157
1158         /* should not be called from interrupt context */
1159         spin_lock(&ftrace_lock);
1160         if (enable)
1161                 ftrace_filtered = 1;
1162         pg = ftrace_pages_start;
1163         while (pg) {
1164                 for (i = 0; i < pg->index; i++) {
1165                         int matched = 0;
1166                         char *ptr;
1167
1168                         rec = &pg->records[i];
1169                         if (rec->flags & FTRACE_FL_FAILED)
1170                                 continue;
1171                         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1172                         switch (type) {
1173                         case MATCH_FULL:
1174                                 if (strcmp(str, buff) == 0)
1175                                         matched = 1;
1176                                 break;
1177                         case MATCH_FRONT_ONLY:
1178                                 if (memcmp(str, buff, match) == 0)
1179                                         matched = 1;
1180                                 break;
1181                         case MATCH_MIDDLE_ONLY:
1182                                 if (strstr(str, search))
1183                                         matched = 1;
1184                                 break;
1185                         case MATCH_END_ONLY:
1186                                 ptr = strstr(str, search);
1187                                 if (ptr && (ptr[search_len] == 0))
1188                                         matched = 1;
1189                                 break;
1190                         }
1191                         if (matched)
1192                                 rec->flags |= flag;
1193                 }
1194                 pg = pg->next;
1195         }
1196         spin_unlock(&ftrace_lock);
1197 }
1198
1199 static ssize_t
1200 ftrace_regex_write(struct file *file, const char __user *ubuf,
1201                    size_t cnt, loff_t *ppos, int enable)
1202 {
1203         struct ftrace_iterator *iter;
1204         char ch;
1205         size_t read = 0;
1206         ssize_t ret;
1207
1208         if (!cnt || cnt < 0)
1209                 return 0;
1210
1211         mutex_lock(&ftrace_regex_lock);
1212
1213         if (file->f_mode & FMODE_READ) {
1214                 struct seq_file *m = file->private_data;
1215                 iter = m->private;
1216         } else
1217                 iter = file->private_data;
1218
1219         if (!*ppos) {
1220                 iter->flags &= ~FTRACE_ITER_CONT;
1221                 iter->buffer_idx = 0;
1222         }
1223
1224         ret = get_user(ch, ubuf++);
1225         if (ret)
1226                 goto out;
1227         read++;
1228         cnt--;
1229
1230         if (!(iter->flags & ~FTRACE_ITER_CONT)) {
1231                 /* skip white space */
1232                 while (cnt && isspace(ch)) {
1233                         ret = get_user(ch, ubuf++);
1234                         if (ret)
1235                                 goto out;
1236                         read++;
1237                         cnt--;
1238                 }
1239
1240                 if (isspace(ch)) {
1241                         file->f_pos += read;
1242                         ret = read;
1243                         goto out;
1244                 }
1245
1246                 iter->buffer_idx = 0;
1247         }
1248
1249         while (cnt && !isspace(ch)) {
1250                 if (iter->buffer_idx < FTRACE_BUFF_MAX)
1251                         iter->buffer[iter->buffer_idx++] = ch;
1252                 else {
1253                         ret = -EINVAL;
1254                         goto out;
1255                 }
1256                 ret = get_user(ch, ubuf++);
1257                 if (ret)
1258                         goto out;
1259                 read++;
1260                 cnt--;
1261         }
1262
1263         if (isspace(ch)) {
1264                 iter->filtered++;
1265                 iter->buffer[iter->buffer_idx] = 0;
1266                 ftrace_match(iter->buffer, iter->buffer_idx, enable);
1267                 iter->buffer_idx = 0;
1268         } else
1269                 iter->flags |= FTRACE_ITER_CONT;
1270
1271
1272         file->f_pos += read;
1273
1274         ret = read;
1275  out:
1276         mutex_unlock(&ftrace_regex_lock);
1277
1278         return ret;
1279 }
1280
1281 static ssize_t
1282 ftrace_filter_write(struct file *file, const char __user *ubuf,
1283                     size_t cnt, loff_t *ppos)
1284 {
1285         return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
1286 }
1287
1288 static ssize_t
1289 ftrace_notrace_write(struct file *file, const char __user *ubuf,
1290                      size_t cnt, loff_t *ppos)
1291 {
1292         return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
1293 }
1294
1295 static void
1296 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
1297 {
1298         if (unlikely(ftrace_disabled))
1299                 return;
1300
1301         mutex_lock(&ftrace_regex_lock);
1302         if (reset)
1303                 ftrace_filter_reset(enable);
1304         if (buf)
1305                 ftrace_match(buf, len, enable);
1306         mutex_unlock(&ftrace_regex_lock);
1307 }
1308
1309 /**
1310  * ftrace_set_filter - set a function to filter on in ftrace
1311  * @buf - the string that holds the function filter text.
1312  * @len - the length of the string.
1313  * @reset - non zero to reset all filters before applying this filter.
1314  *
1315  * Filters denote which functions should be enabled when tracing is enabled.
1316  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
1317  */
1318 void ftrace_set_filter(unsigned char *buf, int len, int reset)
1319 {
1320         ftrace_set_regex(buf, len, reset, 1);
1321 }
1322
1323 /**
1324  * ftrace_set_notrace - set a function to not trace in ftrace
1325  * @buf - the string that holds the function notrace text.
1326  * @len - the length of the string.
1327  * @reset - non zero to reset all filters before applying this filter.
1328  *
1329  * Notrace Filters denote which functions should not be enabled when tracing
1330  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
1331  * for tracing.
1332  */
1333 void ftrace_set_notrace(unsigned char *buf, int len, int reset)
1334 {
1335         ftrace_set_regex(buf, len, reset, 0);
1336 }
1337
1338 static int
1339 ftrace_regex_release(struct inode *inode, struct file *file, int enable)
1340 {
1341         struct seq_file *m = (struct seq_file *)file->private_data;
1342         struct ftrace_iterator *iter;
1343
1344         mutex_lock(&ftrace_regex_lock);
1345         if (file->f_mode & FMODE_READ) {
1346                 iter = m->private;
1347
1348                 seq_release(inode, file);
1349         } else
1350                 iter = file->private_data;
1351
1352         if (iter->buffer_idx) {
1353                 iter->filtered++;
1354                 iter->buffer[iter->buffer_idx] = 0;
1355                 ftrace_match(iter->buffer, iter->buffer_idx, enable);
1356         }
1357
1358         mutex_lock(&ftrace_sysctl_lock);
1359         mutex_lock(&ftrace_start_lock);
1360         if (iter->filtered && ftrace_start && ftrace_enabled)
1361                 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
1362         mutex_unlock(&ftrace_start_lock);
1363         mutex_unlock(&ftrace_sysctl_lock);
1364
1365         kfree(iter);
1366         mutex_unlock(&ftrace_regex_lock);
1367         return 0;
1368 }
1369
1370 static int
1371 ftrace_filter_release(struct inode *inode, struct file *file)
1372 {
1373         return ftrace_regex_release(inode, file, 1);
1374 }
1375
1376 static int
1377 ftrace_notrace_release(struct inode *inode, struct file *file)
1378 {
1379         return ftrace_regex_release(inode, file, 0);
1380 }
1381
1382 static struct file_operations ftrace_avail_fops = {
1383         .open = ftrace_avail_open,
1384         .read = seq_read,
1385         .llseek = seq_lseek,
1386         .release = ftrace_avail_release,
1387 };
1388
1389 static struct file_operations ftrace_failures_fops = {
1390         .open = ftrace_failures_open,
1391         .read = seq_read,
1392         .llseek = seq_lseek,
1393         .release = ftrace_avail_release,
1394 };
1395
1396 static struct file_operations ftrace_filter_fops = {
1397         .open = ftrace_filter_open,
1398         .read = ftrace_regex_read,
1399         .write = ftrace_filter_write,
1400         .llseek = ftrace_regex_lseek,
1401         .release = ftrace_filter_release,
1402 };
1403
1404 static struct file_operations ftrace_notrace_fops = {
1405         .open = ftrace_notrace_open,
1406         .read = ftrace_regex_read,
1407         .write = ftrace_notrace_write,
1408         .llseek = ftrace_regex_lseek,
1409         .release = ftrace_notrace_release,
1410 };
1411
1412 static __init int ftrace_init_debugfs(void)
1413 {
1414         struct dentry *d_tracer;
1415         struct dentry *entry;
1416
1417         d_tracer = tracing_init_dentry();
1418
1419         entry = debugfs_create_file("available_filter_functions", 0444,
1420                                     d_tracer, NULL, &ftrace_avail_fops);
1421         if (!entry)
1422                 pr_warning("Could not create debugfs "
1423                            "'available_filter_functions' entry\n");
1424
1425         entry = debugfs_create_file("failures", 0444,
1426                                     d_tracer, NULL, &ftrace_failures_fops);
1427         if (!entry)
1428                 pr_warning("Could not create debugfs 'failures' entry\n");
1429
1430         entry = debugfs_create_file("set_ftrace_filter", 0644, d_tracer,
1431                                     NULL, &ftrace_filter_fops);
1432         if (!entry)
1433                 pr_warning("Could not create debugfs "
1434                            "'set_ftrace_filter' entry\n");
1435
1436         entry = debugfs_create_file("set_ftrace_notrace", 0644, d_tracer,
1437                                     NULL, &ftrace_notrace_fops);
1438         if (!entry)
1439                 pr_warning("Could not create debugfs "
1440                            "'set_ftrace_notrace' entry\n");
1441
1442         return 0;
1443 }
1444
1445 fs_initcall(ftrace_init_debugfs);
1446
1447 static int ftrace_convert_nops(unsigned long *start,
1448                                unsigned long *end)
1449 {
1450         unsigned long *p;
1451         unsigned long addr;
1452         unsigned long flags;
1453
1454         p = start;
1455         while (p < end) {
1456                 addr = ftrace_call_adjust(*p++);
1457                 /* should not be called from interrupt context */
1458                 spin_lock(&ftrace_lock);
1459                 ftrace_record_ip(addr);
1460                 spin_unlock(&ftrace_lock);
1461                 ftrace_shutdown_replenish();
1462         }
1463
1464         /* p is ignored */
1465         local_irq_save(flags);
1466         ftrace_update_code(p);
1467         local_irq_restore(flags);
1468
1469         return 0;
1470 }
1471
1472 void ftrace_init_module(unsigned long *start, unsigned long *end)
1473 {
1474         if (ftrace_disabled || start == end)
1475                 return;
1476         ftrace_convert_nops(start, end);
1477 }
1478
1479 extern unsigned long __start_mcount_loc[];
1480 extern unsigned long __stop_mcount_loc[];
1481
1482 void __init ftrace_init(void)
1483 {
1484         unsigned long count, addr, flags;
1485         int ret;
1486
1487         /* Keep the ftrace pointer to the stub */
1488         addr = (unsigned long)ftrace_stub;
1489
1490         local_irq_save(flags);
1491         ftrace_dyn_arch_init(&addr);
1492         local_irq_restore(flags);
1493
1494         /* ftrace_dyn_arch_init places the return code in addr */
1495         if (addr)
1496                 goto failed;
1497
1498         count = __stop_mcount_loc - __start_mcount_loc;
1499
1500         ret = ftrace_dyn_table_alloc(count);
1501         if (ret)
1502                 goto failed;
1503
1504         last_ftrace_enabled = ftrace_enabled = 1;
1505
1506         ret = ftrace_convert_nops(__start_mcount_loc,
1507                                   __stop_mcount_loc);
1508
1509         return;
1510  failed:
1511         ftrace_disabled = 1;
1512 }
1513
1514 #else
1515 # define ftrace_startup()               do { } while (0)
1516 # define ftrace_shutdown()              do { } while (0)
1517 # define ftrace_startup_sysctl()        do { } while (0)
1518 # define ftrace_shutdown_sysctl()       do { } while (0)
1519 #endif /* CONFIG_DYNAMIC_FTRACE */
1520
1521 /**
1522  * ftrace_kill - kill ftrace
1523  *
1524  * This function should be used by panic code. It stops ftrace
1525  * but in a not so nice way. If you need to simply kill ftrace
1526  * from a non-atomic section, use ftrace_kill.
1527  */
1528 void ftrace_kill(void)
1529 {
1530         ftrace_disabled = 1;
1531         ftrace_enabled = 0;
1532         clear_ftrace_function();
1533 }
1534
1535 /**
1536  * register_ftrace_function - register a function for profiling
1537  * @ops - ops structure that holds the function for profiling.
1538  *
1539  * Register a function to be called by all functions in the
1540  * kernel.
1541  *
1542  * Note: @ops->func and all the functions it calls must be labeled
1543  *       with "notrace", otherwise it will go into a
1544  *       recursive loop.
1545  */
1546 int register_ftrace_function(struct ftrace_ops *ops)
1547 {
1548         int ret;
1549
1550         if (unlikely(ftrace_disabled))
1551                 return -1;
1552
1553         mutex_lock(&ftrace_sysctl_lock);
1554         ret = __register_ftrace_function(ops);
1555         ftrace_startup();
1556         mutex_unlock(&ftrace_sysctl_lock);
1557
1558         return ret;
1559 }
1560
1561 /**
1562  * unregister_ftrace_function - unresgister a function for profiling.
1563  * @ops - ops structure that holds the function to unregister
1564  *
1565  * Unregister a function that was added to be called by ftrace profiling.
1566  */
1567 int unregister_ftrace_function(struct ftrace_ops *ops)
1568 {
1569         int ret;
1570
1571         mutex_lock(&ftrace_sysctl_lock);
1572         ret = __unregister_ftrace_function(ops);
1573         ftrace_shutdown();
1574         mutex_unlock(&ftrace_sysctl_lock);
1575
1576         return ret;
1577 }
1578
1579 int
1580 ftrace_enable_sysctl(struct ctl_table *table, int write,
1581                      struct file *file, void __user *buffer, size_t *lenp,
1582                      loff_t *ppos)
1583 {
1584         int ret;
1585
1586         if (unlikely(ftrace_disabled))
1587                 return -ENODEV;
1588
1589         mutex_lock(&ftrace_sysctl_lock);
1590
1591         ret  = proc_dointvec(table, write, file, buffer, lenp, ppos);
1592
1593         if (ret || !write || (last_ftrace_enabled == ftrace_enabled))
1594                 goto out;
1595
1596         last_ftrace_enabled = ftrace_enabled;
1597
1598         if (ftrace_enabled) {
1599
1600                 ftrace_startup_sysctl();
1601
1602                 /* we are starting ftrace again */
1603                 if (ftrace_list != &ftrace_list_end) {
1604                         if (ftrace_list->next == &ftrace_list_end)
1605                                 ftrace_trace_function = ftrace_list->func;
1606                         else
1607                                 ftrace_trace_function = ftrace_list_func;
1608                 }
1609
1610         } else {
1611                 /* stopping ftrace calls (just send to ftrace_stub) */
1612                 ftrace_trace_function = ftrace_stub;
1613
1614                 ftrace_shutdown_sysctl();
1615         }
1616
1617  out:
1618         mutex_unlock(&ftrace_sysctl_lock);
1619         return ret;
1620 }