kernel/kprobes.c

   1 /*
   2  *  Kernel Probes (KProbes)
   3  *  kernel/kprobes.c
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18  *
  19  * Copyright (C) IBM Corporation, 2002, 2004
  20  *
  21  * 2002-Oct     Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
  22  *              Probes initial implementation (includes suggestions from
  23  *              Rusty Russell).
  24  * 2004-Aug     Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
  25  *              hlists and exceptions notifier as suggested by Andi Kleen.
  26  * 2004-July    Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
  27  *              interface to access function arguments.
  28  * 2004-Sep     Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
  29  *              exceptions notifier to be first on the priority list.
  30  * 2005-May     Hien Nguyen <hien@us.ibm.com>, Jim Keniston
  31  *              <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
  32  *              <prasanna@in.ibm.com> added function-return probes.
  33  */
  34 #include <linux/kprobes.h>
  35 #include <linux/hash.h>
  36 #include <linux/init.h>
  37 #include <linux/slab.h>
  38 #include <linux/stddef.h>
  39 #include <linux/module.h>
  40 #include <linux/moduleloader.h>
  41 #include <linux/kallsyms.h>
  42 #include <linux/freezer.h>
  43 #include <linux/seq_file.h>
  44 #include <linux/debugfs.h>
  45 #include <linux/sysctl.h>
  46 #include <linux/kdebug.h>
  47 #include <linux/memory.h>
  48 #include <linux/ftrace.h>
  49 #include <linux/cpu.h>
  50 #include <linux/jump_label.h>
  51
  52 #include <asm-generic/sections.h>
  53 #include <asm/cacheflush.h>
  54 #include <asm/errno.h>
  55 #include <asm/uaccess.h>
  56
  57 #define KPROBE_HASH_BITS 6
  58 #define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
  59
  60
  61 /*
  62  * Some oddball architectures like 64bit powerpc have function descriptors
  63  * so this must be overridable.
  64  */
  65 #ifndef kprobe_lookup_name
  66 #define kprobe_lookup_name(name, addr) \
  67         addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name)))
  68 #endif
  69
  70 static int kprobes_initialized;
  71 static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
  72 static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
  73
  74 /* NOTE: change this value only with kprobe_mutex held */
  75 static bool kprobes_all_disarmed;
  76
  77 static DEFINE_MUTEX(kprobe_mutex);      /* Protects kprobe_table */
  78 static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
  79 static struct {
  80         spinlock_t lock ____cacheline_aligned_in_smp;
  81 } kretprobe_table_locks[KPROBE_TABLE_SIZE];
  82
  83 static spinlock_t *kretprobe_table_lock_ptr(unsigned long hash)
  84 {
  85         return &(kretprobe_table_locks[hash].lock);
  86 }
  87
  88 /*
  89  * Normally, functions that we'd want to prohibit kprobes in, are marked
  90  * __kprobes. But, there are cases where such functions already belong to
  91  * a different section (__sched for preempt_schedule)
  92  *
  93  * For such cases, we now have a blacklist
  94  */
  95 static struct kprobe_blackpoint kprobe_blacklist[] = {
  96         {"preempt_schedule",},
  97         {"native_get_debugreg",},
  98         {"irq_entries_start",},
  99         {"common_interrupt",},
 100         {"mcount",},    /* mcount can be called from everywhere */
 101         {NULL}    /* Terminator */
 102 };
 103
 104 #ifdef __ARCH_WANT_KPROBES_INSN_SLOT
 105 /*
 106  * kprobe->ainsn.insn points to the copy of the instruction to be
 107  * single-stepped. x86_64, POWER4 and above have no-exec support and
 108  * stepping on the instruction on a vmalloced/kmalloced/data page
 109  * is a recipe for disaster
 110  */
 111 struct kprobe_insn_page {
 112         struct list_head list;
 113         kprobe_opcode_t *insns;         /* Page of instruction slots */
 114         int nused;
 115         int ngarbage;
 116         char slot_used[];
 117 };
 118
 119 #define KPROBE_INSN_PAGE_SIZE(slots)                    \
 120         (offsetof(struct kprobe_insn_page, slot_used) + \
 121          (sizeof(char) * (slots)))
 122
 123 struct kprobe_insn_cache {
 124         struct list_head pages; /* list of kprobe_insn_page */
 125         size_t insn_size;       /* size of instruction slot */
 126         int nr_garbage;
 127 };
 128
 129 static int slots_per_page(struct kprobe_insn_cache *c)
 130 {
 131         return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));
 132 }
 133
 134 enum kprobe_slot_state {
 135         SLOT_CLEAN = 0,
 136         SLOT_DIRTY = 1,
 137         SLOT_USED = 2,
 138 };
 139
 140 static DEFINE_MUTEX(kprobe_insn_mutex); /* Protects kprobe_insn_slots */
 141 static struct kprobe_insn_cache kprobe_insn_slots = {
 142         .pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),
 143         .insn_size = MAX_INSN_SIZE,
 144         .nr_garbage = 0,
 145 };
 146 static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c);
 147
 148 /**
 149  * __get_insn_slot() - Find a slot on an executable page for an instruction.
 150  * We allocate an executable page if there's no room on existing ones.
 151  */
 152 static kprobe_opcode_t __kprobes *__get_insn_slot(struct kprobe_insn_cache *c)
 153 {
 154         struct kprobe_insn_page *kip;
 155
 156  retry:
 157         list_for_each_entry(kip, &c->pages, list) {
 158                 if (kip->nused < slots_per_page(c)) {
 159                         int i;
 160                         for (i = 0; i < slots_per_page(c); i++) {
 161                                 if (kip->slot_used[i] == SLOT_CLEAN) {
 162                                         kip->slot_used[i] = SLOT_USED;
 163                                         kip->nused++;
 164                                         return kip->insns + (i * c->insn_size);
 165                                 }
 166                         }
 167                         /* kip->nused is broken. Fix it. */
 168                         kip->nused = slots_per_page(c);
 169                         WARN_ON(1);
 170                 }
 171         }
 172
 173         /* If there are any garbage slots, collect it and try again. */
 174         if (c->nr_garbage && collect_garbage_slots(c) == 0)
 175                 goto retry;
 176
 177         /* All out of space.  Need to allocate a new page. */
 178         kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL);
 179         if (!kip)
 180                 return NULL;
 181
 182         /*
 183          * Use module_alloc so this page is within +/- 2GB of where the
 184          * kernel image and loaded module images reside. This is required
 185          * so x86_64 can correctly handle the %rip-relative fixups.
 186          */
 187         kip->insns = module_alloc(PAGE_SIZE);
 188         if (!kip->insns) {
 189                 kfree(kip);
 190                 return NULL;
 191         }
 192         INIT_LIST_HEAD(&kip->list);
 193         memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c));
 194         kip->slot_used[0] = SLOT_USED;
 195         kip->nused = 1;
 196         kip->ngarbage = 0;
 197         list_add(&kip->list, &c->pages);
 198         return kip->insns;
 199 }
 200
 201
 202 kprobe_opcode_t __kprobes *get_insn_slot(void)
 203 {
 204         kprobe_opcode_t *ret = NULL;
 205
 206         mutex_lock(&kprobe_insn_mutex);
 207         ret = __get_insn_slot(&kprobe_insn_slots);
 208         mutex_unlock(&kprobe_insn_mutex);
 209
 210         return ret;
 211 }
 212
 213 /* Return 1 if all garbages are collected, otherwise 0. */
 214 static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx)
 215 {
 216         kip->slot_used[idx] = SLOT_CLEAN;
 217         kip->nused--;
 218         if (kip->nused == 0) {
 219                 /*
 220                  * Page is no longer in use.  Free it unless
 221                  * it's the last one.  We keep the last one
 222                  * so as not to have to set it up again the
 223                  * next time somebody inserts a probe.
 224                  */
 225                 if (!list_is_singular(&kip->list)) {
 226                         list_del(&kip->list);
 227                         module_free(NULL, kip->insns);
 228                         kfree(kip);
 229                 }
 230                 return 1;
 231         }
 232         return 0;
 233 }
 234
 235 static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c)
 236 {
 237         struct kprobe_insn_page *kip, *next;
 238
 239         /* Ensure no-one is interrupted on the garbages */
 240         synchronize_sched();
 241
 242         list_for_each_entry_safe(kip, next, &c->pages, list) {
 243                 int i;
 244                 if (kip->ngarbage == 0)
 245                         continue;
 246                 kip->ngarbage = 0;      /* we will collect all garbages */
 247                 for (i = 0; i < slots_per_page(c); i++) {
 248                         if (kip->slot_used[i] == SLOT_DIRTY &&
 249                             collect_one_slot(kip, i))
 250                                 break;
 251                 }
 252         }
 253         c->nr_garbage = 0;
 254         return 0;
 255 }
 256
 257 static void __kprobes __free_insn_slot(struct kprobe_insn_cache *c,
 258                                        kprobe_opcode_t *slot, int dirty)
 259 {
 260         struct kprobe_insn_page *kip;
 261
 262         list_for_each_entry(kip, &c->pages, list) {
 263                 long idx = ((long)slot - (long)kip->insns) /
 264                                 (c->insn_size * sizeof(kprobe_opcode_t));
 265                 if (idx >= 0 && idx < slots_per_page(c)) {
 266                         WARN_ON(kip->slot_used[idx] != SLOT_USED);
 267                         if (dirty) {
 268                                 kip->slot_used[idx] = SLOT_DIRTY;
 269                                 kip->ngarbage++;
 270                                 if (++c->nr_garbage > slots_per_page(c))
 271                                         collect_garbage_slots(c);
 272                         } else
 273                                 collect_one_slot(kip, idx);
 274                         return;
 275                 }
 276         }
 277         /* Could not free this slot. */
 278         WARN_ON(1);
 279 }
 280
 281 void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty)
 282 {
 283         mutex_lock(&kprobe_insn_mutex);
 284         __free_insn_slot(&kprobe_insn_slots, slot, dirty);
 285         mutex_unlock(&kprobe_insn_mutex);
 286 }
 287 #ifdef CONFIG_OPTPROBES
 288 /* For optimized_kprobe buffer */
 289 static DEFINE_MUTEX(kprobe_optinsn_mutex); /* Protects kprobe_optinsn_slots */
 290 static struct kprobe_insn_cache kprobe_optinsn_slots = {
 291         .pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),
 292         /* .insn_size is initialized later */
 293         .nr_garbage = 0,
 294 };
 295 /* Get a slot for optimized_kprobe buffer */
 296 kprobe_opcode_t __kprobes *get_optinsn_slot(void)
 297 {
 298         kprobe_opcode_t *ret = NULL;
 299
 300         mutex_lock(&kprobe_optinsn_mutex);
 301         ret = __get_insn_slot(&kprobe_optinsn_slots);
 302         mutex_unlock(&kprobe_optinsn_mutex);
 303
 304         return ret;
 305 }
 306
 307 void __kprobes free_optinsn_slot(kprobe_opcode_t * slot, int dirty)
 308 {
 309         mutex_lock(&kprobe_optinsn_mutex);
 310         __free_insn_slot(&kprobe_optinsn_slots, slot, dirty);
 311         mutex_unlock(&kprobe_optinsn_mutex);
 312 }
 313 #endif
 314 #endif
 315
 316 /* We have preemption disabled.. so it is safe to use __ versions */
 317 static inline void set_kprobe_instance(struct kprobe *kp)
 318 {
 319         __get_cpu_var(kprobe_instance) = kp;
 320 }
 321
 322 static inline void reset_kprobe_instance(void)
 323 {
 324         __get_cpu_var(kprobe_instance) = NULL;
 325 }
 326
 327 /*
 328  * This routine is called either:
 329  *      - under the kprobe_mutex - during kprobe_[un]register()
 330  *                              OR
 331  *      - with preemption disabled - from arch/xxx/kernel/kprobes.c
 332  */
 333 struct kprobe __kprobes *get_kprobe(void *addr)
 334 {
 335         struct hlist_head *head;
 336         struct hlist_node *node;
 337         struct kprobe *p;
 338
 339         head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
 340         hlist_for_each_entry_rcu(p, node, head, hlist) {
 341                 if (p->addr == addr)
 342                         return p;
 343         }
 344
 345         return NULL;
 346 }
 347
 348 static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs);
 349
 350 /* Return true if the kprobe is an aggregator */
 351 static inline int kprobe_aggrprobe(struct kprobe *p)
 352 {
 353         return p->pre_handler == aggr_pre_handler;
 354 }
 355
 356 /*
 357  * Keep all fields in the kprobe consistent
 358  */
 359 static inline void copy_kprobe(struct kprobe *old_p, struct kprobe *p)
 360 {
 361         memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t));
 362         memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn));
 363 }
 364
 365 #ifdef CONFIG_OPTPROBES
 366 /* NOTE: change this value only with kprobe_mutex held */
 367 static bool kprobes_allow_optimization;
 368
 369 /*
 370  * Call all pre_handler on the list, but ignores its return value.
 371  * This must be called from arch-dep optimized caller.
 372  */
 373 void __kprobes opt_pre_handler(struct kprobe *p, struct pt_regs *regs)
 374 {
 375         struct kprobe *kp;
 376
 377         list_for_each_entry_rcu(kp, &p->list, list) {
 378                 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
 379                         set_kprobe_instance(kp);
 380                         kp->pre_handler(kp, regs);
 381                 }
 382                 reset_kprobe_instance();
 383         }
 384 }
 385
 386 /* Return true(!0) if the kprobe is ready for optimization. */
 387 static inline int kprobe_optready(struct kprobe *p)
 388 {
 389         struct optimized_kprobe *op;
 390
 391         if (kprobe_aggrprobe(p)) {
 392                 op = container_of(p, struct optimized_kprobe, kp);
 393                 return arch_prepared_optinsn(&op->optinsn);
 394         }
 395
 396         return 0;
 397 }
 398
 399 /*
 400  * Return an optimized kprobe whose optimizing code replaces
 401  * instructions including addr (exclude breakpoint).
 402  */
 403 static struct kprobe *__kprobes get_optimized_kprobe(unsigned long addr)
 404 {
 405         int i;
 406         struct kprobe *p = NULL;
 407         struct optimized_kprobe *op;
 408
 409         /* Don't check i == 0, since that is a breakpoint case. */
 410         for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH; i++)
 411                 p = get_kprobe((void *)(addr - i));
 412
 413         if (p && kprobe_optready(p)) {
 414                 op = container_of(p, struct optimized_kprobe, kp);
 415                 if (arch_within_optimized_kprobe(op, addr))
 416                         return p;
 417         }
 418
 419         return NULL;
 420 }
 421
 422 /* Optimization staging list, protected by kprobe_mutex */
 423 static LIST_HEAD(optimizing_list);
 424
 425 static void kprobe_optimizer(struct work_struct *work);
 426 static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);
 427 #define OPTIMIZE_DELAY 5
 428
 429 /* Kprobe jump optimizer */
 430 static __kprobes void kprobe_optimizer(struct work_struct *work)
 431 {
 432         struct optimized_kprobe *op, *tmp;
 433
 434         /* Lock modules while optimizing kprobes */
 435         mutex_lock(&module_mutex);
 436         mutex_lock(&kprobe_mutex);
 437         if (kprobes_all_disarmed || !kprobes_allow_optimization)
 438                 goto end;
 439
 440         /*
 441          * Wait for quiesence period to ensure all running interrupts
 442          * are done. Because optprobe may modify multiple instructions
 443          * there is a chance that Nth instruction is interrupted. In that
 444          * case, running interrupt can return to 2nd-Nth byte of jump
 445          * instruction. This wait is for avoiding it.
 446          */
 447         synchronize_sched();
 448
 449         /*
 450          * The optimization/unoptimization refers online_cpus via
 451          * stop_machine() and cpu-hotplug modifies online_cpus.
 452          * And same time, text_mutex will be held in cpu-hotplug and here.
 453          * This combination can cause a deadlock (cpu-hotplug try to lock
 454          * text_mutex but stop_machine can not be done because online_cpus
 455          * has been changed)
 456          * To avoid this deadlock, we need to call get_online_cpus()
 457          * for preventing cpu-hotplug outside of text_mutex locking.
 458          */
 459         get_online_cpus();
 460         mutex_lock(&text_mutex);
 461         list_for_each_entry_safe(op, tmp, &optimizing_list, list) {
 462                 WARN_ON(kprobe_disabled(&op->kp));
 463                 if (arch_optimize_kprobe(op) < 0)
 464                         op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
 465                 list_del_init(&op->list);
 466         }
 467         mutex_unlock(&text_mutex);
 468         put_online_cpus();
 469 end:
 470         mutex_unlock(&kprobe_mutex);
 471         mutex_unlock(&module_mutex);
 472 }
 473
 474 /* Optimize kprobe if p is ready to be optimized */
 475 static __kprobes void optimize_kprobe(struct kprobe *p)
 476 {
 477         struct optimized_kprobe *op;
 478
 479         /* Check if the kprobe is disabled or not ready for optimization. */
 480         if (!kprobe_optready(p) || !kprobes_allow_optimization ||
 481             (kprobe_disabled(p) || kprobes_all_disarmed))
 482                 return;
 483
 484         /* Both of break_handler and post_handler are not supported. */
 485         if (p->break_handler || p->post_handler)
 486                 return;
 487
 488         op = container_of(p, struct optimized_kprobe, kp);
 489
 490         /* Check there is no other kprobes at the optimized instructions */
 491         if (arch_check_optimized_kprobe(op) < 0)
 492                 return;
 493
 494         /* Check if it is already optimized. */
 495         if (op->kp.flags & KPROBE_FLAG_OPTIMIZED)
 496                 return;
 497
 498         op->kp.flags |= KPROBE_FLAG_OPTIMIZED;
 499         list_add(&op->list, &optimizing_list);
 500         if (!delayed_work_pending(&optimizing_work))
 501                 schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);
 502 }
 503
 504 /* Unoptimize a kprobe if p is optimized */
 505 static __kprobes void unoptimize_kprobe(struct kprobe *p)
 506 {
 507         struct optimized_kprobe *op;
 508
 509         if ((p->flags & KPROBE_FLAG_OPTIMIZED) && kprobe_aggrprobe(p)) {
 510                 op = container_of(p, struct optimized_kprobe, kp);
 511                 if (!list_empty(&op->list))
 512                         /* Dequeue from the optimization queue */
 513                         list_del_init(&op->list);
 514                 else
 515                         /* Replace jump with break */
 516                         arch_unoptimize_kprobe(op);
 517                 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
 518         }
 519 }
 520
 521 /* Remove optimized instructions */
 522 static void __kprobes kill_optimized_kprobe(struct kprobe *p)
 523 {
 524         struct optimized_kprobe *op;
 525
 526         op = container_of(p, struct optimized_kprobe, kp);
 527         if (!list_empty(&op->list)) {
 528                 /* Dequeue from the optimization queue */
 529                 list_del_init(&op->list);
 530                 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
 531         }
 532         /* Don't unoptimize, because the target code will be freed. */
 533         arch_remove_optimized_kprobe(op);
 534 }
 535
 536 /* Try to prepare optimized instructions */
 537 static __kprobes void prepare_optimized_kprobe(struct kprobe *p)
 538 {
 539         struct optimized_kprobe *op;
 540
 541         op = container_of(p, struct optimized_kprobe, kp);
 542         arch_prepare_optimized_kprobe(op);
 543 }
 544
 545 /* Free optimized instructions and optimized_kprobe */
 546 static __kprobes void free_aggr_kprobe(struct kprobe *p)
 547 {
 548         struct optimized_kprobe *op;
 549
 550         op = container_of(p, struct optimized_kprobe, kp);
 551         arch_remove_optimized_kprobe(op);
 552         kfree(op);
 553 }
 554
 555 /* Allocate new optimized_kprobe and try to prepare optimized instructions */
 556 static __kprobes struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
 557 {
 558         struct optimized_kprobe *op;
 559
 560         op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL);
 561         if (!op)
 562                 return NULL;
 563
 564         INIT_LIST_HEAD(&op->list);
 565         op->kp.addr = p->addr;
 566         arch_prepare_optimized_kprobe(op);
 567
 568         return &op->kp;
 569 }
 570
 571 static void __kprobes init_aggr_kprobe(struct kprobe *ap, struct kprobe *p);
 572
 573 /*
 574  * Prepare an optimized_kprobe and optimize it
 575  * NOTE: p must be a normal registered kprobe
 576  */
 577 static __kprobes void try_to_optimize_kprobe(struct kprobe *p)
 578 {
 579         struct kprobe *ap;
 580         struct optimized_kprobe *op;
 581
 582         ap = alloc_aggr_kprobe(p);
 583         if (!ap)
 584                 return;
 585
 586         op = container_of(ap, struct optimized_kprobe, kp);
 587         if (!arch_prepared_optinsn(&op->optinsn)) {
 588                 /* If failed to setup optimizing, fallback to kprobe */
 589                 free_aggr_kprobe(ap);
 590                 return;
 591         }
 592
 593         init_aggr_kprobe(ap, p);
 594         optimize_kprobe(ap);
 595 }
 596
 597 #ifdef CONFIG_SYSCTL
 598 static void __kprobes optimize_all_kprobes(void)
 599 {
 600         struct hlist_head *head;
 601         struct hlist_node *node;
 602         struct kprobe *p;
 603         unsigned int i;
 604
 605         /* If optimization is already allowed, just return */
 606         if (kprobes_allow_optimization)
 607                 return;
 608
 609         kprobes_allow_optimization = true;
 610         mutex_lock(&text_mutex);
 611         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
 612                 head = &kprobe_table[i];
 613                 hlist_for_each_entry_rcu(p, node, head, hlist)
 614                         if (!kprobe_disabled(p))
 615                                 optimize_kprobe(p);
 616         }
 617         mutex_unlock(&text_mutex);
 618         printk(KERN_INFO "Kprobes globally optimized\n");
 619 }
 620
 621 static void __kprobes unoptimize_all_kprobes(void)
 622 {
 623         struct hlist_head *head;
 624         struct hlist_node *node;
 625         struct kprobe *p;
 626         unsigned int i;
 627
 628         /* If optimization is already prohibited, just return */
 629         if (!kprobes_allow_optimization)
 630                 return;
 631
 632         kprobes_allow_optimization = false;
 633         printk(KERN_INFO "Kprobes globally unoptimized\n");
 634         get_online_cpus();      /* For avoiding text_mutex deadlock */
 635         mutex_lock(&text_mutex);
 636         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
 637                 head = &kprobe_table[i];
 638                 hlist_for_each_entry_rcu(p, node, head, hlist) {
 639                         if (!kprobe_disabled(p))
 640                                 unoptimize_kprobe(p);
 641                 }
 642         }
 643
 644         mutex_unlock(&text_mutex);
 645         put_online_cpus();
 646         /* Allow all currently running kprobes to complete */
 647         synchronize_sched();
 648 }
 649
 650 int sysctl_kprobes_optimization;
 651 int proc_kprobes_optimization_handler(struct ctl_table *table, int write,
 652                                       void __user *buffer, size_t *length,
 653                                       loff_t *ppos)
 654 {
 655         int ret;
 656
 657         mutex_lock(&kprobe_mutex);
 658         sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0;
 659         ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
 660
 661         if (sysctl_kprobes_optimization)
 662                 optimize_all_kprobes();
 663         else
 664                 unoptimize_all_kprobes();
 665         mutex_unlock(&kprobe_mutex);
 666
 667         return ret;
 668 }
 669 #endif /* CONFIG_SYSCTL */
 670
 671 static void __kprobes __arm_kprobe(struct kprobe *p)
 672 {
 673         struct kprobe *old_p;
 674
 675         /* Check collision with other optimized kprobes */
 676         old_p = get_optimized_kprobe((unsigned long)p->addr);
 677         if (unlikely(old_p))
 678                 unoptimize_kprobe(old_p); /* Fallback to unoptimized kprobe */
 679
 680         arch_arm_kprobe(p);
 681         optimize_kprobe(p);     /* Try to optimize (add kprobe to a list) */
 682 }
 683
 684 static void __kprobes __disarm_kprobe(struct kprobe *p)
 685 {
 686         struct kprobe *old_p;
 687
 688         unoptimize_kprobe(p);   /* Try to unoptimize */
 689         arch_disarm_kprobe(p);
 690
 691         /* If another kprobe was blocked, optimize it. */
 692         old_p = get_optimized_kprobe((unsigned long)p->addr);
 693         if (unlikely(old_p))
 694                 optimize_kprobe(old_p);
 695 }
 696
 697 #else /* !CONFIG_OPTPROBES */
 698
 699 #define optimize_kprobe(p)                      do {} while (0)
 700 #define unoptimize_kprobe(p)                    do {} while (0)
 701 #define kill_optimized_kprobe(p)                do {} while (0)
 702 #define prepare_optimized_kprobe(p)             do {} while (0)
 703 #define try_to_optimize_kprobe(p)               do {} while (0)
 704 #define __arm_kprobe(p)                         arch_arm_kprobe(p)
 705 #define __disarm_kprobe(p)                      arch_disarm_kprobe(p)
 706
 707 static __kprobes void free_aggr_kprobe(struct kprobe *p)
 708 {
 709         kfree(p);
 710 }
 711
 712 static __kprobes struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
 713 {
 714         return kzalloc(sizeof(struct kprobe), GFP_KERNEL);
 715 }
 716 #endif /* CONFIG_OPTPROBES */
 717
 718 /* Arm a kprobe with text_mutex */
 719 static void __kprobes arm_kprobe(struct kprobe *kp)
 720 {
 721         /*
 722          * Here, since __arm_kprobe() doesn't use stop_machine(),
 723          * this doesn't cause deadlock on text_mutex. So, we don't
 724          * need get_online_cpus().
 725          */
 726         mutex_lock(&text_mutex);
 727         __arm_kprobe(kp);
 728         mutex_unlock(&text_mutex);
 729 }
 730
 731 /* Disarm a kprobe with text_mutex */
 732 static void __kprobes disarm_kprobe(struct kprobe *kp)
 733 {
 734         get_online_cpus();      /* For avoiding text_mutex deadlock */
 735         mutex_lock(&text_mutex);
 736         __disarm_kprobe(kp);
 737         mutex_unlock(&text_mutex);
 738         put_online_cpus();
 739 }
 740
 741 /*
 742  * Aggregate handlers for multiple kprobes support - these handlers
 743  * take care of invoking the individual kprobe handlers on p->list
 744  */
 745 static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
 746 {
 747         struct kprobe *kp;
 748
 749         list_for_each_entry_rcu(kp, &p->list, list) {
 750                 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
 751                         set_kprobe_instance(kp);
 752                         if (kp->pre_handler(kp, regs))
 753                                 return 1;
 754                 }
 755                 reset_kprobe_instance();
 756         }
 757         return 0;
 758 }
 759
 760 static void __kprobes aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
 761                                         unsigned long flags)
 762 {
 763         struct kprobe *kp;
 764
 765         list_for_each_entry_rcu(kp, &p->list, list) {
 766                 if (kp->post_handler && likely(!kprobe_disabled(kp))) {
 767                         set_kprobe_instance(kp);
 768                         kp->post_handler(kp, regs, flags);
 769                         reset_kprobe_instance();
 770                 }
 771         }
 772 }
 773
 774 static int __kprobes aggr_fault_handler(struct kprobe *p, struct pt_regs *regs,
 775                                         int trapnr)
 776 {
 777         struct kprobe *cur = __get_cpu_var(kprobe_instance);
 778
 779         /*
 780          * if we faulted "during" the execution of a user specified
 781          * probe handler, invoke just that probe's fault handler
 782          */
 783         if (cur && cur->fault_handler) {
 784                 if (cur->fault_handler(cur, regs, trapnr))
 785                         return 1;
 786         }
 787         return 0;
 788 }
 789
 790 static int __kprobes aggr_break_handler(struct kprobe *p, struct pt_regs *regs)
 791 {
 792         struct kprobe *cur = __get_cpu_var(kprobe_instance);
 793         int ret = 0;
 794
 795         if (cur && cur->break_handler) {
 796                 if (cur->break_handler(cur, regs))
 797                         ret = 1;
 798         }
 799         reset_kprobe_instance();
 800         return ret;
 801 }
 802
 803 /* Walks the list and increments nmissed count for multiprobe case */
 804 void __kprobes kprobes_inc_nmissed_count(struct kprobe *p)
 805 {
 806         struct kprobe *kp;
 807         if (!kprobe_aggrprobe(p)) {
 808                 p->nmissed++;
 809         } else {
 810                 list_for_each_entry_rcu(kp, &p->list, list)
 811                         kp->nmissed++;
 812         }
 813         return;
 814 }
 815
 816 void __kprobes recycle_rp_inst(struct kretprobe_instance *ri,
 817                                 struct hlist_head *head)
 818 {
 819         struct kretprobe *rp = ri->rp;
 820
 821         /* remove rp inst off the rprobe_inst_table */
 822         hlist_del(&ri->hlist);
 823         INIT_HLIST_NODE(&ri->hlist);
 824         if (likely(rp)) {
 825                 spin_lock(&rp->lock);
 826                 hlist_add_head(&ri->hlist, &rp->free_instances);
 827                 spin_unlock(&rp->lock);
 828         } else
 829                 /* Unregistering */
 830                 hlist_add_head(&ri->hlist, head);
 831 }
 832
 833 void __kprobes kretprobe_hash_lock(struct task_struct *tsk,
 834                          struct hlist_head **head, unsigned long *flags)
 835 __acquires(hlist_lock)
 836 {
 837         unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
 838         spinlock_t *hlist_lock;
 839
 840         *head = &kretprobe_inst_table[hash];
 841         hlist_lock = kretprobe_table_lock_ptr(hash);
 842         spin_lock_irqsave(hlist_lock, *flags);
 843 }
 844
 845 static void __kprobes kretprobe_table_lock(unsigned long hash,
 846         unsigned long *flags)
 847 __acquires(hlist_lock)
 848 {
 849         spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
 850         spin_lock_irqsave(hlist_lock, *flags);
 851 }
 852
 853 void __kprobes kretprobe_hash_unlock(struct task_struct *tsk,
 854         unsigned long *flags)
 855 __releases(hlist_lock)
 856 {
 857         unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
 858         spinlock_t *hlist_lock;
 859
 860         hlist_lock = kretprobe_table_lock_ptr(hash);
 861         spin_unlock_irqrestore(hlist_lock, *flags);
 862 }
 863
 864 static void __kprobes kretprobe_table_unlock(unsigned long hash,
 865        unsigned long *flags)
 866 __releases(hlist_lock)
 867 {
 868         spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
 869         spin_unlock_irqrestore(hlist_lock, *flags);
 870 }
 871
 872 /*
 873  * This function is called from finish_task_switch when task tk becomes dead,
 874  * so that we can recycle any function-return probe instances associated
 875  * with this task. These left over instances represent probed functions
 876  * that have been called but will never return.
 877  */
 878 void __kprobes kprobe_flush_task(struct task_struct *tk)
 879 {
 880         struct kretprobe_instance *ri;
 881         struct hlist_head *head, empty_rp;
 882         struct hlist_node *node, *tmp;
 883         unsigned long hash, flags = 0;
 884
 885         if (unlikely(!kprobes_initialized))
 886                 /* Early boot.  kretprobe_table_locks not yet initialized. */
 887                 return;
 888
 889         hash = hash_ptr(tk, KPROBE_HASH_BITS);
 890         head = &kretprobe_inst_table[hash];
 891         kretprobe_table_lock(hash, &flags);
 892         hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
 893                 if (ri->task == tk)
 894                         recycle_rp_inst(ri, &empty_rp);
 895         }
 896         kretprobe_table_unlock(hash, &flags);
 897         INIT_HLIST_HEAD(&empty_rp);
 898         hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
 899                 hlist_del(&ri->hlist);
 900                 kfree(ri);
 901         }
 902 }
 903
 904 static inline void free_rp_inst(struct kretprobe *rp)
 905 {
 906         struct kretprobe_instance *ri;
 907         struct hlist_node *pos, *next;
 908
 909         hlist_for_each_entry_safe(ri, pos, next, &rp->free_instances, hlist) {
 910                 hlist_del(&ri->hlist);
 911                 kfree(ri);
 912         }
 913 }
 914
 915 static void __kprobes cleanup_rp_inst(struct kretprobe *rp)
 916 {
 917         unsigned long flags, hash;
 918         struct kretprobe_instance *ri;
 919         struct hlist_node *pos, *next;
 920         struct hlist_head *head;
 921
 922         /* No race here */
 923         for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) {
 924                 kretprobe_table_lock(hash, &flags);
 925                 head = &kretprobe_inst_table[hash];
 926                 hlist_for_each_entry_safe(ri, pos, next, head, hlist) {
 927                         if (ri->rp == rp)
 928                                 ri->rp = NULL;
 929                 }
 930                 kretprobe_table_unlock(hash, &flags);
 931         }
 932         free_rp_inst(rp);
 933 }
 934
 935 /*
 936 * Add the new probe to ap->list. Fail if this is the
 937 * second jprobe at the address - two jprobes can't coexist
 938 */
 939 static int __kprobes add_new_kprobe(struct kprobe *ap, struct kprobe *p)
 940 {
 941         BUG_ON(kprobe_gone(ap) || kprobe_gone(p));
 942
 943         if (p->break_handler || p->post_handler)
 944                 unoptimize_kprobe(ap);  /* Fall back to normal kprobe */
 945
 946         if (p->break_handler) {
 947                 if (ap->break_handler)
 948                         return -EEXIST;
 949                 list_add_tail_rcu(&p->list, &ap->list);
 950                 ap->break_handler = aggr_break_handler;
 951         } else
 952                 list_add_rcu(&p->list, &ap->list);
 953         if (p->post_handler && !ap->post_handler)
 954                 ap->post_handler = aggr_post_handler;
 955
 956         if (kprobe_disabled(ap) && !kprobe_disabled(p)) {
 957                 ap->flags &= ~KPROBE_FLAG_DISABLED;
 958                 if (!kprobes_all_disarmed)
 959                         /* Arm the breakpoint again. */
 960                         __arm_kprobe(ap);
 961         }
 962         return 0;
 963 }
 964
 965 /*
 966  * Fill in the required fields of the "manager kprobe". Replace the
 967  * earlier kprobe in the hlist with the manager kprobe
 968  */
 969 static void __kprobes init_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
 970 {
 971         /* Copy p's insn slot to ap */
 972         copy_kprobe(p, ap);
 973         flush_insn_slot(ap);
 974         ap->addr = p->addr;
 975         ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED;
 976         ap->pre_handler = aggr_pre_handler;
 977         ap->fault_handler = aggr_fault_handler;
 978         /* We don't care the kprobe which has gone. */
 979         if (p->post_handler && !kprobe_gone(p))
 980                 ap->post_handler = aggr_post_handler;
 981         if (p->break_handler && !kprobe_gone(p))
 982                 ap->break_handler = aggr_break_handler;
 983
 984         INIT_LIST_HEAD(&ap->list);
 985         INIT_HLIST_NODE(&ap->hlist);
 986
 987         list_add_rcu(&p->list, &ap->list);
 988         hlist_replace_rcu(&p->hlist, &ap->hlist);
 989 }
 990
 991 /*
 992  * This is the second or subsequent kprobe at the address - handle
 993  * the intricacies
 994  */
 995 static int __kprobes register_aggr_kprobe(struct kprobe *old_p,
 996                                           struct kprobe *p)
 997 {
 998         int ret = 0;
 999         struct kprobe *ap = old_p;
1000
1001         if (!kprobe_aggrprobe(old_p)) {
1002                 /* If old_p is not an aggr_kprobe, create new aggr_kprobe. */
1003                 ap = alloc_aggr_kprobe(old_p);
1004                 if (!ap)
1005                         return -ENOMEM;
1006                 init_aggr_kprobe(ap, old_p);
1007         }
1008
1009         if (kprobe_gone(ap)) {
1010                 /*
1011                  * Attempting to insert new probe at the same location that
1012                  * had a probe in the module vaddr area which already
1013                  * freed. So, the instruction slot has already been
1014                  * released. We need a new slot for the new probe.
1015                  */
1016                 ret = arch_prepare_kprobe(ap);
1017                 if (ret)
1018                         /*
1019                          * Even if fail to allocate new slot, don't need to
1020                          * free aggr_probe. It will be used next time, or
1021                          * freed by unregister_kprobe.
1022                          */
1023                         return ret;
1024
1025                 /* Prepare optimized instructions if possible. */
1026                 prepare_optimized_kprobe(ap);
1027
1028                 /*
1029                  * Clear gone flag to prevent allocating new slot again, and
1030                  * set disabled flag because it is not armed yet.
1031                  */
1032                 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
1033                             | KPROBE_FLAG_DISABLED;
1034         }
1035
1036         /* Copy ap's insn slot to p */
1037         copy_kprobe(ap, p);
1038         return add_new_kprobe(ap, p);
1039 }
1040
1041 /* Try to disable aggr_kprobe, and return 1 if succeeded.*/
1042 static int __kprobes try_to_disable_aggr_kprobe(struct kprobe *p)
1043 {
1044         struct kprobe *kp;
1045
1046         list_for_each_entry_rcu(kp, &p->list, list) {
1047                 if (!kprobe_disabled(kp))
1048                         /*
1049                          * There is an active probe on the list.
1050                          * We can't disable aggr_kprobe.
1051                          */
1052                         return 0;
1053         }
1054         p->flags |= KPROBE_FLAG_DISABLED;
1055         return 1;
1056 }
1057
1058 static int __kprobes in_kprobes_functions(unsigned long addr)
1059 {
1060         struct kprobe_blackpoint *kb;
1061
1062         if (addr >= (unsigned long)__kprobes_text_start &&
1063             addr < (unsigned long)__kprobes_text_end)
1064                 return -EINVAL;
1065         /*
1066          * If there exists a kprobe_blacklist, verify and
1067          * fail any probe registration in the prohibited area
1068          */
1069         for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
1070                 if (kb->start_addr) {
1071                         if (addr >= kb->start_addr &&
1072                             addr < (kb->start_addr + kb->range))
1073                                 return -EINVAL;
1074                 }
1075         }
1076         return 0;
1077 }
1078
1079 /*
1080  * If we have a symbol_name argument, look it up and add the offset field
1081  * to it. This way, we can specify a relative address to a symbol.
1082  */
1083 static kprobe_opcode_t __kprobes *kprobe_addr(struct kprobe *p)
1084 {
1085         kprobe_opcode_t *addr = p->addr;
1086         if (p->symbol_name) {
1087                 if (addr)
1088                         return NULL;
1089                 kprobe_lookup_name(p->symbol_name, addr);
1090         }
1091
1092         if (!addr)
1093                 return NULL;
1094         return (kprobe_opcode_t *)(((char *)addr) + p->offset);
1095 }
1096
1097 /* Check passed kprobe is valid and return kprobe in kprobe_table. */
1098 static struct kprobe * __kprobes __get_valid_kprobe(struct kprobe *p)
1099 {
1100         struct kprobe *old_p, *list_p;
1101
1102         old_p = get_kprobe(p->addr);
1103         if (unlikely(!old_p))
1104                 return NULL;
1105
1106         if (p != old_p) {
1107                 list_for_each_entry_rcu(list_p, &old_p->list, list)
1108                         if (list_p == p)
1109                         /* kprobe p is a valid probe */
1110                                 goto valid;
1111                 return NULL;
1112         }
1113 valid:
1114         return old_p;
1115 }
1116
1117 /* Return error if the kprobe is being re-registered */
1118 static inline int check_kprobe_rereg(struct kprobe *p)
1119 {
1120         int ret = 0;
1121         struct kprobe *old_p;
1122
1123         mutex_lock(&kprobe_mutex);
1124         old_p = __get_valid_kprobe(p);
1125         if (old_p)
1126                 ret = -EINVAL;
1127         mutex_unlock(&kprobe_mutex);
1128         return ret;
1129 }
1130
1131 int __kprobes register_kprobe(struct kprobe *p)
1132 {
1133         int ret = 0;
1134         struct kprobe *old_p;
1135         struct module *probed_mod;
1136         kprobe_opcode_t *addr;
1137
1138         addr = kprobe_addr(p);
1139         if (!addr)
1140                 return -EINVAL;
1141         p->addr = addr;
1142
1143         ret = check_kprobe_rereg(p);
1144         if (ret)
1145                 return ret;
1146
1147         preempt_disable();
1148         if (!kernel_text_address((unsigned long) p->addr) ||
1149             in_kprobes_functions((unsigned long) p->addr) ||
1150             ftrace_text_reserved(p->addr, p->addr) ||
1151             jump_label_text_reserved(p->addr, p->addr)) {
1152                 preempt_enable();
1153                 return -EINVAL;
1154         }
1155
1156         /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */
1157         p->flags &= KPROBE_FLAG_DISABLED;
1158
1159         /*
1160          * Check if are we probing a module.
1161          */
1162         probed_mod = __module_text_address((unsigned long) p->addr);
1163         if (probed_mod) {
1164                 /*
1165                  * We must hold a refcount of the probed module while updating
1166                  * its code to prohibit unexpected unloading.
1167                  */
1168                 if (unlikely(!try_module_get(probed_mod))) {
1169                         preempt_enable();
1170                         return -EINVAL;
1171                 }
1172                 /*
1173                  * If the module freed .init.text, we couldn't insert
1174                  * kprobes in there.
1175                  */
1176                 if (within_module_init((unsigned long)p->addr, probed_mod) &&
1177                     probed_mod->state != MODULE_STATE_COMING) {
1178                         module_put(probed_mod);
1179                         preempt_enable();
1180                         return -EINVAL;
1181                 }
1182         }
1183         preempt_enable();
1184
1185         p->nmissed = 0;
1186         INIT_LIST_HEAD(&p->list);
1187         mutex_lock(&kprobe_mutex);
1188
1189         get_online_cpus();      /* For avoiding text_mutex deadlock. */
1190         mutex_lock(&text_mutex);
1191
1192         old_p = get_kprobe(p->addr);
1193         if (old_p) {
1194                 /* Since this may unoptimize old_p, locking text_mutex. */
1195                 ret = register_aggr_kprobe(old_p, p);
1196                 goto out;
1197         }
1198
1199         ret = arch_prepare_kprobe(p);
1200         if (ret)
1201                 goto out;
1202
1203         INIT_HLIST_NODE(&p->hlist);
1204         hlist_add_head_rcu(&p->hlist,
1205                        &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
1206
1207         if (!kprobes_all_disarmed && !kprobe_disabled(p))
1208                 __arm_kprobe(p);
1209
1210         /* Try to optimize kprobe */
1211         try_to_optimize_kprobe(p);
1212
1213 out:
1214         mutex_unlock(&text_mutex);
1215         put_online_cpus();
1216         mutex_unlock(&kprobe_mutex);
1217
1218         if (probed_mod)
1219                 module_put(probed_mod);
1220
1221         return ret;
1222 }
1223 EXPORT_SYMBOL_GPL(register_kprobe);
1224
1225 /*
1226  * Unregister a kprobe without a scheduler synchronization.
1227  */
1228 static int __kprobes __unregister_kprobe_top(struct kprobe *p)
1229 {
1230         struct kprobe *old_p, *list_p;
1231
1232         old_p = __get_valid_kprobe(p);
1233         if (old_p == NULL)
1234                 return -EINVAL;
1235
1236         if (old_p == p ||
1237             (kprobe_aggrprobe(old_p) &&
1238              list_is_singular(&old_p->list))) {
1239                 /*
1240                  * Only probe on the hash list. Disarm only if kprobes are
1241                  * enabled and not gone - otherwise, the breakpoint would
1242                  * already have been removed. We save on flushing icache.
1243                  */
1244                 if (!kprobes_all_disarmed && !kprobe_disabled(old_p))
1245                         disarm_kprobe(old_p);
1246                 hlist_del_rcu(&old_p->hlist);
1247         } else {
1248                 if (p->break_handler && !kprobe_gone(p))
1249                         old_p->break_handler = NULL;
1250                 if (p->post_handler && !kprobe_gone(p)) {
1251                         list_for_each_entry_rcu(list_p, &old_p->list, list) {
1252                                 if ((list_p != p) && (list_p->post_handler))
1253                                         goto noclean;
1254                         }
1255                         old_p->post_handler = NULL;
1256                 }
1257 noclean:
1258                 list_del_rcu(&p->list);
1259                 if (!kprobe_disabled(old_p)) {
1260                         try_to_disable_aggr_kprobe(old_p);
1261                         if (!kprobes_all_disarmed) {
1262                                 if (kprobe_disabled(old_p))
1263                                         disarm_kprobe(old_p);
1264                                 else
1265                                         /* Try to optimize this probe again */
1266                                         optimize_kprobe(old_p);
1267                         }
1268                 }
1269         }
1270         return 0;
1271 }
1272
1273 static void __kprobes __unregister_kprobe_bottom(struct kprobe *p)
1274 {
1275         struct kprobe *old_p;
1276
1277         if (list_empty(&p->list))
1278                 arch_remove_kprobe(p);
1279         else if (list_is_singular(&p->list)) {
1280                 /* "p" is the last child of an aggr_kprobe */
1281                 old_p = list_entry(p->list.next, struct kprobe, list);
1282                 list_del(&p->list);
1283                 arch_remove_kprobe(old_p);
1284                 free_aggr_kprobe(old_p);
1285         }
1286 }
1287
1288 int __kprobes register_kprobes(struct kprobe **kps, int num)
1289 {
1290         int i, ret = 0;
1291
1292         if (num <= 0)
1293                 return -EINVAL;
1294         for (i = 0; i < num; i++) {
1295                 ret = register_kprobe(kps[i]);
1296                 if (ret < 0) {
1297                         if (i > 0)
1298                                 unregister_kprobes(kps, i);
1299                         break;
1300                 }
1301         }
1302         return ret;
1303 }
1304 EXPORT_SYMBOL_GPL(register_kprobes);
1305
1306 void __kprobes unregister_kprobe(struct kprobe *p)
1307 {
1308         unregister_kprobes(&p, 1);
1309 }
1310 EXPORT_SYMBOL_GPL(unregister_kprobe);
1311
1312 void __kprobes unregister_kprobes(struct kprobe **kps, int num)
1313 {
1314         int i;
1315
1316         if (num <= 0)
1317                 return;
1318         mutex_lock(&kprobe_mutex);
1319         for (i = 0; i < num; i++)
1320                 if (__unregister_kprobe_top(kps[i]) < 0)
1321                         kps[i]->addr = NULL;
1322         mutex_unlock(&kprobe_mutex);
1323
1324         synchronize_sched();
1325         for (i = 0; i < num; i++)
1326                 if (kps[i]->addr)
1327                         __unregister_kprobe_bottom(kps[i]);
1328 }
1329 EXPORT_SYMBOL_GPL(unregister_kprobes);
1330
1331 static struct notifier_block kprobe_exceptions_nb = {
1332         .notifier_call = kprobe_exceptions_notify,
1333         .priority = 0x7fffffff /* we need to be notified first */
1334 };
1335
1336 unsigned long __weak arch_deref_entry_point(void *entry)
1337 {
1338         return (unsigned long)entry;
1339 }
1340
1341 int __kprobes register_jprobes(struct jprobe **jps, int num)
1342 {
1343         struct jprobe *jp;
1344         int ret = 0, i;
1345
1346         if (num <= 0)
1347                 return -EINVAL;
1348         for (i = 0; i < num; i++) {
1349                 unsigned long addr, offset;
1350                 jp = jps[i];
1351                 addr = arch_deref_entry_point(jp->entry);
1352
1353                 /* Verify probepoint is a function entry point */
1354                 if (kallsyms_lookup_size_offset(addr, NULL, &offset) &&
1355                     offset == 0) {
1356                         jp->kp.pre_handler = setjmp_pre_handler;
1357                         jp->kp.break_handler = longjmp_break_handler;
1358                         ret = register_kprobe(&jp->kp);
1359                 } else
1360                         ret = -EINVAL;
1361
1362                 if (ret < 0) {
1363                         if (i > 0)
1364                                 unregister_jprobes(jps, i);
1365                         break;
1366                 }
1367         }
1368         return ret;
1369 }
1370 EXPORT_SYMBOL_GPL(register_jprobes);
1371
1372 int __kprobes register_jprobe(struct jprobe *jp)
1373 {
1374         return register_jprobes(&jp, 1);
1375 }
1376 EXPORT_SYMBOL_GPL(register_jprobe);
1377
1378 void __kprobes unregister_jprobe(struct jprobe *jp)
1379 {
1380         unregister_jprobes(&jp, 1);
1381 }
1382 EXPORT_SYMBOL_GPL(unregister_jprobe);
1383
1384 void __kprobes unregister_jprobes(struct jprobe **jps, int num)
1385 {
1386         int i;
1387
1388         if (num <= 0)
1389                 return;
1390         mutex_lock(&kprobe_mutex);
1391         for (i = 0; i < num; i++)
1392                 if (__unregister_kprobe_top(&jps[i]->kp) < 0)
1393                         jps[i]->kp.addr = NULL;
1394         mutex_unlock(&kprobe_mutex);
1395
1396         synchronize_sched();
1397         for (i = 0; i < num; i++) {
1398                 if (jps[i]->kp.addr)
1399                         __unregister_kprobe_bottom(&jps[i]->kp);
1400         }
1401 }
1402 EXPORT_SYMBOL_GPL(unregister_jprobes);
1403
1404 #ifdef CONFIG_KRETPROBES
1405 /*
1406  * This kprobe pre_handler is registered with every kretprobe. When probe
1407  * hits it will set up the return probe.
1408  */
1409 static int __kprobes pre_handler_kretprobe(struct kprobe *p,
1410                                            struct pt_regs *regs)
1411 {
1412         struct kretprobe *rp = container_of(p, struct kretprobe, kp);
1413         unsigned long hash, flags = 0;
1414         struct kretprobe_instance *ri;
1415
1416         /*TODO: consider to only swap the RA after the last pre_handler fired */
1417         hash = hash_ptr(current, KPROBE_HASH_BITS);
1418         spin_lock_irqsave(&rp->lock, flags);
1419         if (!hlist_empty(&rp->free_instances)) {
1420                 ri = hlist_entry(rp->free_instances.first,
1421                                 struct kretprobe_instance, hlist);
1422                 hlist_del(&ri->hlist);
1423                 spin_unlock_irqrestore(&rp->lock, flags);
1424
1425                 ri->rp = rp;
1426                 ri->task = current;
1427
1428                 if (rp->entry_handler && rp->entry_handler(ri, regs))
1429                         return 0;
1430
1431                 arch_prepare_kretprobe(ri, regs);
1432
1433                 /* XXX(hch): why is there no hlist_move_head? */
1434                 INIT_HLIST_NODE(&ri->hlist);
1435                 kretprobe_table_lock(hash, &flags);
1436                 hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]);
1437                 kretprobe_table_unlock(hash, &flags);
1438         } else {
1439                 rp->nmissed++;
1440                 spin_unlock_irqrestore(&rp->lock, flags);
1441         }
1442         return 0;
1443 }
1444
1445 int __kprobes register_kretprobe(struct kretprobe *rp)
1446 {
1447         int ret = 0;
1448         struct kretprobe_instance *inst;
1449         int i;
1450         void *addr;
1451
1452         if (kretprobe_blacklist_size) {
1453                 addr = kprobe_addr(&rp->kp);
1454                 if (!addr)
1455                         return -EINVAL;
1456
1457                 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1458                         if (kretprobe_blacklist[i].addr == addr)
1459                                 return -EINVAL;
1460                 }
1461         }
1462
1463         rp->kp.pre_handler = pre_handler_kretprobe;
1464         rp->kp.post_handler = NULL;
1465         rp->kp.fault_handler = NULL;
1466         rp->kp.break_handler = NULL;
1467
1468         /* Pre-allocate memory for max kretprobe instances */
1469         if (rp->maxactive <= 0) {
1470 #ifdef CONFIG_PREEMPT
1471                 rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus());
1472 #else
1473                 rp->maxactive = num_possible_cpus();
1474 #endif
1475         }
1476         spin_lock_init(&rp->lock);
1477         INIT_HLIST_HEAD(&rp->free_instances);
1478         for (i = 0; i < rp->maxactive; i++) {
1479                 inst = kmalloc(sizeof(struct kretprobe_instance) +
1480                                rp->data_size, GFP_KERNEL);
1481                 if (inst == NULL) {
1482                         free_rp_inst(rp);
1483                         return -ENOMEM;
1484                 }
1485                 INIT_HLIST_NODE(&inst->hlist);
1486                 hlist_add_head(&inst->hlist, &rp->free_instances);
1487         }
1488
1489         rp->nmissed = 0;
1490         /* Establish function entry probe point */
1491         ret = register_kprobe(&rp->kp);
1492         if (ret != 0)
1493                 free_rp_inst(rp);
1494         return ret;
1495 }
1496 EXPORT_SYMBOL_GPL(register_kretprobe);
1497
1498 int __kprobes register_kretprobes(struct kretprobe **rps, int num)
1499 {
1500         int ret = 0, i;
1501
1502         if (num <= 0)
1503                 return -EINVAL;
1504         for (i = 0; i < num; i++) {
1505                 ret = register_kretprobe(rps[i]);
1506                 if (ret < 0) {
1507                         if (i > 0)
1508                                 unregister_kretprobes(rps, i);
1509                         break;
1510                 }
1511         }
1512         return ret;
1513 }
1514 EXPORT_SYMBOL_GPL(register_kretprobes);
1515
1516 void __kprobes unregister_kretprobe(struct kretprobe *rp)
1517 {
1518         unregister_kretprobes(&rp, 1);
1519 }
1520 EXPORT_SYMBOL_GPL(unregister_kretprobe);
1521
1522 void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1523 {
1524         int i;
1525
1526         if (num <= 0)
1527                 return;
1528         mutex_lock(&kprobe_mutex);
1529         for (i = 0; i < num; i++)
1530                 if (__unregister_kprobe_top(&rps[i]->kp) < 0)
1531                         rps[i]->kp.addr = NULL;
1532         mutex_unlock(&kprobe_mutex);
1533
1534         synchronize_sched();
1535         for (i = 0; i < num; i++) {
1536                 if (rps[i]->kp.addr) {
1537                         __unregister_kprobe_bottom(&rps[i]->kp);
1538                         cleanup_rp_inst(rps[i]);
1539                 }
1540         }
1541 }
1542 EXPORT_SYMBOL_GPL(unregister_kretprobes);
1543
1544 #else /* CONFIG_KRETPROBES */
1545 int __kprobes register_kretprobe(struct kretprobe *rp)
1546 {
1547         return -ENOSYS;
1548 }
1549 EXPORT_SYMBOL_GPL(register_kretprobe);
1550
1551 int __kprobes register_kretprobes(struct kretprobe **rps, int num)
1552 {
1553         return -ENOSYS;
1554 }
1555 EXPORT_SYMBOL_GPL(register_kretprobes);
1556
1557 void __kprobes unregister_kretprobe(struct kretprobe *rp)
1558 {
1559 }
1560 EXPORT_SYMBOL_GPL(unregister_kretprobe);
1561
1562 void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1563 {
1564 }
1565 EXPORT_SYMBOL_GPL(unregister_kretprobes);
1566
1567 static int __kprobes pre_handler_kretprobe(struct kprobe *p,
1568                                            struct pt_regs *regs)
1569 {
1570         return 0;
1571 }
1572
1573 #endif /* CONFIG_KRETPROBES */
1574
1575 /* Set the kprobe gone and remove its instruction buffer. */
1576 static void __kprobes kill_kprobe(struct kprobe *p)
1577 {
1578         struct kprobe *kp;
1579
1580         p->flags |= KPROBE_FLAG_GONE;
1581         if (kprobe_aggrprobe(p)) {
1582                 /*
1583                  * If this is an aggr_kprobe, we have to list all the
1584                  * chained probes and mark them GONE.
1585                  */
1586                 list_for_each_entry_rcu(kp, &p->list, list)
1587                         kp->flags |= KPROBE_FLAG_GONE;
1588                 p->post_handler = NULL;
1589                 p->break_handler = NULL;
1590                 kill_optimized_kprobe(p);
1591         }
1592         /*
1593          * Here, we can remove insn_slot safely, because no thread calls
1594          * the original probed function (which will be freed soon) any more.
1595          */
1596         arch_remove_kprobe(p);
1597 }
1598
1599 /* Disable one kprobe */
1600 int __kprobes disable_kprobe(struct kprobe *kp)
1601 {
1602         int ret = 0;
1603         struct kprobe *p;
1604
1605         mutex_lock(&kprobe_mutex);
1606
1607         /* Check whether specified probe is valid. */
1608         p = __get_valid_kprobe(kp);
1609         if (unlikely(p == NULL)) {
1610                 ret = -EINVAL;
1611                 goto out;
1612         }
1613
1614         /* If the probe is already disabled (or gone), just return */
1615         if (kprobe_disabled(kp))
1616                 goto out;
1617
1618         kp->flags |= KPROBE_FLAG_DISABLED;
1619         if (p != kp)
1620                 /* When kp != p, p is always enabled. */
1621                 try_to_disable_aggr_kprobe(p);
1622
1623         if (!kprobes_all_disarmed && kprobe_disabled(p))
1624                 disarm_kprobe(p);
1625 out:
1626         mutex_unlock(&kprobe_mutex);
1627         return ret;
1628 }
1629 EXPORT_SYMBOL_GPL(disable_kprobe);
1630
1631 /* Enable one kprobe */
1632 int __kprobes enable_kprobe(struct kprobe *kp)
1633 {
1634         int ret = 0;
1635         struct kprobe *p;
1636
1637         mutex_lock(&kprobe_mutex);
1638
1639         /* Check whether specified probe is valid. */
1640         p = __get_valid_kprobe(kp);
1641         if (unlikely(p == NULL)) {
1642                 ret = -EINVAL;
1643                 goto out;
1644         }
1645
1646         if (kprobe_gone(kp)) {
1647                 /* This kprobe has gone, we couldn't enable it. */
1648                 ret = -EINVAL;
1649                 goto out;
1650         }
1651
1652         if (p != kp)
1653                 kp->flags &= ~KPROBE_FLAG_DISABLED;
1654
1655         if (!kprobes_all_disarmed && kprobe_disabled(p)) {
1656                 p->flags &= ~KPROBE_FLAG_DISABLED;
1657                 arm_kprobe(p);
1658         }
1659 out:
1660         mutex_unlock(&kprobe_mutex);
1661         return ret;
1662 }
1663 EXPORT_SYMBOL_GPL(enable_kprobe);
1664
1665 void __kprobes dump_kprobe(struct kprobe *kp)
1666 {
1667         printk(KERN_WARNING "Dumping kprobe:\n");
1668         printk(KERN_WARNING "Name: %s\nAddress: %p\nOffset: %x\n",
1669                kp->symbol_name, kp->addr, kp->offset);
1670 }
1671
1672 /* Module notifier call back, checking kprobes on the module */
1673 static int __kprobes kprobes_module_callback(struct notifier_block *nb,
1674                                              unsigned long val, void *data)
1675 {
1676         struct module *mod = data;
1677         struct hlist_head *head;
1678         struct hlist_node *node;
1679         struct kprobe *p;
1680         unsigned int i;
1681         int checkcore = (val == MODULE_STATE_GOING);
1682
1683         if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
1684                 return NOTIFY_DONE;
1685
1686         /*
1687          * When MODULE_STATE_GOING was notified, both of module .text and
1688          * .init.text sections would be freed. When MODULE_STATE_LIVE was
1689          * notified, only .init.text section would be freed. We need to
1690          * disable kprobes which have been inserted in the sections.
1691          */
1692         mutex_lock(&kprobe_mutex);
1693         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1694                 head = &kprobe_table[i];
1695                 hlist_for_each_entry_rcu(p, node, head, hlist)
1696                         if (within_module_init((unsigned long)p->addr, mod) ||
1697                             (checkcore &&
1698                              within_module_core((unsigned long)p->addr, mod))) {
1699                                 /*
1700                                  * The vaddr this probe is installed will soon
1701                                  * be vfreed buy not synced to disk. Hence,
1702                                  * disarming the breakpoint isn't needed.
1703                                  */
1704                                 kill_kprobe(p);
1705                         }
1706         }
1707         mutex_unlock(&kprobe_mutex);
1708         return NOTIFY_DONE;
1709 }
1710
1711 static struct notifier_block kprobe_module_nb = {
1712         .notifier_call = kprobes_module_callback,
1713         .priority = 0
1714 };
1715
1716 static int __init init_kprobes(void)
1717 {
1718         int i, err = 0;
1719         unsigned long offset = 0, size = 0;
1720         char *modname, namebuf[128];
1721         const char *symbol_name;
1722         void *addr;
1723         struct kprobe_blackpoint *kb;
1724
1725         /* FIXME allocate the probe table, currently defined statically */
1726         /* initialize all list heads */
1727         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1728                 INIT_HLIST_HEAD(&kprobe_table[i]);
1729                 INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
1730                 spin_lock_init(&(kretprobe_table_locks[i].lock));
1731         }
1732
1733         /*
1734          * Lookup and populate the kprobe_blacklist.
1735          *
1736          * Unlike the kretprobe blacklist, we'll need to determine
1737          * the range of addresses that belong to the said functions,
1738          * since a kprobe need not necessarily be at the beginning
1739          * of a function.
1740          */
1741         for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
1742                 kprobe_lookup_name(kb->name, addr);
1743                 if (!addr)
1744                         continue;
1745
1746                 kb->start_addr = (unsigned long)addr;
1747                 symbol_name = kallsyms_lookup(kb->start_addr,
1748                                 &size, &offset, &modname, namebuf);
1749                 if (!symbol_name)
1750                         kb->range = 0;
1751                 else
1752                         kb->range = size;
1753         }
1754
1755         if (kretprobe_blacklist_size) {
1756                 /* lookup the function address from its name */
1757                 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1758                         kprobe_lookup_name(kretprobe_blacklist[i].name,
1759                                            kretprobe_blacklist[i].addr);
1760                         if (!kretprobe_blacklist[i].addr)
1761                                 printk("kretprobe: lookup failed: %s\n",
1762                                        kretprobe_blacklist[i].name);
1763                 }
1764         }
1765
1766 #if defined(CONFIG_OPTPROBES)
1767 #if defined(__ARCH_WANT_KPROBES_INSN_SLOT)
1768         /* Init kprobe_optinsn_slots */
1769         kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
1770 #endif
1771         /* By default, kprobes can be optimized */
1772         kprobes_allow_optimization = true;
1773 #endif
1774
1775         /* By default, kprobes are armed */
1776         kprobes_all_disarmed = false;
1777
1778         err = arch_init_kprobes();
1779         if (!err)
1780                 err = register_die_notifier(&kprobe_exceptions_nb);
1781         if (!err)
1782                 err = register_module_notifier(&kprobe_module_nb);
1783
1784         kprobes_initialized = (err == 0);
1785
1786         if (!err)
1787                 init_test_probes();
1788         return err;
1789 }
1790
1791 #ifdef CONFIG_DEBUG_FS
1792 static void __kprobes report_probe(struct seq_file *pi, struct kprobe *p,
1793                 const char *sym, int offset, char *modname, struct kprobe *pp)
1794 {
1795         char *kprobe_type;
1796
1797         if (p->pre_handler == pre_handler_kretprobe)
1798                 kprobe_type = "r";
1799         else if (p->pre_handler == setjmp_pre_handler)
1800                 kprobe_type = "j";
1801         else
1802                 kprobe_type = "k";
1803
1804         if (sym)
1805                 seq_printf(pi, "%p  %s  %s+0x%x  %s ",
1806                         p->addr, kprobe_type, sym, offset,
1807                         (modname ? modname : " "));
1808         else
1809                 seq_printf(pi, "%p  %s  %p ",
1810                         p->addr, kprobe_type, p->addr);
1811
1812         if (!pp)
1813                 pp = p;
1814         seq_printf(pi, "%s%s%s\n",
1815                 (kprobe_gone(p) ? "[GONE]" : ""),
1816                 ((kprobe_disabled(p) && !kprobe_gone(p)) ?  "[DISABLED]" : ""),
1817                 (kprobe_optimized(pp) ? "[OPTIMIZED]" : ""));
1818 }
1819
1820 static void __kprobes *kprobe_seq_start(struct seq_file *f, loff_t *pos)
1821 {
1822         return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
1823 }
1824
1825 static void __kprobes *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
1826 {
1827         (*pos)++;
1828         if (*pos >= KPROBE_TABLE_SIZE)
1829                 return NULL;
1830         return pos;
1831 }
1832
1833 static void __kprobes kprobe_seq_stop(struct seq_file *f, void *v)
1834 {
1835         /* Nothing to do */
1836 }
1837
1838 static int __kprobes show_kprobe_addr(struct seq_file *pi, void *v)
1839 {
1840         struct hlist_head *head;
1841         struct hlist_node *node;
1842         struct kprobe *p, *kp;
1843         const char *sym = NULL;
1844         unsigned int i = *(loff_t *) v;
1845         unsigned long offset = 0;
1846         char *modname, namebuf[128];
1847
1848         head = &kprobe_table[i];
1849         preempt_disable();
1850         hlist_for_each_entry_rcu(p, node, head, hlist) {
1851                 sym = kallsyms_lookup((unsigned long)p->addr, NULL,
1852                                         &offset, &modname, namebuf);
1853                 if (kprobe_aggrprobe(p)) {
1854                         list_for_each_entry_rcu(kp, &p->list, list)
1855                                 report_probe(pi, kp, sym, offset, modname, p);
1856                 } else
1857                         report_probe(pi, p, sym, offset, modname, NULL);
1858         }
1859         preempt_enable();
1860         return 0;
1861 }
1862
1863 static const struct seq_operations kprobes_seq_ops = {
1864         .start = kprobe_seq_start,
1865         .next  = kprobe_seq_next,
1866         .stop  = kprobe_seq_stop,
1867         .show  = show_kprobe_addr
1868 };
1869
1870 static int __kprobes kprobes_open(struct inode *inode, struct file *filp)
1871 {
1872         return seq_open(filp, &kprobes_seq_ops);
1873 }
1874
1875 static const struct file_operations debugfs_kprobes_operations = {
1876         .open           = kprobes_open,
1877         .read           = seq_read,
1878         .llseek         = seq_lseek,
1879         .release        = seq_release,
1880 };
1881
1882 static void __kprobes arm_all_kprobes(void)
1883 {
1884         struct hlist_head *head;
1885         struct hlist_node *node;
1886         struct kprobe *p;
1887         unsigned int i;
1888
1889         mutex_lock(&kprobe_mutex);
1890
1891         /* If kprobes are armed, just return */
1892         if (!kprobes_all_disarmed)
1893                 goto already_enabled;
1894
1895         /* Arming kprobes doesn't optimize kprobe itself */
1896         mutex_lock(&text_mutex);
1897         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1898                 head = &kprobe_table[i];
1899                 hlist_for_each_entry_rcu(p, node, head, hlist)
1900                         if (!kprobe_disabled(p))
1901                                 __arm_kprobe(p);
1902         }
1903         mutex_unlock(&text_mutex);
1904
1905         kprobes_all_disarmed = false;
1906         printk(KERN_INFO "Kprobes globally enabled\n");
1907
1908 already_enabled:
1909         mutex_unlock(&kprobe_mutex);
1910         return;
1911 }
1912
1913 static void __kprobes disarm_all_kprobes(void)
1914 {
1915         struct hlist_head *head;
1916         struct hlist_node *node;
1917         struct kprobe *p;
1918         unsigned int i;
1919
1920         mutex_lock(&kprobe_mutex);
1921
1922         /* If kprobes are already disarmed, just return */
1923         if (kprobes_all_disarmed)
1924                 goto already_disabled;
1925
1926         kprobes_all_disarmed = true;
1927         printk(KERN_INFO "Kprobes globally disabled\n");
1928
1929         /*
1930          * Here we call get_online_cpus() for avoiding text_mutex deadlock,
1931          * because disarming may also unoptimize kprobes.
1932          */
1933         get_online_cpus();
1934         mutex_lock(&text_mutex);
1935         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1936                 head = &kprobe_table[i];
1937                 hlist_for_each_entry_rcu(p, node, head, hlist) {
1938                         if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p))
1939                                 __disarm_kprobe(p);
1940                 }
1941         }
1942
1943         mutex_unlock(&text_mutex);
1944         put_online_cpus();
1945         mutex_unlock(&kprobe_mutex);
1946         /* Allow all currently running kprobes to complete */
1947         synchronize_sched();
1948         return;
1949
1950 already_disabled:
1951         mutex_unlock(&kprobe_mutex);
1952         return;
1953 }
1954
1955 /*
1956  * XXX: The debugfs bool file interface doesn't allow for callbacks
1957  * when the bool state is switched. We can reuse that facility when
1958  * available
1959  */
1960 static ssize_t read_enabled_file_bool(struct file *file,
1961                char __user *user_buf, size_t count, loff_t *ppos)
1962 {
1963         char buf[3];
1964
1965         if (!kprobes_all_disarmed)
1966                 buf[0] = '1';
1967         else
1968                 buf[0] = '0';
1969         buf[1] = '\n';
1970         buf[2] = 0x00;
1971         return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
1972 }
1973
1974 static ssize_t write_enabled_file_bool(struct file *file,
1975                const char __user *user_buf, size_t count, loff_t *ppos)
1976 {
1977         char buf[32];
1978         int buf_size;
1979
1980         buf_size = min(count, (sizeof(buf)-1));
1981         if (copy_from_user(buf, user_buf, buf_size))
1982                 return -EFAULT;
1983
1984         switch (buf[0]) {
1985         case 'y':
1986         case 'Y':
1987         case '1':
1988                 arm_all_kprobes();
1989                 break;
1990         case 'n':
1991         case 'N':
1992         case '0':
1993                 disarm_all_kprobes();
1994                 break;
1995         }
1996
1997         return count;
1998 }
1999
2000 static const struct file_operations fops_kp = {
2001         .read =         read_enabled_file_bool,
2002         .write =        write_enabled_file_bool,
2003 };
2004
2005 static int __kprobes debugfs_kprobe_init(void)
2006 {
2007         struct dentry *dir, *file;
2008         unsigned int value = 1;
2009
2010         dir = debugfs_create_dir("kprobes", NULL);
2011         if (!dir)
2012                 return -ENOMEM;
2013
2014         file = debugfs_create_file("list", 0444, dir, NULL,
2015                                 &debugfs_kprobes_operations);
2016         if (!file) {
2017                 debugfs_remove(dir);
2018                 return -ENOMEM;
2019         }
2020
2021         file = debugfs_create_file("enabled", 0600, dir,
2022                                         &value, &fops_kp);
2023         if (!file) {
2024                 debugfs_remove(dir);
2025                 return -ENOMEM;
2026         }
2027
2028         return 0;
2029 }
2030
2031 late_initcall(debugfs_kprobe_init);
2032 #endif /* CONFIG_DEBUG_FS */
2033
2034 module_init(init_kprobes);
2035
2036 /* defined in arch/.../kernel/kprobes.c */
2037 EXPORT_SYMBOL_GPL(jprobe_return);