kernel/rcutorture.c

   1 /*
   2  * Read-Copy Update module-based torture test facility
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17  *
  18  * Copyright (C) IBM Corporation, 2005
  19  *
  20  * Authors: Paul E. McKenney <paulmck@us.ibm.com>
  21  *
  22  * See also:  Documentation/RCU/torture.txt
  23  */
  24 #include <linux/types.h>
  25 #include <linux/kernel.h>
  26 #include <linux/init.h>
  27 #include <linux/module.h>
  28 #include <linux/kthread.h>
  29 #include <linux/err.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/smp.h>
  32 #include <linux/rcupdate.h>
  33 #include <linux/interrupt.h>
  34 #include <linux/sched.h>
  35 #include <asm/atomic.h>
  36 #include <linux/bitops.h>
  37 #include <linux/module.h>
  38 #include <linux/completion.h>
  39 #include <linux/moduleparam.h>
  40 #include <linux/percpu.h>
  41 #include <linux/notifier.h>
  42 #include <linux/cpu.h>
  43 #include <linux/random.h>
  44 #include <linux/delay.h>
  45 #include <linux/byteorder/swabb.h>
  46 #include <linux/stat.h>
  47
  48 MODULE_LICENSE("GPL");
  49
  50 static int nreaders = -1;       /* # reader threads, defaults to 2*ncpus */
  51 static int stat_interval;       /* Interval between stats, in seconds. */
  52                                 /*  Defaults to "only at end of test". */
  53 static int verbose;             /* Print more debug info. */
  54 static int test_no_idle_hz;     /* Test RCU's support for tickless idle CPUs. */
  55 static int shuffle_interval = 5; /* Interval between shuffles (in sec)*/
  56 static char *torture_type = "rcu"; /* What to torture. */
  57
  58 module_param(nreaders, int, 0);
  59 MODULE_PARM_DESC(nreaders, "Number of RCU reader threads");
  60 module_param(stat_interval, int, 0);
  61 MODULE_PARM_DESC(stat_interval, "Number of seconds between stats printk()s");
  62 module_param(verbose, bool, 0);
  63 MODULE_PARM_DESC(verbose, "Enable verbose debugging printk()s");
  64 module_param(test_no_idle_hz, bool, 0);
  65 MODULE_PARM_DESC(test_no_idle_hz, "Test support for tickless idle CPUs");
  66 module_param(shuffle_interval, int, 0);
  67 MODULE_PARM_DESC(shuffle_interval, "Number of seconds between shuffles");
  68 module_param(torture_type, charp, 0);
  69 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh)");
  70
  71 #define TORTURE_FLAG "-torture:"
  72 #define PRINTK_STRING(s) \
  73         do { printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)
  74 #define VERBOSE_PRINTK_STRING(s) \
  75         do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)
  76 #define VERBOSE_PRINTK_ERRSTRING(s) \
  77         do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG "!!! " s "\n", torture_type); } while (0)
  78
  79 static char printk_buf[4096];
  80
  81 static int nrealreaders;
  82 static struct task_struct *writer_task;
  83 static struct task_struct **reader_tasks;
  84 static struct task_struct *stats_task;
  85 static struct task_struct *shuffler_task;
  86
  87 #define RCU_TORTURE_PIPE_LEN 10
  88
  89 struct rcu_torture {
  90         struct rcu_head rtort_rcu;
  91         int rtort_pipe_count;
  92         struct list_head rtort_free;
  93         int rtort_mbtest;
  94 };
  95
  96 static int fullstop = 0;        /* stop generating callbacks at test end. */
  97 static LIST_HEAD(rcu_torture_freelist);
  98 static struct rcu_torture *rcu_torture_current = NULL;
  99 static long rcu_torture_current_version = 0;
 100 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];
 101 static DEFINE_SPINLOCK(rcu_torture_lock);
 102 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count) =
 103         { 0 };
 104 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch) =
 105         { 0 };
 106 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
 107 atomic_t n_rcu_torture_alloc;
 108 atomic_t n_rcu_torture_alloc_fail;
 109 atomic_t n_rcu_torture_free;
 110 atomic_t n_rcu_torture_mberror;
 111 atomic_t n_rcu_torture_error;
 112
 113 /*
 114  * Allocate an element from the rcu_tortures pool.
 115  */
 116 static struct rcu_torture *
 117 rcu_torture_alloc(void)
 118 {
 119         struct list_head *p;
 120
 121         spin_lock_bh(&rcu_torture_lock);
 122         if (list_empty(&rcu_torture_freelist)) {
 123                 atomic_inc(&n_rcu_torture_alloc_fail);
 124                 spin_unlock_bh(&rcu_torture_lock);
 125                 return NULL;
 126         }
 127         atomic_inc(&n_rcu_torture_alloc);
 128         p = rcu_torture_freelist.next;
 129         list_del_init(p);
 130         spin_unlock_bh(&rcu_torture_lock);
 131         return container_of(p, struct rcu_torture, rtort_free);
 132 }
 133
 134 /*
 135  * Free an element to the rcu_tortures pool.
 136  */
 137 static void
 138 rcu_torture_free(struct rcu_torture *p)
 139 {
 140         atomic_inc(&n_rcu_torture_free);
 141         spin_lock_bh(&rcu_torture_lock);
 142         list_add_tail(&p->rtort_free, &rcu_torture_freelist);
 143         spin_unlock_bh(&rcu_torture_lock);
 144 }
 145
 146 struct rcu_random_state {
 147         unsigned long rrs_state;
 148         unsigned long rrs_count;
 149 };
 150
 151 #define RCU_RANDOM_MULT 39916801  /* prime */
 152 #define RCU_RANDOM_ADD  479001701 /* prime */
 153 #define RCU_RANDOM_REFRESH 10000
 154
 155 #define DEFINE_RCU_RANDOM(name) struct rcu_random_state name = { 0, 0 }
 156
 157 /*
 158  * Crude but fast random-number generator.  Uses a linear congruential
 159  * generator, with occasional help from get_random_bytes().
 160  */
 161 static long
 162 rcu_random(struct rcu_random_state *rrsp)
 163 {
 164         long refresh;
 165
 166         if (--rrsp->rrs_count < 0) {
 167                 get_random_bytes(&refresh, sizeof(refresh));
 168                 rrsp->rrs_state += refresh;
 169                 rrsp->rrs_count = RCU_RANDOM_REFRESH;
 170         }
 171         rrsp->rrs_state = rrsp->rrs_state * RCU_RANDOM_MULT + RCU_RANDOM_ADD;
 172         return swahw32(rrsp->rrs_state);
 173 }
 174
 175 /*
 176  * Operations vector for selecting different types of tests.
 177  */
 178
 179 struct rcu_torture_ops {
 180         void (*init)(void);
 181         void (*cleanup)(void);
 182         int (*readlock)(void);
 183         void (*readunlock)(int idx);
 184         int (*completed)(void);
 185         void (*deferredfree)(struct rcu_torture *p);
 186         int (*stats)(char *page);
 187         char *name;
 188 };
 189 static struct rcu_torture_ops *cur_ops = NULL;
 190
 191 /*
 192  * Definitions for rcu torture testing.
 193  */
 194
 195 static int rcu_torture_read_lock(void) __acquires(RCU)
 196 {
 197         rcu_read_lock();
 198         return 0;
 199 }
 200
 201 static void rcu_torture_read_unlock(int idx) __releases(RCU)
 202 {
 203         rcu_read_unlock();
 204 }
 205
 206 static int rcu_torture_completed(void)
 207 {
 208         return rcu_batches_completed();
 209 }
 210
 211 static void
 212 rcu_torture_cb(struct rcu_head *p)
 213 {
 214         int i;
 215         struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu);
 216
 217         if (fullstop) {
 218                 /* Test is ending, just drop callbacks on the floor. */
 219                 /* The next initialization will pick up the pieces. */
 220                 return;
 221         }
 222         i = rp->rtort_pipe_count;
 223         if (i > RCU_TORTURE_PIPE_LEN)
 224                 i = RCU_TORTURE_PIPE_LEN;
 225         atomic_inc(&rcu_torture_wcount[i]);
 226         if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
 227                 rp->rtort_mbtest = 0;
 228                 rcu_torture_free(rp);
 229         } else
 230                 cur_ops->deferredfree(rp);
 231 }
 232
 233 static void rcu_torture_deferred_free(struct rcu_torture *p)
 234 {
 235         call_rcu(&p->rtort_rcu, rcu_torture_cb);
 236 }
 237
 238 static struct rcu_torture_ops rcu_ops = {
 239         .init = NULL,
 240         .cleanup = NULL,
 241         .readlock = rcu_torture_read_lock,
 242         .readunlock = rcu_torture_read_unlock,
 243         .completed = rcu_torture_completed,
 244         .deferredfree = rcu_torture_deferred_free,
 245         .stats = NULL,
 246         .name = "rcu"
 247 };
 248
 249 /*
 250  * Definitions for rcu_bh torture testing.
 251  */
 252
 253 static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH)
 254 {
 255         rcu_read_lock_bh();
 256         return 0;
 257 }
 258
 259 static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH)
 260 {
 261         rcu_read_unlock_bh();
 262 }
 263
 264 static int rcu_bh_torture_completed(void)
 265 {
 266         return rcu_batches_completed_bh();
 267 }
 268
 269 static void rcu_bh_torture_deferred_free(struct rcu_torture *p)
 270 {
 271         call_rcu_bh(&p->rtort_rcu, rcu_torture_cb);
 272 }
 273
 274 static struct rcu_torture_ops rcu_bh_ops = {
 275         .init = NULL,
 276         .cleanup = NULL,
 277         .readlock = rcu_bh_torture_read_lock,
 278         .readunlock = rcu_bh_torture_read_unlock,
 279         .completed = rcu_bh_torture_completed,
 280         .deferredfree = rcu_bh_torture_deferred_free,
 281         .stats = NULL,
 282         .name = "rcu_bh"
 283 };
 284
 285 static struct rcu_torture_ops *torture_ops[] =
 286         { &rcu_ops, &rcu_bh_ops, NULL };
 287
 288 /*
 289  * RCU torture writer kthread.  Repeatedly substitutes a new structure
 290  * for that pointed to by rcu_torture_current, freeing the old structure
 291  * after a series of grace periods (the "pipeline").
 292  */
 293 static int
 294 rcu_torture_writer(void *arg)
 295 {
 296         int i;
 297         long oldbatch = rcu_batches_completed();
 298         struct rcu_torture *rp;
 299         struct rcu_torture *old_rp;
 300         static DEFINE_RCU_RANDOM(rand);
 301
 302         VERBOSE_PRINTK_STRING("rcu_torture_writer task started");
 303         set_user_nice(current, 19);
 304
 305         do {
 306                 schedule_timeout_uninterruptible(1);
 307                 if ((rp = rcu_torture_alloc()) == NULL)
 308                         continue;
 309                 rp->rtort_pipe_count = 0;
 310                 udelay(rcu_random(&rand) & 0x3ff);
 311                 old_rp = rcu_torture_current;
 312                 rp->rtort_mbtest = 1;
 313                 rcu_assign_pointer(rcu_torture_current, rp);
 314                 smp_wmb();
 315                 if (old_rp != NULL) {
 316                         i = old_rp->rtort_pipe_count;
 317                         if (i > RCU_TORTURE_PIPE_LEN)
 318                                 i = RCU_TORTURE_PIPE_LEN;
 319                         atomic_inc(&rcu_torture_wcount[i]);
 320                         old_rp->rtort_pipe_count++;
 321                         cur_ops->deferredfree(old_rp);
 322                 }
 323                 rcu_torture_current_version++;
 324                 oldbatch = cur_ops->completed();
 325         } while (!kthread_should_stop() && !fullstop);
 326         VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping");
 327         while (!kthread_should_stop())
 328                 schedule_timeout_uninterruptible(1);
 329         return 0;
 330 }
 331
 332 /*
 333  * RCU torture reader kthread.  Repeatedly dereferences rcu_torture_current,
 334  * incrementing the corresponding element of the pipeline array.  The
 335  * counter in the element should never be greater than 1, otherwise, the
 336  * RCU implementation is broken.
 337  */
 338 static int
 339 rcu_torture_reader(void *arg)
 340 {
 341         int completed;
 342         int idx;
 343         DEFINE_RCU_RANDOM(rand);
 344         struct rcu_torture *p;
 345         int pipe_count;
 346
 347         VERBOSE_PRINTK_STRING("rcu_torture_reader task started");
 348         set_user_nice(current, 19);
 349
 350         do {
 351                 idx = cur_ops->readlock();
 352                 completed = cur_ops->completed();
 353                 p = rcu_dereference(rcu_torture_current);
 354                 if (p == NULL) {
 355                         /* Wait for rcu_torture_writer to get underway */
 356                         cur_ops->readunlock(idx);
 357                         schedule_timeout_interruptible(HZ);
 358                         continue;
 359                 }
 360                 if (p->rtort_mbtest == 0)
 361                         atomic_inc(&n_rcu_torture_mberror);
 362                 udelay(rcu_random(&rand) & 0x7f);
 363                 preempt_disable();
 364                 pipe_count = p->rtort_pipe_count;
 365                 if (pipe_count > RCU_TORTURE_PIPE_LEN) {
 366                         /* Should not happen, but... */
 367                         pipe_count = RCU_TORTURE_PIPE_LEN;
 368                 }
 369                 ++__get_cpu_var(rcu_torture_count)[pipe_count];
 370                 completed = cur_ops->completed() - completed;
 371                 if (completed > RCU_TORTURE_PIPE_LEN) {
 372                         /* Should not happen, but... */
 373                         completed = RCU_TORTURE_PIPE_LEN;
 374                 }
 375                 ++__get_cpu_var(rcu_torture_batch)[completed];
 376                 preempt_enable();
 377                 cur_ops->readunlock(idx);
 378                 schedule();
 379         } while (!kthread_should_stop() && !fullstop);
 380         VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping");
 381         while (!kthread_should_stop())
 382                 schedule_timeout_uninterruptible(1);
 383         return 0;
 384 }
 385
 386 /*
 387  * Create an RCU-torture statistics message in the specified buffer.
 388  */
 389 static int
 390 rcu_torture_printk(char *page)
 391 {
 392         int cnt = 0;
 393         int cpu;
 394         int i;
 395         long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
 396         long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
 397
 398         for_each_possible_cpu(cpu) {
 399                 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
 400                         pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i];
 401                         batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i];
 402                 }
 403         }
 404         for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) {
 405                 if (pipesummary[i] != 0)
 406                         break;
 407         }
 408         cnt += sprintf(&page[cnt], "%s%s ", torture_type, TORTURE_FLAG);
 409         cnt += sprintf(&page[cnt],
 410                        "rtc: %p ver: %ld tfle: %d rta: %d rtaf: %d rtf: %d "
 411                        "rtmbe: %d",
 412                        rcu_torture_current,
 413                        rcu_torture_current_version,
 414                        list_empty(&rcu_torture_freelist),
 415                        atomic_read(&n_rcu_torture_alloc),
 416                        atomic_read(&n_rcu_torture_alloc_fail),
 417                        atomic_read(&n_rcu_torture_free),
 418                        atomic_read(&n_rcu_torture_mberror));
 419         if (atomic_read(&n_rcu_torture_mberror) != 0)
 420                 cnt += sprintf(&page[cnt], " !!!");
 421         cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
 422         if (i > 1) {
 423                 cnt += sprintf(&page[cnt], "!!! ");
 424                 atomic_inc(&n_rcu_torture_error);
 425         }
 426         cnt += sprintf(&page[cnt], "Reader Pipe: ");
 427         for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
 428                 cnt += sprintf(&page[cnt], " %ld", pipesummary[i]);
 429         cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
 430         cnt += sprintf(&page[cnt], "Reader Batch: ");
 431         for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
 432                 cnt += sprintf(&page[cnt], " %ld", batchsummary[i]);
 433         cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
 434         cnt += sprintf(&page[cnt], "Free-Block Circulation: ");
 435         for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
 436                 cnt += sprintf(&page[cnt], " %d",
 437                                atomic_read(&rcu_torture_wcount[i]));
 438         }
 439         cnt += sprintf(&page[cnt], "\n");
 440         if (cur_ops->stats != NULL)
 441                 cnt += cur_ops->stats(&page[cnt]);
 442         return cnt;
 443 }
 444
 445 /*
 446  * Print torture statistics.  Caller must ensure that there is only
 447  * one call to this function at a given time!!!  This is normally
 448  * accomplished by relying on the module system to only have one copy
 449  * of the module loaded, and then by giving the rcu_torture_stats
 450  * kthread full control (or the init/cleanup functions when rcu_torture_stats
 451  * thread is not running).
 452  */
 453 static void
 454 rcu_torture_stats_print(void)
 455 {
 456         int cnt;
 457
 458         cnt = rcu_torture_printk(printk_buf);
 459         printk(KERN_ALERT "%s", printk_buf);
 460 }
 461
 462 /*
 463  * Periodically prints torture statistics, if periodic statistics printing
 464  * was specified via the stat_interval module parameter.
 465  *
 466  * No need to worry about fullstop here, since this one doesn't reference
 467  * volatile state or register callbacks.
 468  */
 469 static int
 470 rcu_torture_stats(void *arg)
 471 {
 472         VERBOSE_PRINTK_STRING("rcu_torture_stats task started");
 473         do {
 474                 schedule_timeout_interruptible(stat_interval * HZ);
 475                 rcu_torture_stats_print();
 476         } while (!kthread_should_stop());
 477         VERBOSE_PRINTK_STRING("rcu_torture_stats task stopping");
 478         return 0;
 479 }
 480
 481 static int rcu_idle_cpu;        /* Force all torture tasks off this CPU */
 482
 483 /* Shuffle tasks such that we allow @rcu_idle_cpu to become idle. A special case
 484  * is when @rcu_idle_cpu = -1, when we allow the tasks to run on all CPUs.
 485  */
 486 void rcu_torture_shuffle_tasks(void)
 487 {
 488         cpumask_t tmp_mask = CPU_MASK_ALL;
 489         int i;
 490
 491         lock_cpu_hotplug();
 492
 493         /* No point in shuffling if there is only one online CPU (ex: UP) */
 494         if (num_online_cpus() == 1) {
 495                 unlock_cpu_hotplug();
 496                 return;
 497         }
 498
 499         if (rcu_idle_cpu != -1)
 500                 cpu_clear(rcu_idle_cpu, tmp_mask);
 501
 502         set_cpus_allowed(current, tmp_mask);
 503
 504         if (reader_tasks != NULL) {
 505                 for (i = 0; i < nrealreaders; i++)
 506                         if (reader_tasks[i])
 507                                 set_cpus_allowed(reader_tasks[i], tmp_mask);
 508         }
 509
 510         if (writer_task)
 511                 set_cpus_allowed(writer_task, tmp_mask);
 512
 513         if (stats_task)
 514                 set_cpus_allowed(stats_task, tmp_mask);
 515
 516         if (rcu_idle_cpu == -1)
 517                 rcu_idle_cpu = num_online_cpus() - 1;
 518         else
 519                 rcu_idle_cpu--;
 520
 521         unlock_cpu_hotplug();
 522 }
 523
 524 /* Shuffle tasks across CPUs, with the intent of allowing each CPU in the
 525  * system to become idle at a time and cut off its timer ticks. This is meant
 526  * to test the support for such tickless idle CPU in RCU.
 527  */
 528 static int
 529 rcu_torture_shuffle(void *arg)
 530 {
 531         VERBOSE_PRINTK_STRING("rcu_torture_shuffle task started");
 532         do {
 533                 schedule_timeout_interruptible(shuffle_interval * HZ);
 534                 rcu_torture_shuffle_tasks();
 535         } while (!kthread_should_stop());
 536         VERBOSE_PRINTK_STRING("rcu_torture_shuffle task stopping");
 537         return 0;
 538 }
 539
 540 static inline void
 541 rcu_torture_print_module_parms(char *tag)
 542 {
 543         printk(KERN_ALERT "%s" TORTURE_FLAG "--- %s: nreaders=%d "
 544                 "stat_interval=%d verbose=%d test_no_idle_hz=%d "
 545                 "shuffle_interval = %d\n",
 546                 torture_type, tag, nrealreaders, stat_interval, verbose,
 547                 test_no_idle_hz, shuffle_interval);
 548 }
 549
 550 static void
 551 rcu_torture_cleanup(void)
 552 {
 553         int i;
 554
 555         fullstop = 1;
 556         if (shuffler_task != NULL) {
 557                 VERBOSE_PRINTK_STRING("Stopping rcu_torture_shuffle task");
 558                 kthread_stop(shuffler_task);
 559         }
 560         shuffler_task = NULL;
 561
 562         if (writer_task != NULL) {
 563                 VERBOSE_PRINTK_STRING("Stopping rcu_torture_writer task");
 564                 kthread_stop(writer_task);
 565         }
 566         writer_task = NULL;
 567
 568         if (reader_tasks != NULL) {
 569                 for (i = 0; i < nrealreaders; i++) {
 570                         if (reader_tasks[i] != NULL) {
 571                                 VERBOSE_PRINTK_STRING(
 572                                         "Stopping rcu_torture_reader task");
 573                                 kthread_stop(reader_tasks[i]);
 574                         }
 575                         reader_tasks[i] = NULL;
 576                 }
 577                 kfree(reader_tasks);
 578                 reader_tasks = NULL;
 579         }
 580         rcu_torture_current = NULL;
 581
 582         if (stats_task != NULL) {
 583                 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stats task");
 584                 kthread_stop(stats_task);
 585         }
 586         stats_task = NULL;
 587
 588         /* Wait for all RCU callbacks to fire.  */
 589         rcu_barrier();
 590
 591         rcu_torture_stats_print();  /* -After- the stats thread is stopped! */
 592
 593         if (cur_ops->cleanup != NULL)
 594                 cur_ops->cleanup();
 595         if (atomic_read(&n_rcu_torture_error))
 596                 rcu_torture_print_module_parms("End of test: FAILURE");
 597         else
 598                 rcu_torture_print_module_parms("End of test: SUCCESS");
 599 }
 600
 601 static int
 602 rcu_torture_init(void)
 603 {
 604         int i;
 605         int cpu;
 606         int firsterr = 0;
 607
 608         /* Process args and tell the world that the torturer is on the job. */
 609
 610         for (i = 0; cur_ops = torture_ops[i], cur_ops != NULL; i++) {
 611                 cur_ops = torture_ops[i];
 612                 if (strcmp(torture_type, cur_ops->name) == 0) {
 613                         break;
 614                 }
 615         }
 616         if (cur_ops == NULL) {
 617                 printk(KERN_ALERT "rcutorture: invalid torture type: \"%s\"\n",
 618                        torture_type);
 619                 return (-EINVAL);
 620         }
 621         if (cur_ops->init != NULL)
 622                 cur_ops->init(); /* no "goto unwind" prior to this point!!! */
 623
 624         if (nreaders >= 0)
 625                 nrealreaders = nreaders;
 626         else
 627                 nrealreaders = 2 * num_online_cpus();
 628         rcu_torture_print_module_parms("Start of test");
 629         fullstop = 0;
 630
 631         /* Set up the freelist. */
 632
 633         INIT_LIST_HEAD(&rcu_torture_freelist);
 634         for (i = 0; i < sizeof(rcu_tortures) / sizeof(rcu_tortures[0]); i++) {
 635                 rcu_tortures[i].rtort_mbtest = 0;
 636                 list_add_tail(&rcu_tortures[i].rtort_free,
 637                               &rcu_torture_freelist);
 638         }
 639
 640         /* Initialize the statistics so that each run gets its own numbers. */
 641
 642         rcu_torture_current = NULL;
 643         rcu_torture_current_version = 0;
 644         atomic_set(&n_rcu_torture_alloc, 0);
 645         atomic_set(&n_rcu_torture_alloc_fail, 0);
 646         atomic_set(&n_rcu_torture_free, 0);
 647         atomic_set(&n_rcu_torture_mberror, 0);
 648         atomic_set(&n_rcu_torture_error, 0);
 649         for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
 650                 atomic_set(&rcu_torture_wcount[i], 0);
 651         for_each_possible_cpu(cpu) {
 652                 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
 653                         per_cpu(rcu_torture_count, cpu)[i] = 0;
 654                         per_cpu(rcu_torture_batch, cpu)[i] = 0;
 655                 }
 656         }
 657
 658         /* Start up the kthreads. */
 659
 660         VERBOSE_PRINTK_STRING("Creating rcu_torture_writer task");
 661         writer_task = kthread_run(rcu_torture_writer, NULL,
 662                                   "rcu_torture_writer");
 663         if (IS_ERR(writer_task)) {
 664                 firsterr = PTR_ERR(writer_task);
 665                 VERBOSE_PRINTK_ERRSTRING("Failed to create writer");
 666                 writer_task = NULL;
 667                 goto unwind;
 668         }
 669         reader_tasks = kmalloc(nrealreaders * sizeof(reader_tasks[0]),
 670                                GFP_KERNEL);
 671         if (reader_tasks == NULL) {
 672                 VERBOSE_PRINTK_ERRSTRING("out of memory");
 673                 firsterr = -ENOMEM;
 674                 goto unwind;
 675         }
 676         for (i = 0; i < nrealreaders; i++) {
 677                 VERBOSE_PRINTK_STRING("Creating rcu_torture_reader task");
 678                 reader_tasks[i] = kthread_run(rcu_torture_reader, NULL,
 679                                               "rcu_torture_reader");
 680                 if (IS_ERR(reader_tasks[i])) {
 681                         firsterr = PTR_ERR(reader_tasks[i]);
 682                         VERBOSE_PRINTK_ERRSTRING("Failed to create reader");
 683                         reader_tasks[i] = NULL;
 684                         goto unwind;
 685                 }
 686         }
 687         if (stat_interval > 0) {
 688                 VERBOSE_PRINTK_STRING("Creating rcu_torture_stats task");
 689                 stats_task = kthread_run(rcu_torture_stats, NULL,
 690                                         "rcu_torture_stats");
 691                 if (IS_ERR(stats_task)) {
 692                         firsterr = PTR_ERR(stats_task);
 693                         VERBOSE_PRINTK_ERRSTRING("Failed to create stats");
 694                         stats_task = NULL;
 695                         goto unwind;
 696                 }
 697         }
 698         if (test_no_idle_hz) {
 699                 rcu_idle_cpu = num_online_cpus() - 1;
 700                 /* Create the shuffler thread */
 701                 shuffler_task = kthread_run(rcu_torture_shuffle, NULL,
 702                                           "rcu_torture_shuffle");
 703                 if (IS_ERR(shuffler_task)) {
 704                         firsterr = PTR_ERR(shuffler_task);
 705                         VERBOSE_PRINTK_ERRSTRING("Failed to create shuffler");
 706                         shuffler_task = NULL;
 707                         goto unwind;
 708                 }
 709         }
 710         return 0;
 711
 712 unwind:
 713         rcu_torture_cleanup();
 714         return firsterr;
 715 }
 716
 717 module_init(rcu_torture_init);
 718 module_exit(rcu_torture_cleanup);