arch/x86_64/kernel/process.c

   1 /*
   2  *  linux/arch/x86-64/kernel/process.c
   3  *
   4  *  Copyright (C) 1995  Linus Torvalds
   5  *
   6  *  Pentium III FXSR, SSE support
   7  *      Gareth Hughes <gareth@valinux.com>, May 2000
   8  *
   9  *  X86-64 port
  10  *      Andi Kleen.
  11  *
  12  *      CPU hotplug support - ashok.raj@intel.com
  13  */
  14
  15 /*
  16  * This file handles the architecture-dependent parts of process handling..
  17  */
  18
  19 #include <stdarg.h>
  20
  21 #include <linux/cpu.h>
  22 #include <linux/errno.h>
  23 #include <linux/sched.h>
  24 #include <linux/kernel.h>
  25 #include <linux/mm.h>
  26 #include <linux/elfcore.h>
  27 #include <linux/smp.h>
  28 #include <linux/slab.h>
  29 #include <linux/user.h>
  30 #include <linux/module.h>
  31 #include <linux/a.out.h>
  32 #include <linux/interrupt.h>
  33 #include <linux/delay.h>
  34 #include <linux/ptrace.h>
  35 #include <linux/utsname.h>
  36 #include <linux/random.h>
  37 #include <linux/notifier.h>
  38 #include <linux/kprobes.h>
  39
  40 #include <asm/uaccess.h>
  41 #include <asm/pgtable.h>
  42 #include <asm/system.h>
  43 #include <asm/io.h>
  44 #include <asm/processor.h>
  45 #include <asm/i387.h>
  46 #include <asm/mmu_context.h>
  47 #include <asm/pda.h>
  48 #include <asm/prctl.h>
  49 #include <asm/kdebug.h>
  50 #include <asm/desc.h>
  51 #include <asm/proto.h>
  52 #include <asm/ia32.h>
  53 #include <asm/idle.h>
  54
  55 asmlinkage extern void ret_from_fork(void);
  56
  57 unsigned long kernel_thread_flags = CLONE_VM | CLONE_UNTRACED;
  58
  59 unsigned long boot_option_idle_override = 0;
  60 EXPORT_SYMBOL(boot_option_idle_override);
  61
  62 /*
  63  * Powermanagement idle function, if any..
  64  */
  65 void (*pm_idle)(void);
  66 EXPORT_SYMBOL(pm_idle);
  67 static DEFINE_PER_CPU(unsigned int, cpu_idle_state);
  68
  69 static ATOMIC_NOTIFIER_HEAD(idle_notifier);
  70
  71 void idle_notifier_register(struct notifier_block *n)
  72 {
  73         atomic_notifier_chain_register(&idle_notifier, n);
  74 }
  75 EXPORT_SYMBOL_GPL(idle_notifier_register);
  76
  77 void idle_notifier_unregister(struct notifier_block *n)
  78 {
  79         atomic_notifier_chain_unregister(&idle_notifier, n);
  80 }
  81 EXPORT_SYMBOL(idle_notifier_unregister);
  82
  83 enum idle_state { CPU_IDLE, CPU_NOT_IDLE };
  84 static DEFINE_PER_CPU(enum idle_state, idle_state) = CPU_NOT_IDLE;
  85
  86 void enter_idle(void)
  87 {
  88         __get_cpu_var(idle_state) = CPU_IDLE;
  89         atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
  90 }
  91
  92 static void __exit_idle(void)
  93 {
  94         __get_cpu_var(idle_state) = CPU_NOT_IDLE;
  95         atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);
  96 }
  97
  98 /* Called from interrupts to signify idle end */
  99 void exit_idle(void)
 100 {
 101         if (current->pid | read_pda(irqcount))
 102                 return;
 103         __exit_idle();
 104 }
 105
 106 /*
 107  * We use this if we don't have any better
 108  * idle routine..
 109  */
 110 static void default_idle(void)
 111 {
 112         local_irq_enable();
 113
 114         current_thread_info()->status &= ~TS_POLLING;
 115         smp_mb__after_clear_bit();
 116         while (!need_resched()) {
 117                 local_irq_disable();
 118                 if (!need_resched())
 119                         safe_halt();
 120                 else
 121                         local_irq_enable();
 122         }
 123         current_thread_info()->status |= TS_POLLING;
 124 }
 125
 126 /*
 127  * On SMP it's slightly faster (but much more power-consuming!)
 128  * to poll the ->need_resched flag instead of waiting for the
 129  * cross-CPU IPI to arrive. Use this option with caution.
 130  */
 131 static void poll_idle (void)
 132 {
 133         local_irq_enable();
 134
 135         asm volatile(
 136                 "2:"
 137                 "testl %0,%1;"
 138                 "rep; nop;"
 139                 "je 2b;"
 140                 : :
 141                 "i" (_TIF_NEED_RESCHED),
 142                 "m" (current_thread_info()->flags));
 143 }
 144
 145 void cpu_idle_wait(void)
 146 {
 147         unsigned int cpu, this_cpu = get_cpu();
 148         cpumask_t map;
 149
 150         set_cpus_allowed(current, cpumask_of_cpu(this_cpu));
 151         put_cpu();
 152
 153         cpus_clear(map);
 154         for_each_online_cpu(cpu) {
 155                 per_cpu(cpu_idle_state, cpu) = 1;
 156                 cpu_set(cpu, map);
 157         }
 158
 159         __get_cpu_var(cpu_idle_state) = 0;
 160
 161         wmb();
 162         do {
 163                 ssleep(1);
 164                 for_each_online_cpu(cpu) {
 165                         if (cpu_isset(cpu, map) &&
 166                                         !per_cpu(cpu_idle_state, cpu))
 167                                 cpu_clear(cpu, map);
 168                 }
 169                 cpus_and(map, map, cpu_online_map);
 170         } while (!cpus_empty(map));
 171 }
 172 EXPORT_SYMBOL_GPL(cpu_idle_wait);
 173
 174 #ifdef CONFIG_HOTPLUG_CPU
 175 DECLARE_PER_CPU(int, cpu_state);
 176
 177 #include <asm/nmi.h>
 178 /* We halt the CPU with physical CPU hotplug */
 179 static inline void play_dead(void)
 180 {
 181         idle_task_exit();
 182         wbinvd();
 183         mb();
 184         /* Ack it */
 185         __get_cpu_var(cpu_state) = CPU_DEAD;
 186
 187         local_irq_disable();
 188         while (1)
 189                 halt();
 190 }
 191 #else
 192 static inline void play_dead(void)
 193 {
 194         BUG();
 195 }
 196 #endif /* CONFIG_HOTPLUG_CPU */
 197
 198 /*
 199  * The idle thread. There's no useful work to be
 200  * done, so just try to conserve power and have a
 201  * low exit latency (ie sit in a loop waiting for
 202  * somebody to say that they'd like to reschedule)
 203  */
 204 void cpu_idle (void)
 205 {
 206         current_thread_info()->status |= TS_POLLING;
 207         /* endless idle loop with no priority at all */
 208         while (1) {
 209                 while (!need_resched()) {
 210                         void (*idle)(void);
 211
 212                         if (__get_cpu_var(cpu_idle_state))
 213                                 __get_cpu_var(cpu_idle_state) = 0;
 214
 215                         rmb();
 216                         idle = pm_idle;
 217                         if (!idle)
 218                                 idle = default_idle;
 219                         if (cpu_is_offline(smp_processor_id()))
 220                                 play_dead();
 221                         enter_idle();
 222                         idle();
 223                         __exit_idle();
 224                 }
 225
 226                 preempt_enable_no_resched();
 227                 schedule();
 228                 preempt_disable();
 229         }
 230 }
 231
 232 /*
 233  * This uses new MONITOR/MWAIT instructions on P4 processors with PNI,
 234  * which can obviate IPI to trigger checking of need_resched.
 235  * We execute MONITOR against need_resched and enter optimized wait state
 236  * through MWAIT. Whenever someone changes need_resched, we would be woken
 237  * up from MWAIT (without an IPI).
 238  */
 239 static void mwait_idle(void)
 240 {
 241         local_irq_enable();
 242
 243         while (!need_resched()) {
 244                 __monitor((void *)&current_thread_info()->flags, 0, 0);
 245                 smp_mb();
 246                 if (need_resched())
 247                         break;
 248                 __mwait(0, 0);
 249         }
 250 }
 251
 252 void __cpuinit select_idle_routine(const struct cpuinfo_x86 *c)
 253 {
 254         static int printed;
 255         if (cpu_has(c, X86_FEATURE_MWAIT)) {
 256                 /*
 257                  * Skip, if setup has overridden idle.
 258                  * One CPU supports mwait => All CPUs supports mwait
 259                  */
 260                 if (!pm_idle) {
 261                         if (!printed) {
 262                                 printk("using mwait in idle threads.\n");
 263                                 printed = 1;
 264                         }
 265                         pm_idle = mwait_idle;
 266                 }
 267         }
 268 }
 269
 270 static int __init idle_setup (char *str)
 271 {
 272         if (!strncmp(str, "poll", 4)) {
 273                 printk("using polling idle threads.\n");
 274                 pm_idle = poll_idle;
 275         }
 276
 277         boot_option_idle_override = 1;
 278         return 1;
 279 }
 280
 281 __setup("idle=", idle_setup);
 282
 283 /* Prints also some state that isn't saved in the pt_regs */
 284 void __show_regs(struct pt_regs * regs)
 285 {
 286         unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;
 287         unsigned int fsindex,gsindex;
 288         unsigned int ds,cs,es;
 289
 290         printk("\n");
 291         print_modules();
 292         printk("Pid: %d, comm: %.20s %s %s %.*s\n",
 293                 current->pid, current->comm, print_tainted(),
 294                 system_utsname.release,
 295                 (int)strcspn(system_utsname.version, " "),
 296                 system_utsname.version);
 297         printk("RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->rip);
 298         printk_address(regs->rip);
 299         printk("RSP: %04lx:%016lx  EFLAGS: %08lx\n", regs->ss, regs->rsp,
 300                 regs->eflags);
 301         printk("RAX: %016lx RBX: %016lx RCX: %016lx\n",
 302                regs->rax, regs->rbx, regs->rcx);
 303         printk("RDX: %016lx RSI: %016lx RDI: %016lx\n",
 304                regs->rdx, regs->rsi, regs->rdi);
 305         printk("RBP: %016lx R08: %016lx R09: %016lx\n",
 306                regs->rbp, regs->r8, regs->r9);
 307         printk("R10: %016lx R11: %016lx R12: %016lx\n",
 308                regs->r10, regs->r11, regs->r12);
 309         printk("R13: %016lx R14: %016lx R15: %016lx\n",
 310                regs->r13, regs->r14, regs->r15);
 311
 312         asm("movl %%ds,%0" : "=r" (ds));
 313         asm("movl %%cs,%0" : "=r" (cs));
 314         asm("movl %%es,%0" : "=r" (es));
 315         asm("movl %%fs,%0" : "=r" (fsindex));
 316         asm("movl %%gs,%0" : "=r" (gsindex));
 317
 318         rdmsrl(MSR_FS_BASE, fs);
 319         rdmsrl(MSR_GS_BASE, gs);
 320         rdmsrl(MSR_KERNEL_GS_BASE, shadowgs);
 321
 322         asm("movq %%cr0, %0": "=r" (cr0));
 323         asm("movq %%cr2, %0": "=r" (cr2));
 324         asm("movq %%cr3, %0": "=r" (cr3));
 325         asm("movq %%cr4, %0": "=r" (cr4));
 326
 327         printk("FS:  %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
 328                fs,fsindex,gs,gsindex,shadowgs);
 329         printk("CS:  %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds, es, cr0);
 330         printk("CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3, cr4);
 331 }
 332
 333 void show_regs(struct pt_regs *regs)
 334 {
 335         printk("CPU %d:", smp_processor_id());
 336         __show_regs(regs);
 337         show_trace(NULL, regs, (void *)(regs + 1));
 338 }
 339
 340 /*
 341  * Free current thread data structures etc..
 342  */
 343 void exit_thread(void)
 344 {
 345         struct task_struct *me = current;
 346         struct thread_struct *t = &me->thread;
 347
 348         if (me->thread.io_bitmap_ptr) {
 349                 struct tss_struct *tss = &per_cpu(init_tss, get_cpu());
 350
 351                 kfree(t->io_bitmap_ptr);
 352                 t->io_bitmap_ptr = NULL;
 353                 /*
 354                  * Careful, clear this in the TSS too:
 355                  */
 356                 memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
 357                 t->io_bitmap_max = 0;
 358                 put_cpu();
 359         }
 360 }
 361
 362 void flush_thread(void)
 363 {
 364         struct task_struct *tsk = current;
 365         struct thread_info *t = current_thread_info();
 366
 367         if (t->flags & _TIF_ABI_PENDING) {
 368                 t->flags ^= (_TIF_ABI_PENDING | _TIF_IA32);
 369                 if (t->flags & _TIF_IA32)
 370                         current_thread_info()->status |= TS_COMPAT;
 371         }
 372
 373         tsk->thread.debugreg0 = 0;
 374         tsk->thread.debugreg1 = 0;
 375         tsk->thread.debugreg2 = 0;
 376         tsk->thread.debugreg3 = 0;
 377         tsk->thread.debugreg6 = 0;
 378         tsk->thread.debugreg7 = 0;
 379         memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
 380         /*
 381          * Forget coprocessor state..
 382          */
 383         clear_fpu(tsk);
 384         clear_used_math();
 385 }
 386
 387 void release_thread(struct task_struct *dead_task)
 388 {
 389         if (dead_task->mm) {
 390                 if (dead_task->mm->context.size) {
 391                         printk("WARNING: dead process %8s still has LDT? <%p/%d>\n",
 392                                         dead_task->comm,
 393                                         dead_task->mm->context.ldt,
 394                                         dead_task->mm->context.size);
 395                         BUG();
 396                 }
 397         }
 398 }
 399
 400 static inline void set_32bit_tls(struct task_struct *t, int tls, u32 addr)
 401 {
 402         struct user_desc ud = {
 403                 .base_addr = addr,
 404                 .limit = 0xfffff,
 405                 .seg_32bit = 1,
 406                 .limit_in_pages = 1,
 407                 .useable = 1,
 408         };
 409         struct n_desc_struct *desc = (void *)t->thread.tls_array;
 410         desc += tls;
 411         desc->a = LDT_entry_a(&ud);
 412         desc->b = LDT_entry_b(&ud);
 413 }
 414
 415 static inline u32 read_32bit_tls(struct task_struct *t, int tls)
 416 {
 417         struct desc_struct *desc = (void *)t->thread.tls_array;
 418         desc += tls;
 419         return desc->base0 |
 420                 (((u32)desc->base1) << 16) |
 421                 (((u32)desc->base2) << 24);
 422 }
 423
 424 /*
 425  * This gets called before we allocate a new thread and copy
 426  * the current task into it.
 427  */
 428 void prepare_to_copy(struct task_struct *tsk)
 429 {
 430         unlazy_fpu(tsk);
 431 }
 432
 433 int copy_thread(int nr, unsigned long clone_flags, unsigned long rsp,
 434                 unsigned long unused,
 435         struct task_struct * p, struct pt_regs * regs)
 436 {
 437         int err;
 438         struct pt_regs * childregs;
 439         struct task_struct *me = current;
 440
 441         childregs = ((struct pt_regs *)
 442                         (THREAD_SIZE + task_stack_page(p))) - 1;
 443         *childregs = *regs;
 444
 445         childregs->rax = 0;
 446         childregs->rsp = rsp;
 447         if (rsp == ~0UL)
 448                 childregs->rsp = (unsigned long)childregs;
 449
 450         p->thread.rsp = (unsigned long) childregs;
 451         p->thread.rsp0 = (unsigned long) (childregs+1);
 452         p->thread.userrsp = me->thread.userrsp;
 453
 454         set_tsk_thread_flag(p, TIF_FORK);
 455
 456         p->thread.fs = me->thread.fs;
 457         p->thread.gs = me->thread.gs;
 458
 459         asm("mov %%gs,%0" : "=m" (p->thread.gsindex));
 460         asm("mov %%fs,%0" : "=m" (p->thread.fsindex));
 461         asm("mov %%es,%0" : "=m" (p->thread.es));
 462         asm("mov %%ds,%0" : "=m" (p->thread.ds));
 463
 464         if (unlikely(me->thread.io_bitmap_ptr != NULL)) {
 465                 p->thread.io_bitmap_ptr = kmalloc(IO_BITMAP_BYTES, GFP_KERNEL);
 466                 if (!p->thread.io_bitmap_ptr) {
 467                         p->thread.io_bitmap_max = 0;
 468                         return -ENOMEM;
 469                 }
 470                 memcpy(p->thread.io_bitmap_ptr, me->thread.io_bitmap_ptr,
 471                                 IO_BITMAP_BYTES);
 472         }
 473
 474         /*
 475          * Set a new TLS for the child thread?
 476          */
 477         if (clone_flags & CLONE_SETTLS) {
 478 #ifdef CONFIG_IA32_EMULATION
 479                 if (test_thread_flag(TIF_IA32))
 480                         err = ia32_child_tls(p, childregs);
 481                 else
 482 #endif
 483                         err = do_arch_prctl(p, ARCH_SET_FS, childregs->r8);
 484                 if (err)
 485                         goto out;
 486         }
 487         err = 0;
 488 out:
 489         if (err && p->thread.io_bitmap_ptr) {
 490                 kfree(p->thread.io_bitmap_ptr);
 491                 p->thread.io_bitmap_max = 0;
 492         }
 493         return err;
 494 }
 495
 496 /*
 497  * This special macro can be used to load a debugging register
 498  */
 499 #define loaddebug(thread,r) set_debugreg(thread->debugreg ## r, r)
 500
 501 /*
 502  *      switch_to(x,y) should switch tasks from x to y.
 503  *
 504  * This could still be optimized:
 505  * - fold all the options into a flag word and test it with a single test.
 506  * - could test fs/gs bitsliced
 507  *
 508  * Kprobes not supported here. Set the probe on schedule instead.
 509  */
 510 __kprobes struct task_struct *
 511 __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 512 {
 513         struct thread_struct *prev = &prev_p->thread,
 514                                  *next = &next_p->thread;
 515         int cpu = smp_processor_id();
 516         struct tss_struct *tss = &per_cpu(init_tss, cpu);
 517
 518         /*
 519          * Reload esp0, LDT and the page table pointer:
 520          */
 521         tss->rsp0 = next->rsp0;
 522
 523         /*
 524          * Switch DS and ES.
 525          * This won't pick up thread selector changes, but I guess that is ok.
 526          */
 527         asm volatile("mov %%es,%0" : "=m" (prev->es));
 528         if (unlikely(next->es | prev->es))
 529                 loadsegment(es, next->es);
 530
 531         asm volatile ("mov %%ds,%0" : "=m" (prev->ds));
 532         if (unlikely(next->ds | prev->ds))
 533                 loadsegment(ds, next->ds);
 534
 535         load_TLS(next, cpu);
 536
 537         /*
 538          * Switch FS and GS.
 539          */
 540         {
 541                 unsigned fsindex;
 542                 asm volatile("movl %%fs,%0" : "=r" (fsindex));
 543                 /* segment register != 0 always requires a reload.
 544                    also reload when it has changed.
 545                    when prev process used 64bit base always reload
 546                    to avoid an information leak. */
 547                 if (unlikely(fsindex | next->fsindex | prev->fs)) {
 548                         loadsegment(fs, next->fsindex);
 549                         /* check if the user used a selector != 0
 550                          * if yes clear 64bit base, since overloaded base
 551                          * is always mapped to the Null selector
 552                          */
 553                         if (fsindex)
 554                         prev->fs = 0;
 555                 }
 556                 /* when next process has a 64bit base use it */
 557                 if (next->fs)
 558                         wrmsrl(MSR_FS_BASE, next->fs);
 559                 prev->fsindex = fsindex;
 560         }
 561         {
 562                 unsigned gsindex;
 563                 asm volatile("movl %%gs,%0" : "=r" (gsindex));
 564                 if (unlikely(gsindex | next->gsindex | prev->gs)) {
 565                         load_gs_index(next->gsindex);
 566                         if (gsindex)
 567                         prev->gs = 0;
 568                 }
 569                 if (next->gs)
 570                         wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
 571                 prev->gsindex = gsindex;
 572         }
 573
 574         /*
 575          * Switch the PDA and FPU contexts.
 576          */
 577         prev->userrsp = read_pda(oldrsp);
 578         write_pda(oldrsp, next->userrsp);
 579         write_pda(pcurrent, next_p);
 580
 581         /* This must be here to ensure both math_state_restore() and
 582            kernel_fpu_begin() work consistently.
 583            And the AMD workaround requires it to be after DS reload. */
 584         unlazy_fpu(prev_p);
 585         write_pda(kernelstack,
 586                   task_stack_page(next_p) + THREAD_SIZE - PDA_STACKOFFSET);
 587
 588         /*
 589          * Now maybe reload the debug registers
 590          */
 591         if (unlikely(next->debugreg7)) {
 592                 loaddebug(next, 0);
 593                 loaddebug(next, 1);
 594                 loaddebug(next, 2);
 595                 loaddebug(next, 3);
 596                 /* no 4 and 5 */
 597                 loaddebug(next, 6);
 598                 loaddebug(next, 7);
 599         }
 600
 601
 602         /*
 603          * Handle the IO bitmap
 604          */
 605         if (unlikely(prev->io_bitmap_ptr || next->io_bitmap_ptr)) {
 606                 if (next->io_bitmap_ptr)
 607                         /*
 608                          * Copy the relevant range of the IO bitmap.
 609                          * Normally this is 128 bytes or less:
 610                          */
 611                         memcpy(tss->io_bitmap, next->io_bitmap_ptr,
 612                                 max(prev->io_bitmap_max, next->io_bitmap_max));
 613                 else {
 614                         /*
 615                          * Clear any possible leftover bits:
 616                          */
 617                         memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
 618                 }
 619         }
 620
 621         return prev_p;
 622 }
 623
 624 /*
 625  * sys_execve() executes a new program.
 626  */
 627 asmlinkage
 628 long sys_execve(char __user *name, char __user * __user *argv,
 629                 char __user * __user *envp, struct pt_regs regs)
 630 {
 631         long error;
 632         char * filename;
 633
 634         filename = getname(name);
 635         error = PTR_ERR(filename);
 636         if (IS_ERR(filename))
 637                 return error;
 638         error = do_execve(filename, argv, envp, &regs);
 639         if (error == 0) {
 640                 task_lock(current);
 641                 current->ptrace &= ~PT_DTRACE;
 642                 task_unlock(current);
 643         }
 644         putname(filename);
 645         return error;
 646 }
 647
 648 void set_personality_64bit(void)
 649 {
 650         /* inherit personality from parent */
 651
 652         /* Make sure to be in 64bit mode */
 653         clear_thread_flag(TIF_IA32);
 654
 655         /* TBD: overwrites user setup. Should have two bits.
 656            But 64bit processes have always behaved this way,
 657            so it's not too bad. The main problem is just that
 658            32bit childs are affected again. */
 659         current->personality &= ~READ_IMPLIES_EXEC;
 660 }
 661
 662 asmlinkage long sys_fork(struct pt_regs *regs)
 663 {
 664         return do_fork(SIGCHLD, regs->rsp, regs, 0, NULL, NULL);
 665 }
 666
 667 asmlinkage long
 668 sys_clone(unsigned long clone_flags, unsigned long newsp,
 669           void __user *parent_tid, void __user *child_tid, struct pt_regs *regs)
 670 {
 671         if (!newsp)
 672                 newsp = regs->rsp;
 673         return do_fork(clone_flags, newsp, regs, 0, parent_tid, child_tid);
 674 }
 675
 676 /*
 677  * This is trivial, and on the face of it looks like it
 678  * could equally well be done in user mode.
 679  *
 680  * Not so, for quite unobvious reasons - register pressure.
 681  * In user mode vfork() cannot have a stack frame, and if
 682  * done by calling the "clone()" system call directly, you
 683  * do not have enough call-clobbered registers to hold all
 684  * the information you need.
 685  */
 686 asmlinkage long sys_vfork(struct pt_regs *regs)
 687 {
 688         return do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, regs->rsp, regs, 0,
 689                     NULL, NULL);
 690 }
 691
 692 unsigned long get_wchan(struct task_struct *p)
 693 {
 694         unsigned long stack;
 695         u64 fp,rip;
 696         int count = 0;
 697
 698         if (!p || p == current || p->state==TASK_RUNNING)
 699                 return 0;
 700         stack = (unsigned long)task_stack_page(p);
 701         if (p->thread.rsp < stack || p->thread.rsp > stack+THREAD_SIZE)
 702                 return 0;
 703         fp = *(u64 *)(p->thread.rsp);
 704         do {
 705                 if (fp < (unsigned long)stack ||
 706                     fp > (unsigned long)stack+THREAD_SIZE)
 707                         return 0;
 708                 rip = *(u64 *)(fp+8);
 709                 if (!in_sched_functions(rip))
 710                         return rip;
 711                 fp = *(u64 *)fp;
 712         } while (count++ < 16);
 713         return 0;
 714 }
 715
 716 long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
 717 {
 718         int ret = 0;
 719         int doit = task == current;
 720         int cpu;
 721
 722         switch (code) {
 723         case ARCH_SET_GS:
 724                 if (addr >= TASK_SIZE_OF(task))
 725                         return -EPERM;
 726                 cpu = get_cpu();
 727                 /* handle small bases via the GDT because that's faster to
 728                    switch. */
 729                 if (addr <= 0xffffffff) {
 730                         set_32bit_tls(task, GS_TLS, addr);
 731                         if (doit) {
 732                                 load_TLS(&task->thread, cpu);
 733                                 load_gs_index(GS_TLS_SEL);
 734                         }
 735                         task->thread.gsindex = GS_TLS_SEL;
 736                         task->thread.gs = 0;
 737                 } else {
 738                         task->thread.gsindex = 0;
 739                         task->thread.gs = addr;
 740                         if (doit) {
 741                                 load_gs_index(0);
 742                                 ret = checking_wrmsrl(MSR_KERNEL_GS_BASE, addr);
 743                         }
 744                 }
 745                 put_cpu();
 746                 break;
 747         case ARCH_SET_FS:
 748                 /* Not strictly needed for fs, but do it for symmetry
 749                    with gs */
 750                 if (addr >= TASK_SIZE_OF(task))
 751                         return -EPERM;
 752                 cpu = get_cpu();
 753                 /* handle small bases via the GDT because that's faster to
 754                    switch. */
 755                 if (addr <= 0xffffffff) {
 756                         set_32bit_tls(task, FS_TLS, addr);
 757                         if (doit) {
 758                                 load_TLS(&task->thread, cpu);
 759                                 asm volatile("movl %0,%%fs" :: "r"(FS_TLS_SEL));
 760                         }
 761                         task->thread.fsindex = FS_TLS_SEL;
 762                         task->thread.fs = 0;
 763                 } else {
 764                         task->thread.fsindex = 0;
 765                         task->thread.fs = addr;
 766                         if (doit) {
 767                                 /* set the selector to 0 to not confuse
 768                                    __switch_to */
 769                                 asm volatile("movl %0,%%fs" :: "r" (0));
 770                                 ret = checking_wrmsrl(MSR_FS_BASE, addr);
 771                         }
 772                 }
 773                 put_cpu();
 774                 break;
 775         case ARCH_GET_FS: {
 776                 unsigned long base;
 777                 if (task->thread.fsindex == FS_TLS_SEL)
 778                         base = read_32bit_tls(task, FS_TLS);
 779                 else if (doit)
 780                         rdmsrl(MSR_FS_BASE, base);
 781                 else
 782                         base = task->thread.fs;
 783                 ret = put_user(base, (unsigned long __user *)addr);
 784                 break;
 785         }
 786         case ARCH_GET_GS: {
 787                 unsigned long base;
 788                 unsigned gsindex;
 789                 if (task->thread.gsindex == GS_TLS_SEL)
 790                         base = read_32bit_tls(task, GS_TLS);
 791                 else if (doit) {
 792                         asm("movl %%gs,%0" : "=r" (gsindex));
 793                         if (gsindex)
 794                                 rdmsrl(MSR_KERNEL_GS_BASE, base);
 795                         else
 796                                 base = task->thread.gs;
 797                 }
 798                 else
 799                         base = task->thread.gs;
 800                 ret = put_user(base, (unsigned long __user *)addr);
 801                 break;
 802         }
 803
 804         default:
 805                 ret = -EINVAL;
 806                 break;
 807         }
 808
 809         return ret;
 810 }
 811
 812 long sys_arch_prctl(int code, unsigned long addr)
 813 {
 814         return do_arch_prctl(current, code, addr);
 815 }
 816
 817 /*
 818  * Capture the user space registers if the task is not running (in user space)
 819  */
 820 int dump_task_regs(struct task_struct *tsk, elf_gregset_t *regs)
 821 {
 822         struct pt_regs *pp, ptregs;
 823
 824         pp = task_pt_regs(tsk);
 825
 826         ptregs = *pp;
 827         ptregs.cs &= 0xffff;
 828         ptregs.ss &= 0xffff;
 829
 830         elf_core_copy_regs(regs, &ptregs);
 831
 832         return 1;
 833 }
 834
 835 unsigned long arch_align_stack(unsigned long sp)
 836 {
 837         if (randomize_va_space)
 838                 sp -= get_random_int() % 8192;
 839         return sp & ~0xf;
 840 }