arch/x86_64/kernel/process.c

   1 /*
   2  *  linux/arch/x86-64/kernel/process.c
   3  *
   4  *  Copyright (C) 1995  Linus Torvalds
   5  *
   6  *  Pentium III FXSR, SSE support
   7  *      Gareth Hughes <gareth@valinux.com>, May 2000
   8  *
   9  *  X86-64 port
  10  *      Andi Kleen.
  11  *
  12  *      CPU hotplug support - ashok.raj@intel.com
  13  */
  14
  15 /*
  16  * This file handles the architecture-dependent parts of process handling..
  17  */
  18
  19 #include <stdarg.h>
  20
  21 #include <linux/cpu.h>
  22 #include <linux/errno.h>
  23 #include <linux/sched.h>
  24 #include <linux/kernel.h>
  25 #include <linux/mm.h>
  26 #include <linux/elfcore.h>
  27 #include <linux/smp.h>
  28 #include <linux/slab.h>
  29 #include <linux/user.h>
  30 #include <linux/module.h>
  31 #include <linux/a.out.h>
  32 #include <linux/interrupt.h>
  33 #include <linux/delay.h>
  34 #include <linux/ptrace.h>
  35 #include <linux/utsname.h>
  36 #include <linux/random.h>
  37 #include <linux/notifier.h>
  38 #include <linux/kprobes.h>
  39
  40 #include <asm/uaccess.h>
  41 #include <asm/pgtable.h>
  42 #include <asm/system.h>
  43 #include <asm/io.h>
  44 #include <asm/processor.h>
  45 #include <asm/i387.h>
  46 #include <asm/mmu_context.h>
  47 #include <asm/pda.h>
  48 #include <asm/prctl.h>
  49 #include <asm/kdebug.h>
  50 #include <asm/desc.h>
  51 #include <asm/proto.h>
  52 #include <asm/ia32.h>
  53 #include <asm/idle.h>
  54
  55 asmlinkage extern void ret_from_fork(void);
  56
  57 unsigned long kernel_thread_flags = CLONE_VM | CLONE_UNTRACED;
  58
  59 unsigned long boot_option_idle_override = 0;
  60 EXPORT_SYMBOL(boot_option_idle_override);
  61
  62 /*
  63  * Powermanagement idle function, if any..
  64  */
  65 void (*pm_idle)(void);
  66 static DEFINE_PER_CPU(unsigned int, cpu_idle_state);
  67
  68 static ATOMIC_NOTIFIER_HEAD(idle_notifier);
  69
  70 void idle_notifier_register(struct notifier_block *n)
  71 {
  72         atomic_notifier_chain_register(&idle_notifier, n);
  73 }
  74 EXPORT_SYMBOL_GPL(idle_notifier_register);
  75
  76 void idle_notifier_unregister(struct notifier_block *n)
  77 {
  78         atomic_notifier_chain_unregister(&idle_notifier, n);
  79 }
  80 EXPORT_SYMBOL(idle_notifier_unregister);
  81
  82 enum idle_state { CPU_IDLE, CPU_NOT_IDLE };
  83 static DEFINE_PER_CPU(enum idle_state, idle_state) = CPU_NOT_IDLE;
  84
  85 void enter_idle(void)
  86 {
  87         __get_cpu_var(idle_state) = CPU_IDLE;
  88         atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
  89 }
  90
  91 static void __exit_idle(void)
  92 {
  93         __get_cpu_var(idle_state) = CPU_NOT_IDLE;
  94         atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);
  95 }
  96
  97 /* Called from interrupts to signify idle end */
  98 void exit_idle(void)
  99 {
 100         if (current->pid | read_pda(irqcount))
 101                 return;
 102         __exit_idle();
 103 }
 104
 105 /*
 106  * We use this if we don't have any better
 107  * idle routine..
 108  */
 109 static void default_idle(void)
 110 {
 111         local_irq_enable();
 112
 113         current_thread_info()->status &= ~TS_POLLING;
 114         smp_mb__after_clear_bit();
 115         while (!need_resched()) {
 116                 local_irq_disable();
 117                 if (!need_resched())
 118                         safe_halt();
 119                 else
 120                         local_irq_enable();
 121         }
 122         current_thread_info()->status |= TS_POLLING;
 123 }
 124
 125 /*
 126  * On SMP it's slightly faster (but much more power-consuming!)
 127  * to poll the ->need_resched flag instead of waiting for the
 128  * cross-CPU IPI to arrive. Use this option with caution.
 129  */
 130 static void poll_idle (void)
 131 {
 132         local_irq_enable();
 133
 134         asm volatile(
 135                 "2:"
 136                 "testl %0,%1;"
 137                 "rep; nop;"
 138                 "je 2b;"
 139                 : :
 140                 "i" (_TIF_NEED_RESCHED),
 141                 "m" (current_thread_info()->flags));
 142 }
 143
 144 void cpu_idle_wait(void)
 145 {
 146         unsigned int cpu, this_cpu = get_cpu();
 147         cpumask_t map;
 148
 149         set_cpus_allowed(current, cpumask_of_cpu(this_cpu));
 150         put_cpu();
 151
 152         cpus_clear(map);
 153         for_each_online_cpu(cpu) {
 154                 per_cpu(cpu_idle_state, cpu) = 1;
 155                 cpu_set(cpu, map);
 156         }
 157
 158         __get_cpu_var(cpu_idle_state) = 0;
 159
 160         wmb();
 161         do {
 162                 ssleep(1);
 163                 for_each_online_cpu(cpu) {
 164                         if (cpu_isset(cpu, map) &&
 165                                         !per_cpu(cpu_idle_state, cpu))
 166                                 cpu_clear(cpu, map);
 167                 }
 168                 cpus_and(map, map, cpu_online_map);
 169         } while (!cpus_empty(map));
 170 }
 171 EXPORT_SYMBOL_GPL(cpu_idle_wait);
 172
 173 #ifdef CONFIG_HOTPLUG_CPU
 174 DECLARE_PER_CPU(int, cpu_state);
 175
 176 #include <asm/nmi.h>
 177 /* We halt the CPU with physical CPU hotplug */
 178 static inline void play_dead(void)
 179 {
 180         idle_task_exit();
 181         wbinvd();
 182         mb();
 183         /* Ack it */
 184         __get_cpu_var(cpu_state) = CPU_DEAD;
 185
 186         local_irq_disable();
 187         while (1)
 188                 halt();
 189 }
 190 #else
 191 static inline void play_dead(void)
 192 {
 193         BUG();
 194 }
 195 #endif /* CONFIG_HOTPLUG_CPU */
 196
 197 /*
 198  * The idle thread. There's no useful work to be
 199  * done, so just try to conserve power and have a
 200  * low exit latency (ie sit in a loop waiting for
 201  * somebody to say that they'd like to reschedule)
 202  */
 203 void cpu_idle (void)
 204 {
 205         current_thread_info()->status |= TS_POLLING;
 206         /* endless idle loop with no priority at all */
 207         while (1) {
 208                 while (!need_resched()) {
 209                         void (*idle)(void);
 210
 211                         if (__get_cpu_var(cpu_idle_state))
 212                                 __get_cpu_var(cpu_idle_state) = 0;
 213
 214                         rmb();
 215                         idle = pm_idle;
 216                         if (!idle)
 217                                 idle = default_idle;
 218                         if (cpu_is_offline(smp_processor_id()))
 219                                 play_dead();
 220                         enter_idle();
 221                         idle();
 222                         __exit_idle();
 223                 }
 224
 225                 preempt_enable_no_resched();
 226                 schedule();
 227                 preempt_disable();
 228         }
 229 }
 230
 231 /*
 232  * This uses new MONITOR/MWAIT instructions on P4 processors with PNI,
 233  * which can obviate IPI to trigger checking of need_resched.
 234  * We execute MONITOR against need_resched and enter optimized wait state
 235  * through MWAIT. Whenever someone changes need_resched, we would be woken
 236  * up from MWAIT (without an IPI).
 237  */
 238 static void mwait_idle(void)
 239 {
 240         local_irq_enable();
 241
 242         while (!need_resched()) {
 243                 __monitor((void *)&current_thread_info()->flags, 0, 0);
 244                 smp_mb();
 245                 if (need_resched())
 246                         break;
 247                 __mwait(0, 0);
 248         }
 249 }
 250
 251 void __cpuinit select_idle_routine(const struct cpuinfo_x86 *c)
 252 {
 253         static int printed;
 254         if (cpu_has(c, X86_FEATURE_MWAIT)) {
 255                 /*
 256                  * Skip, if setup has overridden idle.
 257                  * One CPU supports mwait => All CPUs supports mwait
 258                  */
 259                 if (!pm_idle) {
 260                         if (!printed) {
 261                                 printk("using mwait in idle threads.\n");
 262                                 printed = 1;
 263                         }
 264                         pm_idle = mwait_idle;
 265                 }
 266         }
 267 }
 268
 269 static int __init idle_setup (char *str)
 270 {
 271         if (!strncmp(str, "poll", 4)) {
 272                 printk("using polling idle threads.\n");
 273                 pm_idle = poll_idle;
 274         }
 275
 276         boot_option_idle_override = 1;
 277         return 1;
 278 }
 279
 280 __setup("idle=", idle_setup);
 281
 282 /* Prints also some state that isn't saved in the pt_regs */
 283 void __show_regs(struct pt_regs * regs)
 284 {
 285         unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;
 286         unsigned int fsindex,gsindex;
 287         unsigned int ds,cs,es;
 288
 289         printk("\n");
 290         print_modules();
 291         printk("Pid: %d, comm: %.20s %s %s %.*s\n",
 292                 current->pid, current->comm, print_tainted(),
 293                 system_utsname.release,
 294                 (int)strcspn(system_utsname.version, " "),
 295                 system_utsname.version);
 296         printk("RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->rip);
 297         printk_address(regs->rip);
 298         printk("\nRSP: %04lx:%016lx  EFLAGS: %08lx\n", regs->ss, regs->rsp,
 299                 regs->eflags);
 300         printk("RAX: %016lx RBX: %016lx RCX: %016lx\n",
 301                regs->rax, regs->rbx, regs->rcx);
 302         printk("RDX: %016lx RSI: %016lx RDI: %016lx\n",
 303                regs->rdx, regs->rsi, regs->rdi);
 304         printk("RBP: %016lx R08: %016lx R09: %016lx\n",
 305                regs->rbp, regs->r8, regs->r9);
 306         printk("R10: %016lx R11: %016lx R12: %016lx\n",
 307                regs->r10, regs->r11, regs->r12);
 308         printk("R13: %016lx R14: %016lx R15: %016lx\n",
 309                regs->r13, regs->r14, regs->r15);
 310
 311         asm("movl %%ds,%0" : "=r" (ds));
 312         asm("movl %%cs,%0" : "=r" (cs));
 313         asm("movl %%es,%0" : "=r" (es));
 314         asm("movl %%fs,%0" : "=r" (fsindex));
 315         asm("movl %%gs,%0" : "=r" (gsindex));
 316
 317         rdmsrl(MSR_FS_BASE, fs);
 318         rdmsrl(MSR_GS_BASE, gs);
 319         rdmsrl(MSR_KERNEL_GS_BASE, shadowgs);
 320
 321         asm("movq %%cr0, %0": "=r" (cr0));
 322         asm("movq %%cr2, %0": "=r" (cr2));
 323         asm("movq %%cr3, %0": "=r" (cr3));
 324         asm("movq %%cr4, %0": "=r" (cr4));
 325
 326         printk("FS:  %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
 327                fs,fsindex,gs,gsindex,shadowgs);
 328         printk("CS:  %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds, es, cr0);
 329         printk("CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3, cr4);
 330 }
 331
 332 void show_regs(struct pt_regs *regs)
 333 {
 334         printk("CPU %d:", smp_processor_id());
 335         __show_regs(regs);
 336         show_trace(NULL, regs, (void *)(regs + 1));
 337 }
 338
 339 /*
 340  * Free current thread data structures etc..
 341  */
 342 void exit_thread(void)
 343 {
 344         struct task_struct *me = current;
 345         struct thread_struct *t = &me->thread;
 346
 347         if (me->thread.io_bitmap_ptr) {
 348                 struct tss_struct *tss = &per_cpu(init_tss, get_cpu());
 349
 350                 kfree(t->io_bitmap_ptr);
 351                 t->io_bitmap_ptr = NULL;
 352                 /*
 353                  * Careful, clear this in the TSS too:
 354                  */
 355                 memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
 356                 t->io_bitmap_max = 0;
 357                 put_cpu();
 358         }
 359 }
 360
 361 void flush_thread(void)
 362 {
 363         struct task_struct *tsk = current;
 364         struct thread_info *t = current_thread_info();
 365
 366         if (t->flags & _TIF_ABI_PENDING) {
 367                 t->flags ^= (_TIF_ABI_PENDING | _TIF_IA32);
 368                 if (t->flags & _TIF_IA32)
 369                         current_thread_info()->status |= TS_COMPAT;
 370         }
 371
 372         tsk->thread.debugreg0 = 0;
 373         tsk->thread.debugreg1 = 0;
 374         tsk->thread.debugreg2 = 0;
 375         tsk->thread.debugreg3 = 0;
 376         tsk->thread.debugreg6 = 0;
 377         tsk->thread.debugreg7 = 0;
 378         memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
 379         /*
 380          * Forget coprocessor state..
 381          */
 382         clear_fpu(tsk);
 383         clear_used_math();
 384 }
 385
 386 void release_thread(struct task_struct *dead_task)
 387 {
 388         if (dead_task->mm) {
 389                 if (dead_task->mm->context.size) {
 390                         printk("WARNING: dead process %8s still has LDT? <%p/%d>\n",
 391                                         dead_task->comm,
 392                                         dead_task->mm->context.ldt,
 393                                         dead_task->mm->context.size);
 394                         BUG();
 395                 }
 396         }
 397 }
 398
 399 static inline void set_32bit_tls(struct task_struct *t, int tls, u32 addr)
 400 {
 401         struct user_desc ud = {
 402                 .base_addr = addr,
 403                 .limit = 0xfffff,
 404                 .seg_32bit = 1,
 405                 .limit_in_pages = 1,
 406                 .useable = 1,
 407         };
 408         struct n_desc_struct *desc = (void *)t->thread.tls_array;
 409         desc += tls;
 410         desc->a = LDT_entry_a(&ud);
 411         desc->b = LDT_entry_b(&ud);
 412 }
 413
 414 static inline u32 read_32bit_tls(struct task_struct *t, int tls)
 415 {
 416         struct desc_struct *desc = (void *)t->thread.tls_array;
 417         desc += tls;
 418         return desc->base0 |
 419                 (((u32)desc->base1) << 16) |
 420                 (((u32)desc->base2) << 24);
 421 }
 422
 423 /*
 424  * This gets called before we allocate a new thread and copy
 425  * the current task into it.
 426  */
 427 void prepare_to_copy(struct task_struct *tsk)
 428 {
 429         unlazy_fpu(tsk);
 430 }
 431
 432 int copy_thread(int nr, unsigned long clone_flags, unsigned long rsp,
 433                 unsigned long unused,
 434         struct task_struct * p, struct pt_regs * regs)
 435 {
 436         int err;
 437         struct pt_regs * childregs;
 438         struct task_struct *me = current;
 439
 440         childregs = ((struct pt_regs *)
 441                         (THREAD_SIZE + task_stack_page(p))) - 1;
 442         *childregs = *regs;
 443
 444         childregs->rax = 0;
 445         childregs->rsp = rsp;
 446         if (rsp == ~0UL)
 447                 childregs->rsp = (unsigned long)childregs;
 448
 449         p->thread.rsp = (unsigned long) childregs;
 450         p->thread.rsp0 = (unsigned long) (childregs+1);
 451         p->thread.userrsp = me->thread.userrsp;
 452
 453         set_tsk_thread_flag(p, TIF_FORK);
 454
 455         p->thread.fs = me->thread.fs;
 456         p->thread.gs = me->thread.gs;
 457
 458         asm("mov %%gs,%0" : "=m" (p->thread.gsindex));
 459         asm("mov %%fs,%0" : "=m" (p->thread.fsindex));
 460         asm("mov %%es,%0" : "=m" (p->thread.es));
 461         asm("mov %%ds,%0" : "=m" (p->thread.ds));
 462
 463         if (unlikely(me->thread.io_bitmap_ptr != NULL)) {
 464                 p->thread.io_bitmap_ptr = kmalloc(IO_BITMAP_BYTES, GFP_KERNEL);
 465                 if (!p->thread.io_bitmap_ptr) {
 466                         p->thread.io_bitmap_max = 0;
 467                         return -ENOMEM;
 468                 }
 469                 memcpy(p->thread.io_bitmap_ptr, me->thread.io_bitmap_ptr,
 470                                 IO_BITMAP_BYTES);
 471         }
 472
 473         /*
 474          * Set a new TLS for the child thread?
 475          */
 476         if (clone_flags & CLONE_SETTLS) {
 477 #ifdef CONFIG_IA32_EMULATION
 478                 if (test_thread_flag(TIF_IA32))
 479                         err = ia32_child_tls(p, childregs);
 480                 else
 481 #endif
 482                         err = do_arch_prctl(p, ARCH_SET_FS, childregs->r8);
 483                 if (err)
 484                         goto out;
 485         }
 486         err = 0;
 487 out:
 488         if (err && p->thread.io_bitmap_ptr) {
 489                 kfree(p->thread.io_bitmap_ptr);
 490                 p->thread.io_bitmap_max = 0;
 491         }
 492         return err;
 493 }
 494
 495 /*
 496  * This special macro can be used to load a debugging register
 497  */
 498 #define loaddebug(thread,r) set_debugreg(thread->debugreg ## r, r)
 499
 500 /*
 501  *      switch_to(x,y) should switch tasks from x to y.
 502  *
 503  * This could still be optimized:
 504  * - fold all the options into a flag word and test it with a single test.
 505  * - could test fs/gs bitsliced
 506  *
 507  * Kprobes not supported here. Set the probe on schedule instead.
 508  */
 509 __kprobes struct task_struct *
 510 __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 511 {
 512         struct thread_struct *prev = &prev_p->thread,
 513                                  *next = &next_p->thread;
 514         int cpu = smp_processor_id();
 515         struct tss_struct *tss = &per_cpu(init_tss, cpu);
 516
 517         /*
 518          * Reload esp0, LDT and the page table pointer:
 519          */
 520         tss->rsp0 = next->rsp0;
 521
 522         /*
 523          * Switch DS and ES.
 524          * This won't pick up thread selector changes, but I guess that is ok.
 525          */
 526         asm volatile("mov %%es,%0" : "=m" (prev->es));
 527         if (unlikely(next->es | prev->es))
 528                 loadsegment(es, next->es);
 529
 530         asm volatile ("mov %%ds,%0" : "=m" (prev->ds));
 531         if (unlikely(next->ds | prev->ds))
 532                 loadsegment(ds, next->ds);
 533
 534         load_TLS(next, cpu);
 535
 536         /*
 537          * Switch FS and GS.
 538          */
 539         {
 540                 unsigned fsindex;
 541                 asm volatile("movl %%fs,%0" : "=r" (fsindex));
 542                 /* segment register != 0 always requires a reload.
 543                    also reload when it has changed.
 544                    when prev process used 64bit base always reload
 545                    to avoid an information leak. */
 546                 if (unlikely(fsindex | next->fsindex | prev->fs)) {
 547                         loadsegment(fs, next->fsindex);
 548                         /* check if the user used a selector != 0
 549                          * if yes clear 64bit base, since overloaded base
 550                          * is always mapped to the Null selector
 551                          */
 552                         if (fsindex)
 553                         prev->fs = 0;
 554                 }
 555                 /* when next process has a 64bit base use it */
 556                 if (next->fs)
 557                         wrmsrl(MSR_FS_BASE, next->fs);
 558                 prev->fsindex = fsindex;
 559         }
 560         {
 561                 unsigned gsindex;
 562                 asm volatile("movl %%gs,%0" : "=r" (gsindex));
 563                 if (unlikely(gsindex | next->gsindex | prev->gs)) {
 564                         load_gs_index(next->gsindex);
 565                         if (gsindex)
 566                         prev->gs = 0;
 567                 }
 568                 if (next->gs)
 569                         wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
 570                 prev->gsindex = gsindex;
 571         }
 572
 573         /*
 574          * Switch the PDA and FPU contexts.
 575          */
 576         prev->userrsp = read_pda(oldrsp);
 577         write_pda(oldrsp, next->userrsp);
 578         write_pda(pcurrent, next_p);
 579
 580         /* This must be here to ensure both math_state_restore() and
 581            kernel_fpu_begin() work consistently.
 582            And the AMD workaround requires it to be after DS reload. */
 583         unlazy_fpu(prev_p);
 584         write_pda(kernelstack,
 585                   task_stack_page(next_p) + THREAD_SIZE - PDA_STACKOFFSET);
 586
 587         /*
 588          * Now maybe reload the debug registers
 589          */
 590         if (unlikely(next->debugreg7)) {
 591                 loaddebug(next, 0);
 592                 loaddebug(next, 1);
 593                 loaddebug(next, 2);
 594                 loaddebug(next, 3);
 595                 /* no 4 and 5 */
 596                 loaddebug(next, 6);
 597                 loaddebug(next, 7);
 598         }
 599
 600
 601         /*
 602          * Handle the IO bitmap
 603          */
 604         if (unlikely(prev->io_bitmap_ptr || next->io_bitmap_ptr)) {
 605                 if (next->io_bitmap_ptr)
 606                         /*
 607                          * Copy the relevant range of the IO bitmap.
 608                          * Normally this is 128 bytes or less:
 609                          */
 610                         memcpy(tss->io_bitmap, next->io_bitmap_ptr,
 611                                 max(prev->io_bitmap_max, next->io_bitmap_max));
 612                 else {
 613                         /*
 614                          * Clear any possible leftover bits:
 615                          */
 616                         memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
 617                 }
 618         }
 619
 620         return prev_p;
 621 }
 622
 623 /*
 624  * sys_execve() executes a new program.
 625  */
 626 asmlinkage
 627 long sys_execve(char __user *name, char __user * __user *argv,
 628                 char __user * __user *envp, struct pt_regs regs)
 629 {
 630         long error;
 631         char * filename;
 632
 633         filename = getname(name);
 634         error = PTR_ERR(filename);
 635         if (IS_ERR(filename))
 636                 return error;
 637         error = do_execve(filename, argv, envp, &regs);
 638         if (error == 0) {
 639                 task_lock(current);
 640                 current->ptrace &= ~PT_DTRACE;
 641                 task_unlock(current);
 642         }
 643         putname(filename);
 644         return error;
 645 }
 646
 647 void set_personality_64bit(void)
 648 {
 649         /* inherit personality from parent */
 650
 651         /* Make sure to be in 64bit mode */
 652         clear_thread_flag(TIF_IA32);
 653
 654         /* TBD: overwrites user setup. Should have two bits.
 655            But 64bit processes have always behaved this way,
 656            so it's not too bad. The main problem is just that
 657            32bit childs are affected again. */
 658         current->personality &= ~READ_IMPLIES_EXEC;
 659 }
 660
 661 asmlinkage long sys_fork(struct pt_regs *regs)
 662 {
 663         return do_fork(SIGCHLD, regs->rsp, regs, 0, NULL, NULL);
 664 }
 665
 666 asmlinkage long
 667 sys_clone(unsigned long clone_flags, unsigned long newsp,
 668           void __user *parent_tid, void __user *child_tid, struct pt_regs *regs)
 669 {
 670         if (!newsp)
 671                 newsp = regs->rsp;
 672         return do_fork(clone_flags, newsp, regs, 0, parent_tid, child_tid);
 673 }
 674
 675 /*
 676  * This is trivial, and on the face of it looks like it
 677  * could equally well be done in user mode.
 678  *
 679  * Not so, for quite unobvious reasons - register pressure.
 680  * In user mode vfork() cannot have a stack frame, and if
 681  * done by calling the "clone()" system call directly, you
 682  * do not have enough call-clobbered registers to hold all
 683  * the information you need.
 684  */
 685 asmlinkage long sys_vfork(struct pt_regs *regs)
 686 {
 687         return do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, regs->rsp, regs, 0,
 688                     NULL, NULL);
 689 }
 690
 691 unsigned long get_wchan(struct task_struct *p)
 692 {
 693         unsigned long stack;
 694         u64 fp,rip;
 695         int count = 0;
 696
 697         if (!p || p == current || p->state==TASK_RUNNING)
 698                 return 0;
 699         stack = (unsigned long)task_stack_page(p);
 700         if (p->thread.rsp < stack || p->thread.rsp > stack+THREAD_SIZE)
 701                 return 0;
 702         fp = *(u64 *)(p->thread.rsp);
 703         do {
 704                 if (fp < (unsigned long)stack ||
 705                     fp > (unsigned long)stack+THREAD_SIZE)
 706                         return 0;
 707                 rip = *(u64 *)(fp+8);
 708                 if (!in_sched_functions(rip))
 709                         return rip;
 710                 fp = *(u64 *)fp;
 711         } while (count++ < 16);
 712         return 0;
 713 }
 714
 715 long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
 716 {
 717         int ret = 0;
 718         int doit = task == current;
 719         int cpu;
 720
 721         switch (code) {
 722         case ARCH_SET_GS:
 723                 if (addr >= TASK_SIZE_OF(task))
 724                         return -EPERM;
 725                 cpu = get_cpu();
 726                 /* handle small bases via the GDT because that's faster to
 727                    switch. */
 728                 if (addr <= 0xffffffff) {
 729                         set_32bit_tls(task, GS_TLS, addr);
 730                         if (doit) {
 731                                 load_TLS(&task->thread, cpu);
 732                                 load_gs_index(GS_TLS_SEL);
 733                         }
 734                         task->thread.gsindex = GS_TLS_SEL;
 735                         task->thread.gs = 0;
 736                 } else {
 737                         task->thread.gsindex = 0;
 738                         task->thread.gs = addr;
 739                         if (doit) {
 740                                 load_gs_index(0);
 741                                 ret = checking_wrmsrl(MSR_KERNEL_GS_BASE, addr);
 742                         }
 743                 }
 744                 put_cpu();
 745                 break;
 746         case ARCH_SET_FS:
 747                 /* Not strictly needed for fs, but do it for symmetry
 748                    with gs */
 749                 if (addr >= TASK_SIZE_OF(task))
 750                         return -EPERM;
 751                 cpu = get_cpu();
 752                 /* handle small bases via the GDT because that's faster to
 753                    switch. */
 754                 if (addr <= 0xffffffff) {
 755                         set_32bit_tls(task, FS_TLS, addr);
 756                         if (doit) {
 757                                 load_TLS(&task->thread, cpu);
 758                                 asm volatile("movl %0,%%fs" :: "r"(FS_TLS_SEL));
 759                         }
 760                         task->thread.fsindex = FS_TLS_SEL;
 761                         task->thread.fs = 0;
 762                 } else {
 763                         task->thread.fsindex = 0;
 764                         task->thread.fs = addr;
 765                         if (doit) {
 766                                 /* set the selector to 0 to not confuse
 767                                    __switch_to */
 768                                 asm volatile("movl %0,%%fs" :: "r" (0));
 769                                 ret = checking_wrmsrl(MSR_FS_BASE, addr);
 770                         }
 771                 }
 772                 put_cpu();
 773                 break;
 774         case ARCH_GET_FS: {
 775                 unsigned long base;
 776                 if (task->thread.fsindex == FS_TLS_SEL)
 777                         base = read_32bit_tls(task, FS_TLS);
 778                 else if (doit)
 779                         rdmsrl(MSR_FS_BASE, base);
 780                 else
 781                         base = task->thread.fs;
 782                 ret = put_user(base, (unsigned long __user *)addr);
 783                 break;
 784         }
 785         case ARCH_GET_GS: {
 786                 unsigned long base;
 787                 unsigned gsindex;
 788                 if (task->thread.gsindex == GS_TLS_SEL)
 789                         base = read_32bit_tls(task, GS_TLS);
 790                 else if (doit) {
 791                         asm("movl %%gs,%0" : "=r" (gsindex));
 792                         if (gsindex)
 793                                 rdmsrl(MSR_KERNEL_GS_BASE, base);
 794                         else
 795                                 base = task->thread.gs;
 796                 }
 797                 else
 798                         base = task->thread.gs;
 799                 ret = put_user(base, (unsigned long __user *)addr);
 800                 break;
 801         }
 802
 803         default:
 804                 ret = -EINVAL;
 805                 break;
 806         }
 807
 808         return ret;
 809 }
 810
 811 long sys_arch_prctl(int code, unsigned long addr)
 812 {
 813         return do_arch_prctl(current, code, addr);
 814 }
 815
 816 /*
 817  * Capture the user space registers if the task is not running (in user space)
 818  */
 819 int dump_task_regs(struct task_struct *tsk, elf_gregset_t *regs)
 820 {
 821         struct pt_regs *pp, ptregs;
 822
 823         pp = task_pt_regs(tsk);
 824
 825         ptregs = *pp;
 826         ptregs.cs &= 0xffff;
 827         ptregs.ss &= 0xffff;
 828
 829         elf_core_copy_regs(regs, &ptregs);
 830
 831         return 1;
 832 }
 833
 834 unsigned long arch_align_stack(unsigned long sp)
 835 {
 836         if (randomize_va_space)
 837                 sp -= get_random_int() % 8192;
 838         return sp & ~0xf;
 839 }