2 * linux/arch/x86-64/kernel/process.c
4 * Copyright (C) 1995 Linus Torvalds
6 * Pentium III FXSR, SSE support
7 * Gareth Hughes <gareth@valinux.com>, May 2000
12 * CPU hotplug support - ashok.raj@intel.com
16 * This file handles the architecture-dependent parts of process handling..
21 #include <linux/cpu.h>
22 #include <linux/errno.h>
23 #include <linux/sched.h>
24 #include <linux/kernel.h>
26 #include <linux/elfcore.h>
27 #include <linux/smp.h>
28 #include <linux/slab.h>
29 #include <linux/user.h>
30 #include <linux/module.h>
31 #include <linux/a.out.h>
32 #include <linux/interrupt.h>
33 #include <linux/delay.h>
34 #include <linux/ptrace.h>
35 #include <linux/utsname.h>
36 #include <linux/random.h>
37 #include <linux/notifier.h>
38 #include <linux/kprobes.h>
40 #include <asm/uaccess.h>
41 #include <asm/pgtable.h>
42 #include <asm/system.h>
44 #include <asm/processor.h>
46 #include <asm/mmu_context.h>
48 #include <asm/prctl.h>
49 #include <asm/kdebug.h>
51 #include <asm/proto.h>
55 asmlinkage extern void ret_from_fork(void);
57 unsigned long kernel_thread_flags = CLONE_VM | CLONE_UNTRACED;
59 unsigned long boot_option_idle_override = 0;
60 EXPORT_SYMBOL(boot_option_idle_override);
63 * Powermanagement idle function, if any..
65 void (*pm_idle)(void);
66 EXPORT_SYMBOL(pm_idle);
67 static DEFINE_PER_CPU(unsigned int, cpu_idle_state);
69 static ATOMIC_NOTIFIER_HEAD(idle_notifier);
71 void idle_notifier_register(struct notifier_block *n)
73 atomic_notifier_chain_register(&idle_notifier, n);
75 EXPORT_SYMBOL_GPL(idle_notifier_register);
77 void idle_notifier_unregister(struct notifier_block *n)
79 atomic_notifier_chain_unregister(&idle_notifier, n);
81 EXPORT_SYMBOL(idle_notifier_unregister);
86 atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
89 static void __exit_idle(void)
91 if (test_and_clear_bit_pda(0, isidle) == 0)
93 atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);
96 /* Called from interrupts to signify idle end */
99 /* idle loop has pid 0 */
106 * We use this if we don't have any better
109 static void default_idle(void)
111 current_thread_info()->status &= ~TS_POLLING;
112 smp_mb__after_clear_bit();
114 if (!need_resched()) {
115 /* Enables interrupts one instruction before HLT.
116 x86 special cases this so there is no race. */
120 current_thread_info()->status |= TS_POLLING;
124 * On SMP it's slightly faster (but much more power-consuming!)
125 * to poll the ->need_resched flag instead of waiting for the
126 * cross-CPU IPI to arrive. Use this option with caution.
128 static void poll_idle (void)
133 void cpu_idle_wait(void)
135 unsigned int cpu, this_cpu = get_cpu();
136 cpumask_t map, tmp = current->cpus_allowed;
138 set_cpus_allowed(current, cpumask_of_cpu(this_cpu));
142 for_each_online_cpu(cpu) {
143 per_cpu(cpu_idle_state, cpu) = 1;
147 __get_cpu_var(cpu_idle_state) = 0;
152 for_each_online_cpu(cpu) {
153 if (cpu_isset(cpu, map) &&
154 !per_cpu(cpu_idle_state, cpu))
157 cpus_and(map, map, cpu_online_map);
158 } while (!cpus_empty(map));
160 set_cpus_allowed(current, tmp);
162 EXPORT_SYMBOL_GPL(cpu_idle_wait);
164 #ifdef CONFIG_HOTPLUG_CPU
165 DECLARE_PER_CPU(int, cpu_state);
168 /* We halt the CPU with physical CPU hotplug */
169 static inline void play_dead(void)
175 __get_cpu_var(cpu_state) = CPU_DEAD;
182 static inline void play_dead(void)
186 #endif /* CONFIG_HOTPLUG_CPU */
189 * The idle thread. There's no useful work to be
190 * done, so just try to conserve power and have a
191 * low exit latency (ie sit in a loop waiting for
192 * somebody to say that they'd like to reschedule)
196 current_thread_info()->status |= TS_POLLING;
197 /* endless idle loop with no priority at all */
199 while (!need_resched()) {
202 if (__get_cpu_var(cpu_idle_state))
203 __get_cpu_var(cpu_idle_state) = 0;
209 if (cpu_is_offline(smp_processor_id()))
213 /* In many cases the interrupt that ended idle
214 has already called exit_idle. But some idle
215 loops can be woken up without interrupt. */
219 preempt_enable_no_resched();
226 * This uses new MONITOR/MWAIT instructions on P4 processors with PNI,
227 * which can obviate IPI to trigger checking of need_resched.
228 * We execute MONITOR against need_resched and enter optimized wait state
229 * through MWAIT. Whenever someone changes need_resched, we would be woken
230 * up from MWAIT (without an IPI).
232 * New with Core Duo processors, MWAIT can take some hints based on CPU
235 void mwait_idle_with_hints(unsigned long eax, unsigned long ecx)
237 if (!need_resched()) {
238 __monitor((void *)¤t_thread_info()->flags, 0, 0);
245 /* Default MONITOR/MWAIT with no hints, used for default C1 state */
246 static void mwait_idle(void)
249 mwait_idle_with_hints(0,0);
252 void __cpuinit select_idle_routine(const struct cpuinfo_x86 *c)
255 if (cpu_has(c, X86_FEATURE_MWAIT)) {
257 * Skip, if setup has overridden idle.
258 * One CPU supports mwait => All CPUs supports mwait
262 printk("using mwait in idle threads.\n");
265 pm_idle = mwait_idle;
270 static int __init idle_setup (char *str)
272 if (!strncmp(str, "poll", 4)) {
273 printk("using polling idle threads.\n");
277 boot_option_idle_override = 1;
281 __setup("idle=", idle_setup);
283 /* Prints also some state that isn't saved in the pt_regs */
284 void __show_regs(struct pt_regs * regs)
286 unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;
287 unsigned int fsindex,gsindex;
288 unsigned int ds,cs,es;
292 printk("Pid: %d, comm: %.20s %s %s %.*s\n",
293 current->pid, current->comm, print_tainted(),
294 init_utsname()->release,
295 (int)strcspn(init_utsname()->version, " "),
296 init_utsname()->version);
297 printk("RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->rip);
298 printk_address(regs->rip);
299 printk("RSP: %04lx:%016lx EFLAGS: %08lx\n", regs->ss, regs->rsp,
301 printk("RAX: %016lx RBX: %016lx RCX: %016lx\n",
302 regs->rax, regs->rbx, regs->rcx);
303 printk("RDX: %016lx RSI: %016lx RDI: %016lx\n",
304 regs->rdx, regs->rsi, regs->rdi);
305 printk("RBP: %016lx R08: %016lx R09: %016lx\n",
306 regs->rbp, regs->r8, regs->r9);
307 printk("R10: %016lx R11: %016lx R12: %016lx\n",
308 regs->r10, regs->r11, regs->r12);
309 printk("R13: %016lx R14: %016lx R15: %016lx\n",
310 regs->r13, regs->r14, regs->r15);
312 asm("movl %%ds,%0" : "=r" (ds));
313 asm("movl %%cs,%0" : "=r" (cs));
314 asm("movl %%es,%0" : "=r" (es));
315 asm("movl %%fs,%0" : "=r" (fsindex));
316 asm("movl %%gs,%0" : "=r" (gsindex));
318 rdmsrl(MSR_FS_BASE, fs);
319 rdmsrl(MSR_GS_BASE, gs);
320 rdmsrl(MSR_KERNEL_GS_BASE, shadowgs);
322 asm("movq %%cr0, %0": "=r" (cr0));
323 asm("movq %%cr2, %0": "=r" (cr2));
324 asm("movq %%cr3, %0": "=r" (cr3));
325 asm("movq %%cr4, %0": "=r" (cr4));
327 printk("FS: %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
328 fs,fsindex,gs,gsindex,shadowgs);
329 printk("CS: %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds, es, cr0);
330 printk("CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3, cr4);
333 void show_regs(struct pt_regs *regs)
335 printk("CPU %d:", smp_processor_id());
337 show_trace(NULL, regs, (void *)(regs + 1));
341 * Free current thread data structures etc..
343 void exit_thread(void)
345 struct task_struct *me = current;
346 struct thread_struct *t = &me->thread;
348 if (me->thread.io_bitmap_ptr) {
349 struct tss_struct *tss = &per_cpu(init_tss, get_cpu());
351 kfree(t->io_bitmap_ptr);
352 t->io_bitmap_ptr = NULL;
353 clear_thread_flag(TIF_IO_BITMAP);
355 * Careful, clear this in the TSS too:
357 memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
358 t->io_bitmap_max = 0;
363 void flush_thread(void)
365 struct task_struct *tsk = current;
366 struct thread_info *t = current_thread_info();
368 if (t->flags & _TIF_ABI_PENDING) {
369 t->flags ^= (_TIF_ABI_PENDING | _TIF_IA32);
370 if (t->flags & _TIF_IA32)
371 current_thread_info()->status |= TS_COMPAT;
373 t->flags &= ~_TIF_DEBUG;
375 tsk->thread.debugreg0 = 0;
376 tsk->thread.debugreg1 = 0;
377 tsk->thread.debugreg2 = 0;
378 tsk->thread.debugreg3 = 0;
379 tsk->thread.debugreg6 = 0;
380 tsk->thread.debugreg7 = 0;
381 memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
383 * Forget coprocessor state..
389 void release_thread(struct task_struct *dead_task)
392 if (dead_task->mm->context.size) {
393 printk("WARNING: dead process %8s still has LDT? <%p/%d>\n",
395 dead_task->mm->context.ldt,
396 dead_task->mm->context.size);
402 static inline void set_32bit_tls(struct task_struct *t, int tls, u32 addr)
404 struct user_desc ud = {
411 struct n_desc_struct *desc = (void *)t->thread.tls_array;
413 desc->a = LDT_entry_a(&ud);
414 desc->b = LDT_entry_b(&ud);
417 static inline u32 read_32bit_tls(struct task_struct *t, int tls)
419 struct desc_struct *desc = (void *)t->thread.tls_array;
422 (((u32)desc->base1) << 16) |
423 (((u32)desc->base2) << 24);
427 * This gets called before we allocate a new thread and copy
428 * the current task into it.
430 void prepare_to_copy(struct task_struct *tsk)
435 int copy_thread(int nr, unsigned long clone_flags, unsigned long rsp,
436 unsigned long unused,
437 struct task_struct * p, struct pt_regs * regs)
440 struct pt_regs * childregs;
441 struct task_struct *me = current;
443 childregs = ((struct pt_regs *)
444 (THREAD_SIZE + task_stack_page(p))) - 1;
448 childregs->rsp = rsp;
450 childregs->rsp = (unsigned long)childregs;
452 p->thread.rsp = (unsigned long) childregs;
453 p->thread.rsp0 = (unsigned long) (childregs+1);
454 p->thread.userrsp = me->thread.userrsp;
456 set_tsk_thread_flag(p, TIF_FORK);
458 p->thread.fs = me->thread.fs;
459 p->thread.gs = me->thread.gs;
461 asm("mov %%gs,%0" : "=m" (p->thread.gsindex));
462 asm("mov %%fs,%0" : "=m" (p->thread.fsindex));
463 asm("mov %%es,%0" : "=m" (p->thread.es));
464 asm("mov %%ds,%0" : "=m" (p->thread.ds));
466 if (unlikely(test_tsk_thread_flag(me, TIF_IO_BITMAP))) {
467 p->thread.io_bitmap_ptr = kmalloc(IO_BITMAP_BYTES, GFP_KERNEL);
468 if (!p->thread.io_bitmap_ptr) {
469 p->thread.io_bitmap_max = 0;
472 memcpy(p->thread.io_bitmap_ptr, me->thread.io_bitmap_ptr,
474 set_tsk_thread_flag(p, TIF_IO_BITMAP);
478 * Set a new TLS for the child thread?
480 if (clone_flags & CLONE_SETTLS) {
481 #ifdef CONFIG_IA32_EMULATION
482 if (test_thread_flag(TIF_IA32))
483 err = ia32_child_tls(p, childregs);
486 err = do_arch_prctl(p, ARCH_SET_FS, childregs->r8);
492 if (err && p->thread.io_bitmap_ptr) {
493 kfree(p->thread.io_bitmap_ptr);
494 p->thread.io_bitmap_max = 0;
500 * This special macro can be used to load a debugging register
502 #define loaddebug(thread,r) set_debugreg(thread->debugreg ## r, r)
504 static inline void __switch_to_xtra(struct task_struct *prev_p,
505 struct task_struct *next_p,
506 struct tss_struct *tss)
508 struct thread_struct *prev, *next;
510 prev = &prev_p->thread,
511 next = &next_p->thread;
513 if (test_tsk_thread_flag(next_p, TIF_DEBUG)) {
523 if (test_tsk_thread_flag(next_p, TIF_IO_BITMAP)) {
525 * Copy the relevant range of the IO bitmap.
526 * Normally this is 128 bytes or less:
528 memcpy(tss->io_bitmap, next->io_bitmap_ptr,
529 max(prev->io_bitmap_max, next->io_bitmap_max));
530 } else if (test_tsk_thread_flag(prev_p, TIF_IO_BITMAP)) {
532 * Clear any possible leftover bits:
534 memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
539 * switch_to(x,y) should switch tasks from x to y.
541 * This could still be optimized:
542 * - fold all the options into a flag word and test it with a single test.
543 * - could test fs/gs bitsliced
545 * Kprobes not supported here. Set the probe on schedule instead.
547 __kprobes struct task_struct *
548 __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
550 struct thread_struct *prev = &prev_p->thread,
551 *next = &next_p->thread;
552 int cpu = smp_processor_id();
553 struct tss_struct *tss = &per_cpu(init_tss, cpu);
555 /* we're going to use this soon, after a few expensive things */
556 if (next_p->fpu_counter>5)
557 prefetch(&next->i387.fxsave);
560 * Reload esp0, LDT and the page table pointer:
562 tss->rsp0 = next->rsp0;
566 * This won't pick up thread selector changes, but I guess that is ok.
568 asm volatile("mov %%es,%0" : "=m" (prev->es));
569 if (unlikely(next->es | prev->es))
570 loadsegment(es, next->es);
572 asm volatile ("mov %%ds,%0" : "=m" (prev->ds));
573 if (unlikely(next->ds | prev->ds))
574 loadsegment(ds, next->ds);
583 asm volatile("movl %%fs,%0" : "=r" (fsindex));
584 /* segment register != 0 always requires a reload.
585 also reload when it has changed.
586 when prev process used 64bit base always reload
587 to avoid an information leak. */
588 if (unlikely(fsindex | next->fsindex | prev->fs)) {
589 loadsegment(fs, next->fsindex);
590 /* check if the user used a selector != 0
591 * if yes clear 64bit base, since overloaded base
592 * is always mapped to the Null selector
597 /* when next process has a 64bit base use it */
599 wrmsrl(MSR_FS_BASE, next->fs);
600 prev->fsindex = fsindex;
604 asm volatile("movl %%gs,%0" : "=r" (gsindex));
605 if (unlikely(gsindex | next->gsindex | prev->gs)) {
606 load_gs_index(next->gsindex);
611 wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
612 prev->gsindex = gsindex;
615 /* Must be after DS reload */
619 * Switch the PDA and FPU contexts.
621 prev->userrsp = read_pda(oldrsp);
622 write_pda(oldrsp, next->userrsp);
623 write_pda(pcurrent, next_p);
625 write_pda(kernelstack,
626 (unsigned long)task_stack_page(next_p) + THREAD_SIZE - PDA_STACKOFFSET);
627 #ifdef CONFIG_CC_STACKPROTECTOR
628 write_pda(stack_canary, next_p->stack_canary);
630 * Build time only check to make sure the stack_canary is at
631 * offset 40 in the pda; this is a gcc ABI requirement
633 BUILD_BUG_ON(offsetof(struct x8664_pda, stack_canary) != 40);
637 * Now maybe reload the debug registers and handle I/O bitmaps
639 if (unlikely((task_thread_info(next_p)->flags & _TIF_WORK_CTXSW))
640 || test_tsk_thread_flag(prev_p, TIF_IO_BITMAP))
641 __switch_to_xtra(prev_p, next_p, tss);
643 /* If the task has used fpu the last 5 timeslices, just do a full
644 * restore of the math state immediately to avoid the trap; the
645 * chances of needing FPU soon are obviously high now
647 if (next_p->fpu_counter>5)
648 math_state_restore();
653 * sys_execve() executes a new program.
656 long sys_execve(char __user *name, char __user * __user *argv,
657 char __user * __user *envp, struct pt_regs regs)
662 filename = getname(name);
663 error = PTR_ERR(filename);
664 if (IS_ERR(filename))
666 error = do_execve(filename, argv, envp, ®s);
669 current->ptrace &= ~PT_DTRACE;
670 task_unlock(current);
676 void set_personality_64bit(void)
678 /* inherit personality from parent */
680 /* Make sure to be in 64bit mode */
681 clear_thread_flag(TIF_IA32);
683 /* TBD: overwrites user setup. Should have two bits.
684 But 64bit processes have always behaved this way,
685 so it's not too bad. The main problem is just that
686 32bit childs are affected again. */
687 current->personality &= ~READ_IMPLIES_EXEC;
690 asmlinkage long sys_fork(struct pt_regs *regs)
692 return do_fork(SIGCHLD, regs->rsp, regs, 0, NULL, NULL);
696 sys_clone(unsigned long clone_flags, unsigned long newsp,
697 void __user *parent_tid, void __user *child_tid, struct pt_regs *regs)
701 return do_fork(clone_flags, newsp, regs, 0, parent_tid, child_tid);
705 * This is trivial, and on the face of it looks like it
706 * could equally well be done in user mode.
708 * Not so, for quite unobvious reasons - register pressure.
709 * In user mode vfork() cannot have a stack frame, and if
710 * done by calling the "clone()" system call directly, you
711 * do not have enough call-clobbered registers to hold all
712 * the information you need.
714 asmlinkage long sys_vfork(struct pt_regs *regs)
716 return do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, regs->rsp, regs, 0,
720 unsigned long get_wchan(struct task_struct *p)
726 if (!p || p == current || p->state==TASK_RUNNING)
728 stack = (unsigned long)task_stack_page(p);
729 if (p->thread.rsp < stack || p->thread.rsp > stack+THREAD_SIZE)
731 fp = *(u64 *)(p->thread.rsp);
733 if (fp < (unsigned long)stack ||
734 fp > (unsigned long)stack+THREAD_SIZE)
736 rip = *(u64 *)(fp+8);
737 if (!in_sched_functions(rip))
740 } while (count++ < 16);
744 long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
747 int doit = task == current;
752 if (addr >= TASK_SIZE_OF(task))
755 /* handle small bases via the GDT because that's faster to
757 if (addr <= 0xffffffff) {
758 set_32bit_tls(task, GS_TLS, addr);
760 load_TLS(&task->thread, cpu);
761 load_gs_index(GS_TLS_SEL);
763 task->thread.gsindex = GS_TLS_SEL;
766 task->thread.gsindex = 0;
767 task->thread.gs = addr;
770 ret = checking_wrmsrl(MSR_KERNEL_GS_BASE, addr);
776 /* Not strictly needed for fs, but do it for symmetry
778 if (addr >= TASK_SIZE_OF(task))
781 /* handle small bases via the GDT because that's faster to
783 if (addr <= 0xffffffff) {
784 set_32bit_tls(task, FS_TLS, addr);
786 load_TLS(&task->thread, cpu);
787 asm volatile("movl %0,%%fs" :: "r"(FS_TLS_SEL));
789 task->thread.fsindex = FS_TLS_SEL;
792 task->thread.fsindex = 0;
793 task->thread.fs = addr;
795 /* set the selector to 0 to not confuse
797 asm volatile("movl %0,%%fs" :: "r" (0));
798 ret = checking_wrmsrl(MSR_FS_BASE, addr);
805 if (task->thread.fsindex == FS_TLS_SEL)
806 base = read_32bit_tls(task, FS_TLS);
808 rdmsrl(MSR_FS_BASE, base);
810 base = task->thread.fs;
811 ret = put_user(base, (unsigned long __user *)addr);
817 if (task->thread.gsindex == GS_TLS_SEL)
818 base = read_32bit_tls(task, GS_TLS);
820 asm("movl %%gs,%0" : "=r" (gsindex));
822 rdmsrl(MSR_KERNEL_GS_BASE, base);
824 base = task->thread.gs;
827 base = task->thread.gs;
828 ret = put_user(base, (unsigned long __user *)addr);
840 long sys_arch_prctl(int code, unsigned long addr)
842 return do_arch_prctl(current, code, addr);
846 * Capture the user space registers if the task is not running (in user space)
848 int dump_task_regs(struct task_struct *tsk, elf_gregset_t *regs)
850 struct pt_regs *pp, ptregs;
852 pp = task_pt_regs(tsk);
858 elf_core_copy_regs(regs, &ptregs);
863 unsigned long arch_align_stack(unsigned long sp)
865 if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
866 sp -= get_random_int() % 8192;