2 * linux/arch/x86_64/entry.S
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
6 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
10 * entry.S contains the system-call and fault low-level handling routines.
12 * NOTE: This code handles signal-recognition, which happens every time
13 * after an interrupt and after each system call.
15 * Normal syscalls and interrupts don't save a full stack frame, this is
16 * only done for syscall tracing, signals or fork/exec et.al.
18 * A note on terminology:
19 * - top of stack: Architecture defined interrupt frame from SS to RIP
20 * at the top of the kernel process stack.
21 * - partial stack frame: partially saved registers upto R11.
22 * - full stack frame: Like partial stack frame, but all register saved.
25 * - CFI macros are used to generate dwarf2 unwind information for better
26 * backtraces. They don't change any code.
27 * - SAVE_ALL/RESTORE_ALL - Save/restore all registers
28 * - SAVE_ARGS/RESTORE_ARGS - Save/restore registers that C functions modify.
29 * There are unfortunately lots of special cases where some registers
30 * not touched. The macro is a big mess that should be cleaned up.
31 * - SAVE_REST/RESTORE_REST - Handle the registers not saved by SAVE_ARGS.
32 * Gives a full stack frame.
33 * - ENTRY/END Define functions in the symbol table.
34 * - FIXUP_TOP_OF_STACK/RESTORE_TOP_OF_STACK - Fix up the hardware stack
35 * frame that is otherwise undefined after a SYSCALL
36 * - TRACE_IRQ_* - Trace hard interrupt state for lock debugging.
37 * - errorentry/paranoidentry/zeroentry - Define exception entry points.
40 #include <linux/linkage.h>
41 #include <asm/segment.h>
42 #include <asm/cache.h>
43 #include <asm/errno.h>
44 #include <asm/dwarf2.h>
45 #include <asm/calling.h>
46 #include <asm/asm-offsets.h>
48 #include <asm/unistd.h>
49 #include <asm/thread_info.h>
50 #include <asm/hw_irq.h>
52 #include <asm/irqflags.h>
56 #ifndef CONFIG_PREEMPT
57 #define retint_kernel retint_restore_args
61 .macro TRACE_IRQS_IRETQ offset=ARGOFFSET
62 #ifdef CONFIG_TRACE_IRQFLAGS
63 bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */
71 * C code is not supposed to know about undefined top of stack. Every time
72 * a C function with an pt_regs argument is called from the SYSCALL based
73 * fast path FIXUP_TOP_OF_STACK is needed.
74 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
78 /* %rsp:at FRAMEEND */
79 .macro FIXUP_TOP_OF_STACK tmp
80 movq %gs:pda_oldrsp,\tmp
82 movq $__USER_DS,SS(%rsp)
83 movq $__USER_CS,CS(%rsp)
85 movq R11(%rsp),\tmp /* get eflags */
86 movq \tmp,EFLAGS(%rsp)
89 .macro RESTORE_TOP_OF_STACK tmp,offset=0
90 movq RSP-\offset(%rsp),\tmp
91 movq \tmp,%gs:pda_oldrsp
92 movq EFLAGS-\offset(%rsp),\tmp
93 movq \tmp,R11-\offset(%rsp)
96 .macro FAKE_STACK_FRAME child_rip
97 /* push in order ss, rsp, eflags, cs, rip */
100 CFI_ADJUST_CFA_OFFSET 8
101 /*CFI_REL_OFFSET ss,0*/
103 CFI_ADJUST_CFA_OFFSET 8
105 pushq $(1<<9) /* eflags - interrupts on */
106 CFI_ADJUST_CFA_OFFSET 8
107 /*CFI_REL_OFFSET rflags,0*/
108 pushq $__KERNEL_CS /* cs */
109 CFI_ADJUST_CFA_OFFSET 8
110 /*CFI_REL_OFFSET cs,0*/
111 pushq \child_rip /* rip */
112 CFI_ADJUST_CFA_OFFSET 8
114 pushq %rax /* orig rax */
115 CFI_ADJUST_CFA_OFFSET 8
118 .macro UNFAKE_STACK_FRAME
120 CFI_ADJUST_CFA_OFFSET -(6*8)
123 .macro CFI_DEFAULT_STACK start=1
129 CFI_DEF_CFA_OFFSET SS+8
131 CFI_REL_OFFSET r15,R15
132 CFI_REL_OFFSET r14,R14
133 CFI_REL_OFFSET r13,R13
134 CFI_REL_OFFSET r12,R12
135 CFI_REL_OFFSET rbp,RBP
136 CFI_REL_OFFSET rbx,RBX
137 CFI_REL_OFFSET r11,R11
138 CFI_REL_OFFSET r10,R10
141 CFI_REL_OFFSET rax,RAX
142 CFI_REL_OFFSET rcx,RCX
143 CFI_REL_OFFSET rdx,RDX
144 CFI_REL_OFFSET rsi,RSI
145 CFI_REL_OFFSET rdi,RDI
146 CFI_REL_OFFSET rip,RIP
147 /*CFI_REL_OFFSET cs,CS*/
148 /*CFI_REL_OFFSET rflags,EFLAGS*/
149 CFI_REL_OFFSET rsp,RSP
150 /*CFI_REL_OFFSET ss,SS*/
153 * A newly forked process directly context switches into this.
159 GET_THREAD_INFO(%rcx)
160 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),threadinfo_flags(%rcx)
164 testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread?
165 je int_ret_from_sys_call
166 testl $_TIF_IA32,threadinfo_flags(%rcx)
167 jnz int_ret_from_sys_call
168 RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
169 jmp ret_from_sys_call
172 call syscall_trace_leave
173 GET_THREAD_INFO(%rcx)
179 * System call entry. Upto 6 arguments in registers are supported.
181 * SYSCALL does not save anything on the stack and does not change the
187 * rax system call number
189 * rcx return address for syscall/sysret, C arg3
192 * r10 arg3 (--> moved to rcx for C)
195 * r11 eflags for syscall/sysret, temporary for C
196 * r12-r15,rbp,rbx saved by C code, not touched.
198 * Interrupts are off on entry.
199 * Only called from user space.
201 * XXX if we had a free scratch register we could save the RSP into the stack frame
202 * and report it properly in ps. Unfortunately we haven't.
204 * When user can change the frames always force IRET. That is because
205 * it deals with uncanonical addresses better. SYSRET has trouble
206 * with them due to bugs in both AMD and Intel CPUs.
212 CFI_DEF_CFA rsp,PDA_STACKOFFSET
214 /*CFI_REGISTER rflags,r11*/
216 movq %rsp,%gs:pda_oldrsp
217 movq %gs:pda_kernelstack,%rsp
219 * No need to follow this irqs off/on section - it's straight
224 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
225 movq %rcx,RIP-ARGOFFSET(%rsp)
226 CFI_REL_OFFSET rip,RIP-ARGOFFSET
227 GET_THREAD_INFO(%rcx)
228 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
231 cmpq $__NR_syscall_max,%rax
234 call *sys_call_table(,%rax,8) # XXX: rip relative
235 movq %rax,RAX-ARGOFFSET(%rsp)
237 * Syscall return path ending with SYSRET (fast path)
238 * Has incomplete stack frame and undefined top of stack.
240 .globl ret_from_sys_call
242 movl $_TIF_ALLWORK_MASK,%edi
245 GET_THREAD_INFO(%rcx)
248 movl threadinfo_flags(%rcx),%edx
253 * sysretq will re-enable interrupts:
256 movq RIP-ARGOFFSET(%rsp),%rcx
258 RESTORE_ARGS 0,-ARG_SKIP,1
259 /*CFI_REGISTER rflags,r11*/
260 movq %gs:pda_oldrsp,%rsp
264 /* Handle reschedules */
265 /* edx: work, edi: workmask */
268 bt $TIF_NEED_RESCHED,%edx
273 CFI_ADJUST_CFA_OFFSET 8
276 CFI_ADJUST_CFA_OFFSET -8
279 /* Handle a signal */
283 testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
286 /* Really a signal */
287 /* edx: work flags (arg3) */
288 leaq do_notify_resume(%rip),%rax
289 leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
290 xorl %esi,%esi # oldset -> arg2
291 call ptregscall_common
292 1: movl $_TIF_NEED_RESCHED,%edi
293 /* Use IRET because user could have changed frame. This
294 works because ptregscall_common has called FIXUP_TOP_OF_STACK. */
300 movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
301 jmp ret_from_sys_call
303 /* Do syscall tracing */
307 movq $-ENOSYS,RAX(%rsp)
308 FIXUP_TOP_OF_STACK %rdi
310 call syscall_trace_enter
311 LOAD_ARGS ARGOFFSET /* reload args from stack in case ptrace changed it */
313 cmpq $__NR_syscall_max,%rax
315 movq %r10,%rcx /* fixup for C */
316 call *sys_call_table(,%rax,8)
317 1: movq %rax,RAX-ARGOFFSET(%rsp)
318 /* Use IRET because user could have changed frame */
319 jmp int_ret_from_sys_call
324 * Syscall return path ending with IRET.
325 * Has correct top of stack, but partial stack frame.
327 ENTRY(int_ret_from_sys_call)
330 CFI_DEF_CFA rsp,SS+8-ARGOFFSET
331 /*CFI_REL_OFFSET ss,SS-ARGOFFSET*/
332 CFI_REL_OFFSET rsp,RSP-ARGOFFSET
333 /*CFI_REL_OFFSET rflags,EFLAGS-ARGOFFSET*/
334 /*CFI_REL_OFFSET cs,CS-ARGOFFSET*/
335 CFI_REL_OFFSET rip,RIP-ARGOFFSET
336 CFI_REL_OFFSET rdx,RDX-ARGOFFSET
337 CFI_REL_OFFSET rcx,RCX-ARGOFFSET
338 CFI_REL_OFFSET rax,RAX-ARGOFFSET
339 CFI_REL_OFFSET rdi,RDI-ARGOFFSET
340 CFI_REL_OFFSET rsi,RSI-ARGOFFSET
341 CFI_REL_OFFSET r8,R8-ARGOFFSET
342 CFI_REL_OFFSET r9,R9-ARGOFFSET
343 CFI_REL_OFFSET r10,R10-ARGOFFSET
344 CFI_REL_OFFSET r11,R11-ARGOFFSET
347 testl $3,CS-ARGOFFSET(%rsp)
348 je retint_restore_args
349 movl $_TIF_ALLWORK_MASK,%edi
350 /* edi: mask to check */
352 GET_THREAD_INFO(%rcx)
353 movl threadinfo_flags(%rcx),%edx
356 andl $~TS_COMPAT,threadinfo_status(%rcx)
359 /* Either reschedule or signal or syscall exit tracking needed. */
360 /* First do a reschedule test. */
361 /* edx: work, edi: workmask */
363 bt $TIF_NEED_RESCHED,%edx
368 CFI_ADJUST_CFA_OFFSET 8
371 CFI_ADJUST_CFA_OFFSET -8
376 /* handle signals and tracing -- both require a full stack frame */
381 /* Check for syscall exit trace */
382 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
385 CFI_ADJUST_CFA_OFFSET 8
386 leaq 8(%rsp),%rdi # &ptregs -> arg1
387 call syscall_trace_leave
389 CFI_ADJUST_CFA_OFFSET -8
390 andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
396 testl $(_TIF_NOTIFY_RESUME|_TIF_SIGPENDING|_TIF_SINGLESTEP),%edx
398 movq %rsp,%rdi # &ptregs -> arg1
399 xorl %esi,%esi # oldset -> arg2
400 call do_notify_resume
401 1: movl $_TIF_NEED_RESCHED,%edi
408 END(int_ret_from_sys_call)
411 * Certain special system calls that need to save a complete full stack frame.
414 .macro PTREGSCALL label,func,arg
417 leaq \func(%rip),%rax
418 leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
419 jmp ptregscall_common
425 PTREGSCALL stub_clone, sys_clone, %r8
426 PTREGSCALL stub_fork, sys_fork, %rdi
427 PTREGSCALL stub_vfork, sys_vfork, %rdi
428 PTREGSCALL stub_rt_sigsuspend, sys_rt_sigsuspend, %rdx
429 PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
430 PTREGSCALL stub_iopl, sys_iopl, %rsi
432 ENTRY(ptregscall_common)
434 CFI_ADJUST_CFA_OFFSET -8
435 CFI_REGISTER rip, r11
438 CFI_REGISTER rip, r15
439 FIXUP_TOP_OF_STACK %r11
441 RESTORE_TOP_OF_STACK %r11
443 CFI_REGISTER rip, r11
446 CFI_ADJUST_CFA_OFFSET 8
447 CFI_REL_OFFSET rip, 0
450 END(ptregscall_common)
455 CFI_ADJUST_CFA_OFFSET -8
456 CFI_REGISTER rip, r11
458 FIXUP_TOP_OF_STACK %r11
460 RESTORE_TOP_OF_STACK %r11
463 jmp int_ret_from_sys_call
468 * sigreturn is special because it needs to restore all registers on return.
469 * This cannot be done with SYSRET, so use the IRET return path instead.
471 ENTRY(stub_rt_sigreturn)
474 CFI_ADJUST_CFA_OFFSET -8
477 FIXUP_TOP_OF_STACK %r11
478 call sys_rt_sigreturn
479 movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
481 jmp int_ret_from_sys_call
483 END(stub_rt_sigreturn)
486 * initial frame state for interrupts and exceptions
491 CFI_DEF_CFA rsp,SS+8-\ref
492 /*CFI_REL_OFFSET ss,SS-\ref*/
493 CFI_REL_OFFSET rsp,RSP-\ref
494 /*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
495 /*CFI_REL_OFFSET cs,CS-\ref*/
496 CFI_REL_OFFSET rip,RIP-\ref
499 /* initial frame state for interrupts (and exceptions without error code) */
500 #define INTR_FRAME _frame RIP
501 /* initial frame state for exceptions with error code (and interrupts with
502 vector already pushed) */
503 #define XCPT_FRAME _frame ORIG_RAX
506 * Interrupt entry/exit.
508 * Interrupt entry points save only callee clobbered registers in fast path.
510 * Entry runs with interrupts off.
513 /* 0(%rsp): interrupt number */
514 .macro interrupt func
517 leaq -ARGOFFSET(%rsp),%rdi # arg1 for handler
519 CFI_ADJUST_CFA_OFFSET 8
520 CFI_REL_OFFSET rbp, 0
522 CFI_DEF_CFA_REGISTER rbp
526 /* irqcount is used to check if a CPU is already on an interrupt
527 stack or not. While this is essentially redundant with preempt_count
528 it is a little cheaper to use a separate counter in the PDA
529 (short of moving irq_enter into assembly, which would be too
531 1: incl %gs:pda_irqcount
532 cmoveq %gs:pda_irqstackptr,%rsp
533 push %rbp # backlink for old unwinder
535 * We entered an interrupt context - irqs are off:
541 ENTRY(common_interrupt)
544 /* 0(%rsp): oldrsp-ARGOFFSET */
548 decl %gs:pda_irqcount
550 CFI_DEF_CFA_REGISTER rsp
551 CFI_ADJUST_CFA_OFFSET -8
553 GET_THREAD_INFO(%rcx)
554 testl $3,CS-ARGOFFSET(%rsp)
557 /* Interrupt came from user space */
559 * Has a correct top of stack, but a partial stack frame
560 * %rcx: thread info. Interrupts off.
562 retint_with_reschedule:
563 movl $_TIF_WORK_MASK,%edi
565 movl threadinfo_flags(%rcx),%edx
571 * The iretq could re-enable interrupts:
581 * The iretq could re-enable interrupts:
589 .section __ex_table,"a"
590 .quad iret_label,bad_iret
593 /* force a signal here? this matches i386 behaviour */
594 /* running with kernel gs */
596 movq $11,%rdi /* SIGSEGV */
602 /* edi: workmask, edx: work */
605 bt $TIF_NEED_RESCHED,%edx
610 CFI_ADJUST_CFA_OFFSET 8
613 CFI_ADJUST_CFA_OFFSET -8
614 GET_THREAD_INFO(%rcx)
620 testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
625 movq $-1,ORIG_RAX(%rsp)
626 xorl %esi,%esi # oldset
627 movq %rsp,%rdi # &pt_regs
628 call do_notify_resume
632 movl $_TIF_NEED_RESCHED,%edi
633 GET_THREAD_INFO(%rcx)
636 #ifdef CONFIG_PREEMPT
637 /* Returning to kernel space. Check if we need preemption */
638 /* rcx: threadinfo. interrupts off. */
640 cmpl $0,threadinfo_preempt_count(%rcx)
641 jnz retint_restore_args
642 bt $TIF_NEED_RESCHED,threadinfo_flags(%rcx)
643 jnc retint_restore_args
644 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
645 jnc retint_restore_args
646 call preempt_schedule_irq
651 END(common_interrupt)
656 .macro apicinterrupt num,func
659 CFI_ADJUST_CFA_OFFSET 8
665 ENTRY(thermal_interrupt)
666 apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt
667 END(thermal_interrupt)
669 ENTRY(threshold_interrupt)
670 apicinterrupt THRESHOLD_APIC_VECTOR,mce_threshold_interrupt
671 END(threshold_interrupt)
674 ENTRY(reschedule_interrupt)
675 apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
676 END(reschedule_interrupt)
678 .macro INVALIDATE_ENTRY num
679 ENTRY(invalidate_interrupt\num)
680 apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt
681 END(invalidate_interrupt\num)
693 ENTRY(call_function_interrupt)
694 apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
695 END(call_function_interrupt)
698 ENTRY(apic_timer_interrupt)
699 apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
700 END(apic_timer_interrupt)
702 ENTRY(error_interrupt)
703 apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
706 ENTRY(spurious_interrupt)
707 apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
708 END(spurious_interrupt)
711 * Exception entry points.
715 pushq $0 /* push error code/oldrax */
716 CFI_ADJUST_CFA_OFFSET 8
717 pushq %rax /* push real oldrax to the rdi slot */
718 CFI_ADJUST_CFA_OFFSET 8
724 .macro errorentry sym
727 CFI_ADJUST_CFA_OFFSET 8
733 /* error code is on the stack already */
734 /* handle NMI like exceptions that can happen everywhere */
735 .macro paranoidentry sym, ist=0, irqtrace=1
739 movl $MSR_GS_BASE,%ecx
747 movq %gs:pda_data_offset, %rbp
750 movq ORIG_RAX(%rsp),%rsi
751 movq $-1,ORIG_RAX(%rsp)
753 subq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
757 addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
766 * "Paranoid" exit path from exception stack.
767 * Paranoid because this is used by NMIs and cannot take
768 * any kernel state for granted.
769 * We don't do kernel preemption checks here, because only
770 * NMI should be common and it does not enable IRQs and
771 * cannot get reschedule ticks.
773 * "trace" is 0 for the NMI handler only, because irq-tracing
774 * is fundamentally NMI-unsafe. (we cannot change the soft and
775 * hard flags at once, atomically)
777 .macro paranoidexit trace=1
778 /* ebx: no swapgs flag */
780 testl %ebx,%ebx /* swapgs needed? */
781 jnz paranoid_restore\trace
783 jnz paranoid_userspace\trace
784 paranoid_swapgs\trace:
789 paranoid_restore\trace:
792 paranoid_userspace\trace:
793 GET_THREAD_INFO(%rcx)
794 movl threadinfo_flags(%rcx),%ebx
795 andl $_TIF_WORK_MASK,%ebx
796 jz paranoid_swapgs\trace
797 movq %rsp,%rdi /* &pt_regs */
799 movq %rax,%rsp /* switch stack for scheduling */
800 testl $_TIF_NEED_RESCHED,%ebx
801 jnz paranoid_schedule\trace
802 movl %ebx,%edx /* arg3: thread flags */
807 xorl %esi,%esi /* arg2: oldset */
808 movq %rsp,%rdi /* arg1: &pt_regs */
809 call do_notify_resume
814 jmp paranoid_userspace\trace
815 paranoid_schedule\trace:
825 jmp paranoid_userspace\trace
830 * Exception entry point. This expects an error code/orig_rax on the stack
831 * and the exception handler in %rax.
833 KPROBE_ENTRY(error_entry)
835 /* rdi slot contains rax, oldrax contains error code */
838 CFI_ADJUST_CFA_OFFSET (14*8)
840 CFI_REL_OFFSET rsi,RSI
841 movq 14*8(%rsp),%rsi /* load rax from rdi slot */
843 CFI_REL_OFFSET rdx,RDX
845 CFI_REL_OFFSET rcx,RCX
846 movq %rsi,10*8(%rsp) /* store rax */
847 CFI_REL_OFFSET rax,RAX
853 CFI_REL_OFFSET r10,R10
855 CFI_REL_OFFSET r11,R11
857 CFI_REL_OFFSET rbx,RBX
859 CFI_REL_OFFSET rbp,RBP
861 CFI_REL_OFFSET r12,R12
863 CFI_REL_OFFSET r13,R13
865 CFI_REL_OFFSET r14,R14
867 CFI_REL_OFFSET r15,R15
876 movq ORIG_RAX(%rsp),%rsi /* get error code */
877 movq $-1,ORIG_RAX(%rsp)
879 /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
885 GET_THREAD_INFO(%rcx)
888 movl threadinfo_flags(%rcx),%edx
889 movl $_TIF_WORK_MASK,%edi
893 * The iret might restore flags:
903 /* There are two places in the kernel that can potentially fault with
904 usergs. Handle them here. The exception handlers after
905 iret run with kernel gs again, so don't set the user space flag.
906 B stepping K8s sometimes report an truncated RIP for IRET
907 exceptions returning to compat mode. Check for these here too. */
908 leaq iret_label(%rip),%rbp
911 movl %ebp,%ebp /* zero extend */
914 cmpq $gs_change,RIP(%rsp)
917 KPROBE_END(error_entry)
919 /* Reload gs selector with exception handling */
920 /* edi: new selector */
924 CFI_ADJUST_CFA_OFFSET 8
929 2: mfence /* workaround */
932 CFI_ADJUST_CFA_OFFSET -8
935 ENDPROC(load_gs_index)
937 .section __ex_table,"a"
939 .quad gs_change,bad_gs
942 /* running with kernelgs */
944 swapgs /* switch back to user gs */
951 * Create a kernel thread.
953 * C extern interface:
954 * extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
956 * asm input arguments:
957 * rdi: fn, rsi: arg, rdx: flags
961 FAKE_STACK_FRAME $child_rip
964 # rdi: flags, rsi: usp, rdx: will be &pt_regs
966 orq kernel_thread_flags(%rip),%rdi
979 * It isn't worth to check for reschedule here,
980 * so internally to the x86_64 port you can rely on kernel_thread()
981 * not to reschedule the child before returning, this avoids the need
982 * of hacks for example to fork off the per-CPU idle tasks.
983 * [Hopefully no generic code relies on the reschedule -AK]
989 ENDPROC(kernel_thread)
992 pushq $0 # fake return address
995 * Here we are in the child and the registers are set as they were
996 * at kernel_thread() invocation in the parent.
1008 * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
1010 * C extern interface:
1011 * extern long execve(char *name, char **argv, char **envp)
1013 * asm input arguments:
1014 * rdi: name, rsi: argv, rdx: envp
1016 * We want to fallback into:
1017 * extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs regs)
1019 * do_sys_execve asm fallback arguments:
1020 * rdi: name, rsi: argv, rdx: envp, fake frame on the stack
1027 movq %rax, RAX(%rsp)
1030 je int_ret_from_sys_call
1037 KPROBE_ENTRY(page_fault)
1038 errorentry do_page_fault
1039 KPROBE_END(page_fault)
1041 ENTRY(coprocessor_error)
1042 zeroentry do_coprocessor_error
1043 END(coprocessor_error)
1045 ENTRY(simd_coprocessor_error)
1046 zeroentry do_simd_coprocessor_error
1047 END(simd_coprocessor_error)
1049 ENTRY(device_not_available)
1050 zeroentry math_state_restore
1051 END(device_not_available)
1053 /* runs on exception stack */
1057 CFI_ADJUST_CFA_OFFSET 8
1058 paranoidentry do_debug, DEBUG_STACK
1062 /* runs on exception stack */
1066 CFI_ADJUST_CFA_OFFSET 8
1067 paranoidentry do_nmi, 0, 0
1068 #ifdef CONFIG_TRACE_IRQFLAGS
1079 CFI_ADJUST_CFA_OFFSET 8
1080 paranoidentry do_int3, DEBUG_STACK
1086 zeroentry do_overflow
1094 zeroentry do_invalid_op
1097 ENTRY(coprocessor_segment_overrun)
1098 zeroentry do_coprocessor_segment_overrun
1099 END(coprocessor_segment_overrun)
1102 zeroentry do_reserved
1105 /* runs on exception stack */
1108 paranoidentry do_double_fault
1114 errorentry do_invalid_TSS
1117 ENTRY(segment_not_present)
1118 errorentry do_segment_not_present
1119 END(segment_not_present)
1121 /* runs on exception stack */
1122 ENTRY(stack_segment)
1124 paranoidentry do_stack_segment
1129 KPROBE_ENTRY(general_protection)
1130 errorentry do_general_protection
1131 KPROBE_END(general_protection)
1133 ENTRY(alignment_check)
1134 errorentry do_alignment_check
1135 END(alignment_check)
1138 zeroentry do_divide_error
1141 ENTRY(spurious_interrupt_bug)
1142 zeroentry do_spurious_interrupt_bug
1143 END(spurious_interrupt_bug)
1145 #ifdef CONFIG_X86_MCE
1146 /* runs on exception stack */
1147 ENTRY(machine_check)
1150 CFI_ADJUST_CFA_OFFSET 8
1151 paranoidentry do_machine_check
1157 /* Call softirq on interrupt stack. Interrupts are off. */
1161 CFI_ADJUST_CFA_OFFSET 8
1162 CFI_REL_OFFSET rbp,0
1164 CFI_DEF_CFA_REGISTER rbp
1165 incl %gs:pda_irqcount
1166 cmove %gs:pda_irqstackptr,%rsp
1167 push %rbp # backlink for old unwinder
1170 CFI_DEF_CFA_REGISTER rsp
1171 CFI_ADJUST_CFA_OFFSET -8
1172 decl %gs:pda_irqcount
1175 ENDPROC(call_softirq)
1177 #ifdef CONFIG_STACK_UNWIND
1178 ENTRY(arch_unwind_init_running)
1180 movq %r15, R15(%rdi)
1181 movq %r14, R14(%rdi)
1183 movq %r13, R13(%rdi)
1184 movq %r12, R12(%rdi)
1186 movq %rbp, RBP(%rdi)
1187 movq %rbx, RBX(%rdi)
1189 movq %rax, R11(%rdi)
1190 movq %rax, R10(%rdi)
1193 movq %rax, RAX(%rdi)
1194 movq %rax, RCX(%rdi)
1195 movq %rax, RDX(%rdi)
1196 movq %rax, RSI(%rdi)
1197 movq %rax, RDI(%rdi)
1198 movq %rax, ORIG_RAX(%rdi)
1199 movq %rcx, RIP(%rdi)
1201 movq $__KERNEL_CS, CS(%rdi)
1202 movq %rax, EFLAGS(%rdi)
1203 movq %rcx, RSP(%rdi)
1204 movq $__KERNEL_DS, SS(%rdi)
1207 ENDPROC(arch_unwind_init_running)