Merge branches 'softirq-for-linus', 'x86-debug-for-linus', 'x86-numa-for-linus',...

[pandora-kernel.git] / arch / x86 / kernel / entry_64.S

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 17be5ec..fe2690d 100644 (file)
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -213,23 +213,17 @@ ENDPROC(native_usergs_sysret64)
        .macro FAKE_STACK_FRAME child_rip
        /* push in order ss, rsp, eflags, cs, rip */
        xorl %eax, %eax
-       pushq $__KERNEL_DS /* ss */
-       CFI_ADJUST_CFA_OFFSET   8
+       pushq_cfi $__KERNEL_DS /* ss */
        /*CFI_REL_OFFSET        ss,0*/
-       pushq %rax /* rsp */
-       CFI_ADJUST_CFA_OFFSET   8
+       pushq_cfi %rax /* rsp */
        CFI_REL_OFFSET  rsp,0
-       pushq $X86_EFLAGS_IF /* eflags - interrupts on */
-       CFI_ADJUST_CFA_OFFSET   8
+       pushq_cfi $X86_EFLAGS_IF /* eflags - interrupts on */
        /*CFI_REL_OFFSET        rflags,0*/
-       pushq $__KERNEL_CS /* cs */
-       CFI_ADJUST_CFA_OFFSET   8
+       pushq_cfi $__KERNEL_CS /* cs */
        /*CFI_REL_OFFSET        cs,0*/
-       pushq \child_rip /* rip */
-       CFI_ADJUST_CFA_OFFSET   8
+       pushq_cfi \child_rip /* rip */
        CFI_REL_OFFSET  rip,0
-       pushq   %rax /* orig rax */
-       CFI_ADJUST_CFA_OFFSET   8
+       pushq_cfi %rax /* orig rax */
        .endm
 
        .macro UNFAKE_STACK_FRAME
@@ -398,10 +392,8 @@ ENTRY(ret_from_fork)
 
        LOCK ; btr $TIF_FORK,TI_flags(%r8)
 
-       push kernel_eflags(%rip)
-       CFI_ADJUST_CFA_OFFSET 8
-       popf                                    # reset kernel eflags
-       CFI_ADJUST_CFA_OFFSET -8
+       pushq_cfi kernel_eflags(%rip)
+       popfq_cfi                               # reset kernel eflags
 
        call schedule_tail                      # rdi: 'prev' task parameter
 
@@ -521,11 +513,9 @@ sysret_careful:
        jnc sysret_signal
        TRACE_IRQS_ON
        ENABLE_INTERRUPTS(CLBR_NONE)
-       pushq %rdi
-       CFI_ADJUST_CFA_OFFSET 8
+       pushq_cfi %rdi
        call schedule
-       popq  %rdi
-       CFI_ADJUST_CFA_OFFSET -8
+       popq_cfi %rdi
        jmp sysret_check
 
        /* Handle a signal */
@@ -634,11 +624,9 @@ int_careful:
        jnc  int_very_careful
        TRACE_IRQS_ON
        ENABLE_INTERRUPTS(CLBR_NONE)
-       pushq %rdi
-       CFI_ADJUST_CFA_OFFSET 8
+       pushq_cfi %rdi
        call schedule
-       popq %rdi
-       CFI_ADJUST_CFA_OFFSET -8
+       popq_cfi %rdi
        DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
        jmp int_with_check
@@ -652,12 +640,10 @@ int_check_syscall_exit_work:
        /* Check for syscall exit trace */
        testl $_TIF_WORK_SYSCALL_EXIT,%edx
        jz int_signal
-       pushq %rdi
-       CFI_ADJUST_CFA_OFFSET 8
+       pushq_cfi %rdi
        leaq 8(%rsp),%rdi       # &ptregs -> arg1
        call syscall_trace_leave
-       popq %rdi
-       CFI_ADJUST_CFA_OFFSET -8
+       popq_cfi %rdi
        andl $~(_TIF_WORK_SYSCALL_EXIT|_TIF_SYSCALL_EMU),%edi
        jmp int_restore_rest
 
@@ -714,9 +700,8 @@ END(ptregscall_common)
 
 ENTRY(stub_execve)
        CFI_STARTPROC
-       popq %r11
-       CFI_ADJUST_CFA_OFFSET -8
-       CFI_REGISTER rip, r11
+       addq $8, %rsp
+       PARTIAL_FRAME 0
        SAVE_REST
        FIXUP_TOP_OF_STACK %r11
        movq %rsp, %rcx
@@ -735,7 +720,7 @@ END(stub_execve)
 ENTRY(stub_rt_sigreturn)
        CFI_STARTPROC
        addq $8, %rsp
-       CFI_ADJUST_CFA_OFFSET   -8
+       PARTIAL_FRAME 0
        SAVE_REST
        movq %rsp,%rdi
        FIXUP_TOP_OF_STACK %r11
@@ -766,8 +751,7 @@ vector=FIRST_EXTERNAL_VECTOR
       .if vector <> FIRST_EXTERNAL_VECTOR
        CFI_ADJUST_CFA_OFFSET -8
       .endif
-1:     pushq $(~vector+0x80)   /* Note: always in signed byte range */
-       CFI_ADJUST_CFA_OFFSET 8
+1:     pushq_cfi $(~vector+0x80)       /* Note: always in signed byte range */
       .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6
        jmp 2f
       .endif
@@ -796,8 +780,8 @@ END(interrupt)
 
 /* 0(%rsp): ~(interrupt number) */
        .macro interrupt func
-       subq $10*8, %rsp
-       CFI_ADJUST_CFA_OFFSET 10*8
+       subq $ORIG_RAX-ARGOFFSET+8, %rsp
+       CFI_ADJUST_CFA_OFFSET ORIG_RAX-ARGOFFSET+8
        call save_args
        PARTIAL_FRAME 0
        call \func
@@ -822,6 +806,7 @@ ret_from_intr:
        TRACE_IRQS_OFF
        decl PER_CPU_VAR(irq_count)
        leaveq
+       CFI_RESTORE             rbp
        CFI_DEF_CFA_REGISTER    rsp
        CFI_ADJUST_CFA_OFFSET   -8
 exit_intr:
@@ -903,11 +888,9 @@ retint_careful:
        jnc   retint_signal
        TRACE_IRQS_ON
        ENABLE_INTERRUPTS(CLBR_NONE)
-       pushq %rdi
-       CFI_ADJUST_CFA_OFFSET   8
+       pushq_cfi %rdi
        call  schedule
-       popq %rdi
-       CFI_ADJUST_CFA_OFFSET   -8
+       popq_cfi %rdi
        GET_THREAD_INFO(%rcx)
        DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
@@ -956,8 +939,7 @@ END(common_interrupt)
 .macro apicinterrupt num sym do_sym
 ENTRY(\sym)
        INTR_FRAME
-       pushq $~(\num)
-       CFI_ADJUST_CFA_OFFSET 8
+       pushq_cfi $~(\num)
        interrupt \do_sym
        jmp ret_from_intr
        CFI_ENDPROC
@@ -981,22 +963,10 @@ apicinterrupt X86_PLATFORM_IPI_VECTOR \
        x86_platform_ipi smp_x86_platform_ipi
 
 #ifdef CONFIG_SMP
-apicinterrupt INVALIDATE_TLB_VECTOR_START+0 \
-       invalidate_interrupt0 smp_invalidate_interrupt
-apicinterrupt INVALIDATE_TLB_VECTOR_START+1 \
-       invalidate_interrupt1 smp_invalidate_interrupt
-apicinterrupt INVALIDATE_TLB_VECTOR_START+2 \
-       invalidate_interrupt2 smp_invalidate_interrupt
-apicinterrupt INVALIDATE_TLB_VECTOR_START+3 \
-       invalidate_interrupt3 smp_invalidate_interrupt
-apicinterrupt INVALIDATE_TLB_VECTOR_START+4 \
-       invalidate_interrupt4 smp_invalidate_interrupt
-apicinterrupt INVALIDATE_TLB_VECTOR_START+5 \
-       invalidate_interrupt5 smp_invalidate_interrupt
-apicinterrupt INVALIDATE_TLB_VECTOR_START+6 \
-       invalidate_interrupt6 smp_invalidate_interrupt
-apicinterrupt INVALIDATE_TLB_VECTOR_START+7 \
-       invalidate_interrupt7 smp_invalidate_interrupt
+.irpc idx, "01234567"
+apicinterrupt (INVALIDATE_TLB_VECTOR_START)+\idx \
+       invalidate_interrupt\idx smp_invalidate_interrupt
+.endr
 #endif
 
 apicinterrupt THRESHOLD_APIC_VECTOR \
@@ -1023,9 +993,9 @@ apicinterrupt ERROR_APIC_VECTOR \
 apicinterrupt SPURIOUS_APIC_VECTOR \
        spurious_interrupt smp_spurious_interrupt
 
-#ifdef CONFIG_PERF_EVENTS
-apicinterrupt LOCAL_PENDING_VECTOR \
-       perf_pending_interrupt smp_perf_pending_interrupt
+#ifdef CONFIG_IRQ_WORK
+apicinterrupt IRQ_WORK_VECTOR \
+       irq_work_interrupt smp_irq_work_interrupt
 #endif
 
 /*
@@ -1036,8 +1006,8 @@ ENTRY(\sym)
        INTR_FRAME
        PARAVIRT_ADJUST_EXCEPTION_FRAME
        pushq_cfi $-1           /* ORIG_RAX: no syscall to restart */
-       subq $15*8,%rsp
-       CFI_ADJUST_CFA_OFFSET 15*8
+       subq $ORIG_RAX-R15, %rsp
+       CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
        call error_entry
        DEFAULT_FRAME 0
        movq %rsp,%rdi          /* pt_regs pointer */
@@ -1052,9 +1022,9 @@ END(\sym)
 ENTRY(\sym)
        INTR_FRAME
        PARAVIRT_ADJUST_EXCEPTION_FRAME
-       pushq $-1               /* ORIG_RAX: no syscall to restart */
-       CFI_ADJUST_CFA_OFFSET 8
-       subq $15*8, %rsp
+       pushq_cfi $-1           /* ORIG_RAX: no syscall to restart */
+       subq $ORIG_RAX-R15, %rsp
+       CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
        call save_paranoid
        TRACE_IRQS_OFF
        movq %rsp,%rdi          /* pt_regs pointer */
@@ -1070,9 +1040,9 @@ END(\sym)
 ENTRY(\sym)
        INTR_FRAME
        PARAVIRT_ADJUST_EXCEPTION_FRAME
-       pushq $-1               /* ORIG_RAX: no syscall to restart */
-       CFI_ADJUST_CFA_OFFSET 8
-       subq $15*8, %rsp
+       pushq_cfi $-1           /* ORIG_RAX: no syscall to restart */
+       subq $ORIG_RAX-R15, %rsp
+       CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
        call save_paranoid
        TRACE_IRQS_OFF
        movq %rsp,%rdi          /* pt_regs pointer */
@@ -1089,8 +1059,8 @@ END(\sym)
 ENTRY(\sym)
        XCPT_FRAME
        PARAVIRT_ADJUST_EXCEPTION_FRAME
-       subq $15*8,%rsp
-       CFI_ADJUST_CFA_OFFSET 15*8
+       subq $ORIG_RAX-R15, %rsp
+       CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
        call error_entry
        DEFAULT_FRAME 0
        movq %rsp,%rdi                  /* pt_regs pointer */
@@ -1107,8 +1077,8 @@ END(\sym)
 ENTRY(\sym)
        XCPT_FRAME
        PARAVIRT_ADJUST_EXCEPTION_FRAME
-       subq $15*8,%rsp
-       CFI_ADJUST_CFA_OFFSET 15*8
+       subq $ORIG_RAX-R15, %rsp
+       CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
        call save_paranoid
        DEFAULT_FRAME 0
        TRACE_IRQS_OFF
@@ -1139,16 +1109,14 @@ zeroentry simd_coprocessor_error do_simd_coprocessor_error
        /* edi:  new selector */
 ENTRY(native_load_gs_index)
        CFI_STARTPROC
-       pushf
-       CFI_ADJUST_CFA_OFFSET 8
+       pushfq_cfi
        DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI)
        SWAPGS
 gs_change:
        movl %edi,%gs
 2:     mfence          /* workaround */
        SWAPGS
-       popf
-       CFI_ADJUST_CFA_OFFSET -8
+       popfq_cfi
        ret
        CFI_ENDPROC
 END(native_load_gs_index)
@@ -1215,8 +1183,7 @@ END(kernel_execve)
 /* Call softirq on interrupt stack. Interrupts are off. */
 ENTRY(call_softirq)
        CFI_STARTPROC
-       push %rbp
-       CFI_ADJUST_CFA_OFFSET   8
+       pushq_cfi %rbp
        CFI_REL_OFFSET rbp,0
        mov  %rsp,%rbp
        CFI_DEF_CFA_REGISTER rbp
@@ -1225,6 +1192,7 @@ ENTRY(call_softirq)
        push  %rbp                      # backlink for old unwinder
        call __do_softirq
        leaveq
+       CFI_RESTORE             rbp
        CFI_DEF_CFA_REGISTER    rsp
        CFI_ADJUST_CFA_OFFSET   -8
        decl PER_CPU_VAR(irq_count)
@@ -1368,7 +1336,7 @@ paranoidzeroentry machine_check *machine_check_vector(%rip)
 
        /* ebx: no swapgs flag */
 ENTRY(paranoid_exit)
-       INTR_FRAME
+       DEFAULT_FRAME
        DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
        testl %ebx,%ebx                         /* swapgs needed? */
@@ -1445,7 +1413,6 @@ error_swapgs:
 error_sti:
        TRACE_IRQS_OFF
        ret
-       CFI_ENDPROC
 
 /*
  * There are two places in the kernel that can potentially fault with
@@ -1470,6 +1437,7 @@ bstep_iret:
        /* Fix truncated RIP */
        movq %rcx,RIP+8(%rsp)
        jmp error_swapgs
+       CFI_ENDPROC
 END(error_entry)
 
 
@@ -1498,8 +1466,8 @@ ENTRY(nmi)
        INTR_FRAME
        PARAVIRT_ADJUST_EXCEPTION_FRAME
        pushq_cfi $-1
-       subq $15*8, %rsp
-       CFI_ADJUST_CFA_OFFSET 15*8
+       subq $ORIG_RAX-R15, %rsp
+       CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
        call save_paranoid
        DEFAULT_FRAME 0
        /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */