x86/retpoline/ftrace: Convert ftrace assembler indirect jumps

[pandora-kernel.git] / arch / x86 / kernel / entry_32.S

diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index f3f6f53..f3801c7 100644 (file)
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -55,6 +55,7 @@
 #include <asm/irq_vectors.h>
 #include <asm/cpufeature.h>
 #include <asm/alternative-asm.h>
+#include <asm/nospec-branch.h>
 
 /* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
 #include <linux/elf-em.h>
@@ -99,12 +100,6 @@
 #endif
 .endm
 
-#ifdef CONFIG_VM86
-#define resume_userspace_sig   check_userspace
-#else
-#define resume_userspace_sig   resume_userspace
-#endif
-
 /*
  * User gs save/restore
  *
@@ -328,10 +323,19 @@ ret_from_exception:
        preempt_stop(CLBR_ANY)
 ret_from_intr:
        GET_THREAD_INFO(%ebp)
-check_userspace:
+resume_userspace_sig:
+#ifdef CONFIG_VM86
        movl PT_EFLAGS(%esp), %eax      # mix EFLAGS and CS
        movb PT_CS(%esp), %al
        andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
+#else
+       /*
+        * We can be coming here from a syscall done in the kernel space,
+        * e.g. a failed kernel_execve().
+        */
+       movl PT_CS(%esp), %eax
+       andl $SEGMENT_RPL_MASK, %eax
+#endif
        cmpl $USER_RPL, %eax
        jb resume_kernel                # not returning to v8086 or userspace
 
@@ -424,8 +428,14 @@ sysenter_past_esp:
        jnz sysenter_audit
 sysenter_do_call:
        cmpl $(nr_syscalls), %eax
-       jae syscall_badsys
+       jae sysenter_badsys
+#ifdef CONFIG_RETPOLINE
+       movl sys_call_table(,%eax,4),%eax
+       call __x86_indirect_thunk_eax
+#else
        call *sys_call_table(,%eax,4)
+#endif
+sysenter_after_call:
        movl %eax,PT_EAX(%esp)
        LOCKDEP_SYS_EXIT
        DISABLE_INTERRUPTS(CLBR_ANY)
@@ -507,7 +517,13 @@ ENTRY(system_call)
        cmpl $(nr_syscalls), %eax
        jae syscall_badsys
 syscall_call:
+#ifdef CONFIG_RETPOLINE
+       movl sys_call_table(,%eax,4),%eax
+       call __x86_indirect_thunk_eax
+#else
        call *sys_call_table(,%eax,4)
+#endif
+syscall_after_call:
        movl %eax,PT_EAX(%esp)          # store the return value
 syscall_exit:
        LOCKDEP_SYS_EXIT
@@ -522,6 +538,7 @@ syscall_exit:
 restore_all:
        TRACE_IRQS_IRET
 restore_all_notrace:
+#ifdef CONFIG_X86_ESPFIX32
        movl PT_EFLAGS(%esp), %eax      # mix EFLAGS, SS and CS
        # Warning: PT_OLDSS(%esp) contains the wrong/random values if we
        # are returning to the kernel.
@@ -532,6 +549,7 @@ restore_all_notrace:
        cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
        CFI_REMEMBER_STATE
        je ldt_ss                       # returning to user-space with LDT SS
+#endif
 restore_nocheck:
        RESTORE_REGS 4                  # skip orig_eax/error_code
 irq_return:
@@ -547,13 +565,9 @@ ENTRY(iret_exc)
        .long irq_return,iret_exc
 .previous
 
+#ifdef CONFIG_X86_ESPFIX32
        CFI_RESTORE_STATE
 ldt_ss:
-       larl PT_OLDSS(%esp), %eax
-       jnz restore_nocheck
-       testl $0x00400000, %eax         # returning to 32bit stack?
-       jnz restore_nocheck             # allright, normal return
-
 #ifdef CONFIG_PARAVIRT
        /*
         * The kernel can't run on a non-flat stack if paravirt mode
@@ -595,6 +609,7 @@ ldt_ss:
        lss (%esp), %esp                /* switch to espfix segment */
        CFI_ADJUST_CFA_OFFSET -8
        jmp restore_nocheck
+#endif
        CFI_ENDPROC
 ENDPROC(system_call)
 
@@ -677,8 +692,13 @@ syscall_fault:
 END(syscall_fault)
 
 syscall_badsys:
-       movl $-ENOSYS,PT_EAX(%esp)
-       jmp resume_userspace
+       movl $-ENOSYS,%eax
+       jmp syscall_after_call
+END(syscall_badsys)
+
+sysenter_badsys:
+       movl $-ENOSYS,%eax
+       jmp sysenter_after_call
 END(syscall_badsys)
        CFI_ENDPROC
 /*
@@ -761,6 +781,7 @@ ENDPROC(ptregs_clone)
  * the high word of the segment base from the GDT and swiches to the
  * normal stack and adjusts ESP with the matching offset.
  */
+#ifdef CONFIG_X86_ESPFIX32
        /* fixup the stack */
        mov GDT_ESPFIX_SS + 4, %al /* bits 16..23 */
        mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */
@@ -770,8 +791,10 @@ ENDPROC(ptregs_clone)
        pushl_cfi %eax
        lss (%esp), %esp                /* switch to the normal stack segment */
        CFI_ADJUST_CFA_OFFSET -8
+#endif
 .endm
 .macro UNWIND_ESPFIX_STACK
+#ifdef CONFIG_X86_ESPFIX32
        movl %ss, %eax
        /* see if on espfix stack */
        cmpw $__ESPFIX_SS, %ax
@@ -782,6 +805,7 @@ ENDPROC(ptregs_clone)
        /* switch to normal stack */
        FIXUP_ESPFIX_STACK
 27:
+#endif
 .endm
 
 /*
@@ -874,7 +898,7 @@ ENTRY(simd_coprocessor_error)
 661:   pushl_cfi $do_general_protection
 662:
 .section .altinstructions,"a"
-       altinstruction_entry 661b, 663f, X86_FEATURE_XMM, 662b-661b, 664f-663f
+       altinstruction_entry 661b, 663f, X86_FEATURE_XMM, 662b-661b, 664f-663f, 0
 .previous
 .section .altinstr_replacement,"ax"
 663:   pushl $do_simd_coprocessor_error
@@ -1004,7 +1028,7 @@ ENTRY(kernel_thread_helper)
        pushl $0                # fake return address for unwinder
        CFI_STARTPROC
        movl %edi,%eax
-       call *%esi
+       CALL_NOSPEC %esi
        call do_exit
        ud2                     # padding for call trace
        CFI_ENDPROC
@@ -1022,7 +1046,7 @@ ENTRY(xen_sysenter_target)
 
 ENTRY(xen_hypervisor_callback)
        CFI_STARTPROC
-       pushl_cfi $0
+       pushl_cfi $-1 /* orig_ax = -1 => not a system call */
        SAVE_ALL
        TRACE_IRQS_OFF
 
@@ -1064,14 +1088,15 @@ ENTRY(xen_failsafe_callback)
 2:     mov 8(%esp),%es
 3:     mov 12(%esp),%fs
 4:     mov 16(%esp),%gs
+       /* EAX == 0 => Category 1 (Bad segment)
+          EAX != 0 => Category 2 (Bad IRET) */
        testl %eax,%eax
        popl_cfi %eax
        lea 16(%esp),%esp
        CFI_ADJUST_CFA_OFFSET -16
        jz 5f
-       addl $16,%esp
-       jmp iret_exc            # EAX != 0 => Category 2 (Bad IRET)
-5:     pushl_cfi $0            # EAX == 0 => Category 1 (Bad segment)
+       jmp iret_exc
+5:     pushl_cfi $-1 /* orig_ax = -1 => not a system call */
        SAVE_ALL
        jmp ret_from_exception
        CFI_ENDPROC
@@ -1168,7 +1193,8 @@ trace:
        movl 0x4(%ebp), %edx
        subl $MCOUNT_INSN_SIZE, %eax
 
-       call *ftrace_trace_function
+       movl ftrace_trace_function, %ecx
+       CALL_NOSPEC %ecx
 
        popl %edx
        popl %ecx
@@ -1206,7 +1232,7 @@ return_to_handler:
        movl %eax, %ecx
        popl %edx
        popl %eax
-       jmp *%ecx
+       JMP_NOSPEC %ecx
 #endif
 
 .section .rodata,"a"
@@ -1260,7 +1286,7 @@ error_code:
        movl %ecx, %es
        TRACE_IRQS_OFF
        movl %esp,%eax                  # pt_regs pointer
-       call *%edi
+       CALL_NOSPEC %edi
        jmp ret_from_exception
        CFI_ENDPROC
 END(page_fault)
@@ -1317,11 +1343,13 @@ END(debug)
  */
 ENTRY(nmi)
        RING0_INT_FRAME
+#ifdef CONFIG_X86_ESPFIX32
        pushl_cfi %eax
        movl %ss, %eax
        cmpw $__ESPFIX_SS, %ax
        popl_cfi %eax
        je nmi_espfix_stack
+#endif
        cmpl $ia32_sysenter_target,(%esp)
        je nmi_stack_fixup
        pushl_cfi %eax
@@ -1361,6 +1389,7 @@ nmi_debug_stack_check:
        FIX_STACK 24, nmi_stack_correct, 1
        jmp nmi_stack_correct
 
+#ifdef CONFIG_X86_ESPFIX32
 nmi_espfix_stack:
        /* We have a RING0_INT_FRAME here.
         *
@@ -1382,6 +1411,7 @@ nmi_espfix_stack:
        lss 12+4(%esp), %esp            # back to espfix stack
        CFI_ADJUST_CFA_OFFSET -24
        jmp irq_return
+#endif
        CFI_ENDPROC
 END(nmi)