x86_64, traps: Rework bad_iret

[pandora-kernel.git] / arch / x86 / kernel / entry_64.S

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 67fd181..9d28dba 100644 (file)
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -861,33 +861,30 @@ restore_args:
        RESTORE_ARGS 1,8,1
 
 irq_return:
+       INTERRUPT_RETURN
+
+ENTRY(native_iret)
        /*
         * Are we returning to a stack segment from the LDT?  Note: in
         * 64-bit mode SS:RSP on the exception stack is always valid.
         */
 #ifdef CONFIG_X86_ESPFIX64
        testb $4,(SS-RIP)(%rsp)
-       jnz irq_return_ldt
+       jnz native_irq_return_ldt
 #endif
 
-irq_return_iret:
-       INTERRUPT_RETURN
-
-       .section __ex_table, "a"
-       .quad irq_return_iret, bad_iret
-       .previous
-
-#ifdef CONFIG_PARAVIRT
-ENTRY(native_iret)
+.global native_irq_return_iret
+native_irq_return_iret:
+       /*
+        * This may fault.  Non-paranoid faults on return to userspace are
+        * handled by fixup_bad_iret.  These include #SS, #GP, and #NP.
+        * Double-faults due to espfix64 are handled in do_double_fault.
+        * Other faults here are fatal.
+        */
        iretq
 
-       .section __ex_table,"a"
-       .quad native_iret, bad_iret
-       .previous
-#endif
-
 #ifdef CONFIG_X86_ESPFIX64
-irq_return_ldt:
+native_irq_return_ldt:
        pushq_cfi %rax
        pushq_cfi %rdi
        SWAPGS
@@ -909,28 +906,9 @@ irq_return_ldt:
        SWAPGS
        movq %rax,%rsp
        popq_cfi %rax
-       jmp irq_return_iret
+       jmp native_irq_return_iret
 #endif
 
-       .section .fixup,"ax"
-bad_iret:
-       /*
-        * The iret traps when the %cs or %ss being restored is bogus.
-        * We've lost the original trap vector and error code.
-        * #GPF is the most likely one to get for an invalid selector.
-        * So pretend we completed the iret and took the #GPF in user mode.
-        *
-        * We are now running with the kernel GS after exception recovery.
-        * But error_entry expects us to have user GS to match the user %cs,
-        * so swap back.
-        */
-       pushq $0
-
-       SWAPGS
-       jmp general_protection
-
-       .previous
-
        /* edi: workmask, edx: work */
 retint_careful:
        CFI_RESTORE_STATE
@@ -978,42 +956,6 @@ ENTRY(retint_kernel)
        CFI_ENDPROC
 END(common_interrupt)
 
-       /*
-        * If IRET takes a fault on the espfix stack, then we
-        * end up promoting it to a doublefault.  In that case,
-        * modify the stack to make it look like we just entered
-        * the #GP handler from user space, similar to bad_iret.
-        */
-#ifdef CONFIG_X86_ESPFIX64
-       ALIGN
-__do_double_fault:
-       XCPT_FRAME 1 RDI+8
-       movq RSP(%rdi),%rax             /* Trap on the espfix stack? */
-       sarq $PGDIR_SHIFT,%rax
-       cmpl $ESPFIX_PGD_ENTRY,%eax
-       jne do_double_fault             /* No, just deliver the fault */
-       cmpl $__KERNEL_CS,CS(%rdi)
-       jne do_double_fault
-       movq RIP(%rdi),%rax
-       cmpq $irq_return_iret,%rax
-#ifdef CONFIG_PARAVIRT
-       je 1f
-       cmpq $native_iret,%rax
-#endif
-       jne do_double_fault             /* This shouldn't happen... */
-1:
-       movq PER_CPU_VAR(kernel_stack),%rax
-       subq $(6*8-KERNEL_STACK_OFFSET),%rax    /* Reset to original stack */
-       movq %rax,RSP(%rdi)
-       movq $0,(%rax)                  /* Missing (lost) #GP error code */
-       movq $general_protection,RIP(%rdi)
-       retq
-       CFI_ENDPROC
-END(__do_double_fault)
-#else
-# define __do_double_fault do_double_fault
-#endif
-
 /*
  * End of kprobes section
  */
@@ -1180,7 +1122,7 @@ zeroentry overflow do_overflow
 zeroentry bounds do_bounds
 zeroentry invalid_op do_invalid_op
 zeroentry device_not_available do_device_not_available
-paranoiderrorentry double_fault __do_double_fault
+paranoiderrorentry double_fault do_double_fault
 zeroentry coprocessor_segment_overrun do_coprocessor_segment_overrun
 errorentry invalid_TSS do_invalid_TSS
 errorentry segment_not_present do_segment_not_present
@@ -1394,7 +1336,7 @@ apicinterrupt XEN_HVM_EVTCHN_CALLBACK \
 
 paranoidzeroentry_ist debug do_debug DEBUG_STACK
 paranoidzeroentry_ist int3 do_int3 DEBUG_STACK
-paranoiderrorentry stack_segment do_stack_segment
+errorentry stack_segment do_stack_segment
 #ifdef CONFIG_XEN
 zeroentry xen_debug do_debug
 zeroentry xen_int3 do_int3
@@ -1504,16 +1446,15 @@ error_sti:
 
 /*
  * There are two places in the kernel that can potentially fault with
- * usergs. Handle them here. The exception handlers after iret run with
- * kernel gs again, so don't set the user space flag. B stepping K8s
- * sometimes report an truncated RIP for IRET exceptions returning to
- * compat mode. Check for these here too.
+ * usergs. Handle them here.  B stepping K8s sometimes report a
+ * truncated RIP for IRET exceptions returning to compat mode. Check
+ * for these here too.
  */
 error_kernelspace:
        incl %ebx
-       leaq irq_return_iret(%rip),%rcx
+       leaq native_irq_return_iret(%rip),%rcx
        cmpq %rcx,RIP+8(%rsp)
-       je error_swapgs
+       je error_bad_iret
        movl %ecx,%eax  /* zero extend */
        cmpq %rax,RIP+8(%rsp)
        je bstep_iret
@@ -1524,7 +1465,15 @@ error_kernelspace:
 bstep_iret:
        /* Fix truncated RIP */
        movq %rcx,RIP+8(%rsp)
-       jmp error_swapgs
+       /* fall through */
+
+error_bad_iret:
+       SWAPGS
+       mov %rsp,%rdi
+       call fixup_bad_iret
+       mov %rax,%rsp
+       decl %ebx       /* Return to usergs */
+       jmp error_sti
        CFI_ENDPROC
 END(error_entry)