x86, espfix: Make it possible to disable 16-bit support

[pandora-kernel.git] / arch / x86 / kernel / entry_64.S

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index faf8d5e..67fd181 100644 (file)
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -55,6 +55,7 @@
 #include <asm/paravirt.h>
 #include <asm/ftrace.h>
 #include <asm/percpu.h>
+#include <asm/pgtable_types.h>
 
 /* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
 #include <linux/elf-em.h>
@@ -860,10 +861,20 @@ restore_args:
        RESTORE_ARGS 1,8,1
 
 irq_return:
+       /*
+        * Are we returning to a stack segment from the LDT?  Note: in
+        * 64-bit mode SS:RSP on the exception stack is always valid.
+        */
+#ifdef CONFIG_X86_ESPFIX64
+       testb $4,(SS-RIP)(%rsp)
+       jnz irq_return_ldt
+#endif
+
+irq_return_iret:
        INTERRUPT_RETURN
 
        .section __ex_table, "a"
-       .quad irq_return, bad_iret
+       .quad irq_return_iret, bad_iret
        .previous
 
 #ifdef CONFIG_PARAVIRT
@@ -875,6 +886,32 @@ ENTRY(native_iret)
        .previous
 #endif
 
+#ifdef CONFIG_X86_ESPFIX64
+irq_return_ldt:
+       pushq_cfi %rax
+       pushq_cfi %rdi
+       SWAPGS
+       movq PER_CPU_VAR(espfix_waddr),%rdi
+       movq %rax,(0*8)(%rdi)   /* RAX */
+       movq (2*8)(%rsp),%rax   /* RIP */
+       movq %rax,(1*8)(%rdi)
+       movq (3*8)(%rsp),%rax   /* CS */
+       movq %rax,(2*8)(%rdi)
+       movq (4*8)(%rsp),%rax   /* RFLAGS */
+       movq %rax,(3*8)(%rdi)
+       movq (6*8)(%rsp),%rax   /* SS */
+       movq %rax,(5*8)(%rdi)
+       movq (5*8)(%rsp),%rax   /* RSP */
+       movq %rax,(4*8)(%rdi)
+       andl $0xffff0000,%eax
+       popq_cfi %rdi
+       orq PER_CPU_VAR(espfix_stack),%rax
+       SWAPGS
+       movq %rax,%rsp
+       popq_cfi %rax
+       jmp irq_return_iret
+#endif
+
        .section .fixup,"ax"
 bad_iret:
        /*
@@ -938,9 +975,45 @@ ENTRY(retint_kernel)
        call preempt_schedule_irq
        jmp exit_intr
 #endif
-
        CFI_ENDPROC
 END(common_interrupt)
+
+       /*
+        * If IRET takes a fault on the espfix stack, then we
+        * end up promoting it to a doublefault.  In that case,
+        * modify the stack to make it look like we just entered
+        * the #GP handler from user space, similar to bad_iret.
+        */
+#ifdef CONFIG_X86_ESPFIX64
+       ALIGN
+__do_double_fault:
+       XCPT_FRAME 1 RDI+8
+       movq RSP(%rdi),%rax             /* Trap on the espfix stack? */
+       sarq $PGDIR_SHIFT,%rax
+       cmpl $ESPFIX_PGD_ENTRY,%eax
+       jne do_double_fault             /* No, just deliver the fault */
+       cmpl $__KERNEL_CS,CS(%rdi)
+       jne do_double_fault
+       movq RIP(%rdi),%rax
+       cmpq $irq_return_iret,%rax
+#ifdef CONFIG_PARAVIRT
+       je 1f
+       cmpq $native_iret,%rax
+#endif
+       jne do_double_fault             /* This shouldn't happen... */
+1:
+       movq PER_CPU_VAR(kernel_stack),%rax
+       subq $(6*8-KERNEL_STACK_OFFSET),%rax    /* Reset to original stack */
+       movq %rax,RSP(%rdi)
+       movq $0,(%rax)                  /* Missing (lost) #GP error code */
+       movq $general_protection,RIP(%rdi)
+       retq
+       CFI_ENDPROC
+END(__do_double_fault)
+#else
+# define __do_double_fault do_double_fault
+#endif
+
 /*
  * End of kprobes section
  */
@@ -1107,7 +1180,7 @@ zeroentry overflow do_overflow
 zeroentry bounds do_bounds
 zeroentry invalid_op do_invalid_op
 zeroentry device_not_available do_device_not_available
-paranoiderrorentry double_fault do_double_fault
+paranoiderrorentry double_fault __do_double_fault
 zeroentry coprocessor_segment_overrun do_coprocessor_segment_overrun
 errorentry invalid_TSS do_invalid_TSS
 errorentry segment_not_present do_segment_not_present
@@ -1303,7 +1376,7 @@ ENTRY(xen_failsafe_callback)
        CFI_RESTORE r11
        addq $0x30,%rsp
        CFI_ADJUST_CFA_OFFSET -0x30
-       pushq_cfi $0
+       pushq_cfi $-1 /* orig_ax = -1 => not a system call */
        SAVE_ALL
        jmp error_exit
        CFI_ENDPROC
@@ -1438,7 +1511,7 @@ error_sti:
  */
 error_kernelspace:
        incl %ebx
-       leaq irq_return(%rip),%rcx
+       leaq irq_return_iret(%rip),%rcx
        cmpq %rcx,RIP+8(%rsp)
        je error_swapgs
        movl %ecx,%eax  /* zero extend */