[PATCH] x86-64: Make remote TLB flush more scalable

[pandora-kernel.git] / arch / x86_64 / kernel / entry.S

diff --git a/arch/x86_64/kernel/entry.S b/arch/x86_64/kernel/entry.S
index 2881749..873c39d 100644 (file)
--- a/arch/x86_64/kernel/entry.S
+++ b/arch/x86_64/kernel/entry.S
@@ -36,7 +36,7 @@
 #include <asm/errno.h>
 #include <asm/dwarf2.h>
 #include <asm/calling.h>
-#include <asm/offset.h>
+#include <asm/asm-offsets.h>
 #include <asm/msr.h>
 #include <asm/unistd.h>
 #include <asm/thread_info.h>
@@ -76,7 +76,7 @@
 
        .macro FAKE_STACK_FRAME child_rip
        /* push in order ss, rsp, eflags, cs, rip */
-       xorq %rax, %rax
+       xorl %eax, %eax
        pushq %rax /* ss */
        CFI_ADJUST_CFA_OFFSET   8
        pushq %rax /* rsp */
@@ -423,7 +423,7 @@ ENTRY(stub_rt_sigreturn)
        testl $3,CS(%rdi)
        je 1f
        swapgs  
-1:     addl $1,%gs:pda_irqcount        # RED-PEN should check preempt count
+1:     incl    %gs:pda_irqcount        # RED-PEN should check preempt count
        movq %gs:pda_irqstackptr,%rax
        cmoveq %rax,%rsp                                                        
        pushq %rdi                      # save old stack        
@@ -436,7 +436,7 @@ ENTRY(common_interrupt)
 ret_from_intr:         
        popq  %rdi
        cli     
-       subl $1,%gs:pda_irqcount
+       decl %gs:pda_irqcount
 #ifdef CONFIG_DEBUG_INFO
        movq RBP(%rdi),%rbp
 #endif
@@ -494,7 +494,7 @@ retint_signal:
        sti
        SAVE_REST
        movq $-1,ORIG_RAX(%rsp)                         
-       xorq %rsi,%rsi          # oldset
+       xorl %esi,%esi          # oldset
        movq %rsp,%rdi          # &pt_regs
        call do_notify_resume
        RESTORE_REST
@@ -536,8 +536,19 @@ ENTRY(thermal_interrupt)
 ENTRY(reschedule_interrupt)
        apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
 
-ENTRY(invalidate_interrupt)
-       apicinterrupt INVALIDATE_TLB_VECTOR,smp_invalidate_interrupt
+       .macro INVALIDATE_ENTRY num
+ENTRY(invalidate_interrupt\num)
+       apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt 
+       .endm
+
+       INVALIDATE_ENTRY 0
+       INVALIDATE_ENTRY 1
+       INVALIDATE_ENTRY 2
+       INVALIDATE_ENTRY 3
+       INVALIDATE_ENTRY 4
+       INVALIDATE_ENTRY 5
+       INVALIDATE_ENTRY 6
+       INVALIDATE_ENTRY 7
 
 ENTRY(call_function_interrupt)
        apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
@@ -752,7 +763,7 @@ child_rip:
        movq %rsi, %rdi
        call *%rax
        # exit
-       xorq %rdi, %rdi
+       xorl %edi, %edi
        call do_exit
 
 /*
@@ -784,8 +795,9 @@ ENTRY(execve)
        ret
        CFI_ENDPROC
 
-ENTRY(page_fault)
+KPROBE_ENTRY(page_fault)
        errorentry do_page_fault
+       .previous .text
 
 ENTRY(coprocessor_error)
        zeroentry do_coprocessor_error
@@ -797,13 +809,14 @@ ENTRY(device_not_available)
        zeroentry math_state_restore
 
        /* runs on exception stack */
-ENTRY(debug)
+KPROBE_ENTRY(debug)
        CFI_STARTPROC
        pushq $0
        CFI_ADJUST_CFA_OFFSET 8         
        paranoidentry do_debug
        jmp paranoid_exit
        CFI_ENDPROC
+       .previous .text
 
        /* runs on exception stack */   
 ENTRY(nmi)
@@ -854,8 +867,9 @@ paranoid_schedule:
        jmp paranoid_userspace
        CFI_ENDPROC
 
-ENTRY(int3)
+KPROBE_ENTRY(int3)
        zeroentry do_int3       
+       .previous .text
 
 ENTRY(overflow)
        zeroentry do_overflow
@@ -892,8 +906,9 @@ ENTRY(stack_segment)
        jmp paranoid_exit
        CFI_ENDPROC
 
-ENTRY(general_protection)
+KPROBE_ENTRY(general_protection)
        errorentry do_general_protection
+       .previous .text
 
 ENTRY(alignment_check)
        errorentry do_alignment_check
@@ -918,3 +933,15 @@ ENTRY(machine_check)
 ENTRY(call_debug)
        zeroentry do_call_debug
 
+ENTRY(call_softirq)
+       movq %gs:pda_irqstackptr,%rax
+       pushq %r15
+       movq %rsp,%r15
+       incl %gs:pda_irqcount
+       cmove %rax,%rsp
+       call __do_softirq
+       movq %r15,%rsp
+       decl %gs:pda_irqcount
+       popq %r15
+       ret
+