2 * This file contains the 64-bit "server" PowerPC variant
3 * of the low level exception handling including exception
4 * vectors, exception return, part of the slb and stab
5 * handling and other fixed offset specific things.
7 * This file is meant to be #included from head_64.S due to
8 * position dependant assembly.
10 * Most of this originates from head_64.S and thus has the same
15 #include <asm/exception-64s.h>
18 * We layout physical memory as follows:
19 * 0x0000 - 0x00ff : Secondary processor spin code
20 * 0x0100 - 0x2fff : pSeries Interrupt prologs
21 * 0x3000 - 0x5fff : interrupt support, iSeries and common interrupt prologs
22 * 0x6000 - 0x6fff : Initial (CPU0) segment table
23 * 0x7000 - 0x7fff : FWNMI data area
24 * 0x8000 - : Early init and support code
28 * This is the start of the interrupt handlers for pSeries
29 * This code runs with relocation off.
30 * Code from here to __end_interrupts gets copied down to real
31 * address 0x100 when we are running a relocatable kernel.
32 * Therefore any relative branches in this section must only
33 * branch to labels in this section.
36 .globl __start_interrupts
39 STD_EXCEPTION_PSERIES(0x100, system_reset)
42 _machine_check_pSeries:
45 mtspr SPRN_SPRG_SCRATCH0,r13 /* save r13 */
46 EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common)
49 .globl data_access_pSeries
53 mtspr SPRN_SPRG_SCRATCH0,r13
55 mfspr r13,SPRN_SPRG_PACA
56 std r9,PACA_EXSLB+EX_R9(r13)
57 std r10,PACA_EXSLB+EX_R10(r13)
64 beq do_stab_bolted_pSeries
65 ld r10,PACA_EXSLB+EX_R10(r13)
66 std r11,PACA_EXGEN+EX_R11(r13)
67 ld r11,PACA_EXSLB+EX_R9(r13)
68 std r12,PACA_EXGEN+EX_R12(r13)
69 mfspr r12,SPRN_SPRG_SCRATCH0
70 std r10,PACA_EXGEN+EX_R10(r13)
71 std r11,PACA_EXGEN+EX_R9(r13)
72 std r12,PACA_EXGEN+EX_R13(r13)
73 EXCEPTION_PROLOG_PSERIES_1(data_access_common)
75 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, data_access_common)
76 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_SLB)
79 .globl data_access_slb_pSeries
80 data_access_slb_pSeries:
83 mtspr SPRN_SPRG_SCRATCH0,r13
84 mfspr r13,SPRN_SPRG_PACA /* get paca address into r13 */
85 std r3,PACA_EXSLB+EX_R3(r13)
87 std r9,PACA_EXSLB+EX_R9(r13) /* save r9 - r12 */
90 /* Keep that around for when we re-implement dynamic VSIDs */
92 bge slb_miss_user_pseries
93 #endif /* __DISABLED__ */
94 std r10,PACA_EXSLB+EX_R10(r13)
95 std r11,PACA_EXSLB+EX_R11(r13)
96 std r12,PACA_EXSLB+EX_R12(r13)
97 mfspr r10,SPRN_SPRG_SCRATCH0
98 std r10,PACA_EXSLB+EX_R13(r13)
99 mfspr r12,SPRN_SRR1 /* and SRR1 */
100 #ifndef CONFIG_RELOCATABLE
104 * We can't just use a direct branch to .slb_miss_realmode
105 * because the distance from here to there depends on where
106 * the kernel ends up being put.
109 ld r10,PACAKBASE(r13)
110 LOAD_HANDLER(r10, .slb_miss_realmode)
115 STD_EXCEPTION_PSERIES(0x400, instruction_access)
118 .globl instruction_access_slb_pSeries
119 instruction_access_slb_pSeries:
122 mtspr SPRN_SPRG_SCRATCH0,r13
123 mfspr r13,SPRN_SPRG_PACA /* get paca address into r13 */
124 std r3,PACA_EXSLB+EX_R3(r13)
125 mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */
126 std r9,PACA_EXSLB+EX_R9(r13) /* save r9 - r12 */
129 /* Keep that around for when we re-implement dynamic VSIDs */
131 bge slb_miss_user_pseries
132 #endif /* __DISABLED__ */
133 std r10,PACA_EXSLB+EX_R10(r13)
134 std r11,PACA_EXSLB+EX_R11(r13)
135 std r12,PACA_EXSLB+EX_R12(r13)
136 mfspr r10,SPRN_SPRG_SCRATCH0
137 std r10,PACA_EXSLB+EX_R13(r13)
138 mfspr r12,SPRN_SRR1 /* and SRR1 */
139 #ifndef CONFIG_RELOCATABLE
143 ld r10,PACAKBASE(r13)
144 LOAD_HANDLER(r10, .slb_miss_realmode)
149 MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt)
150 STD_EXCEPTION_PSERIES(0x600, alignment)
151 STD_EXCEPTION_PSERIES(0x700, program_check)
152 STD_EXCEPTION_PSERIES(0x800, fp_unavailable)
153 MASKABLE_EXCEPTION_PSERIES(0x900, decrementer)
154 STD_EXCEPTION_PSERIES(0xa00, trap_0a)
155 STD_EXCEPTION_PSERIES(0xb00, trap_0b)
158 .globl system_call_pSeries
165 END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE)
167 mfspr r13,SPRN_SPRG_PACA
169 ld r12,PACAKBASE(r13)
171 LOAD_HANDLER(r12, system_call_entry)
176 b . /* prevent speculative execution */
178 /* Fast LE/BE switch system call */
179 1: mfspr r12,SPRN_SRR1
182 rfid /* return to userspace */
185 STD_EXCEPTION_PSERIES(0xd00, single_step)
186 STD_EXCEPTION_PSERIES(0xe00, trap_0e)
188 /* We need to deal with the Altivec unavailable exception
189 * here which is at 0xf20, thus in the middle of the
190 * prolog code of the PerformanceMonitor one. A little
191 * trickery is thus necessary
193 performance_monitor_pSeries_1:
196 b performance_monitor_pSeries
198 altivec_unavailable_pSeries_1:
201 b altivec_unavailable_pSeries
203 vsx_unavailable_pSeries_1:
206 b vsx_unavailable_pSeries
208 #ifdef CONFIG_CBE_RAS
209 HSTD_EXCEPTION_PSERIES(0x1200, cbe_system_error)
210 #endif /* CONFIG_CBE_RAS */
211 STD_EXCEPTION_PSERIES(0x1300, instruction_breakpoint)
212 #ifdef CONFIG_CBE_RAS
213 HSTD_EXCEPTION_PSERIES(0x1600, cbe_maintenance)
214 #endif /* CONFIG_CBE_RAS */
215 STD_EXCEPTION_PSERIES(0x1700, altivec_assist)
216 #ifdef CONFIG_CBE_RAS
217 HSTD_EXCEPTION_PSERIES(0x1800, cbe_thermal)
218 #endif /* CONFIG_CBE_RAS */
222 /*** pSeries interrupt support ***/
224 /* moved from 0xf00 */
225 STD_EXCEPTION_PSERIES(., performance_monitor)
226 STD_EXCEPTION_PSERIES(., altivec_unavailable)
227 STD_EXCEPTION_PSERIES(., vsx_unavailable)
230 * An interrupt came in while soft-disabled; clear EE in SRR1,
231 * clear paca->hard_enabled and return.
234 stb r10,PACAHARDIRQEN(r13)
236 ld r9,PACA_EXGEN+EX_R9(r13)
238 rldicl r10,r10,48,1 /* clear MSR_EE */
241 ld r10,PACA_EXGEN+EX_R10(r13)
242 mfspr r13,SPRN_SPRG_SCRATCH0
247 do_stab_bolted_pSeries:
248 std r11,PACA_EXSLB+EX_R11(r13)
249 std r12,PACA_EXSLB+EX_R12(r13)
250 mfspr r10,SPRN_SPRG_SCRATCH0
251 std r10,PACA_EXSLB+EX_R13(r13)
252 EXCEPTION_PROLOG_PSERIES_1(.do_stab_bolted)
254 #ifdef CONFIG_PPC_PSERIES
256 * Vectors for the FWNMI option. Share common code.
258 .globl system_reset_fwnmi
262 mtspr SPRN_SPRG_SCRATCH0,r13 /* save r13 */
263 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common)
265 .globl machine_check_fwnmi
269 mtspr SPRN_SPRG_SCRATCH0,r13 /* save r13 */
270 EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common)
272 #endif /* CONFIG_PPC_PSERIES */
276 * This is used for when the SLB miss handler has to go virtual,
277 * which doesn't happen for now anymore but will once we re-implement
278 * dynamic VSIDs for shared page tables
280 slb_miss_user_pseries:
281 std r10,PACA_EXGEN+EX_R10(r13)
282 std r11,PACA_EXGEN+EX_R11(r13)
283 std r12,PACA_EXGEN+EX_R12(r13)
284 mfspr r10,SPRG_SCRATCH0
285 ld r11,PACA_EXSLB+EX_R9(r13)
286 ld r12,PACA_EXSLB+EX_R3(r13)
287 std r10,PACA_EXGEN+EX_R13(r13)
288 std r11,PACA_EXGEN+EX_R9(r13)
289 std r12,PACA_EXGEN+EX_R3(r13)
292 mfspr r11,SRR0 /* save SRR0 */
293 ori r12,r12,slb_miss_user_common@l /* virt addr of handler */
294 ori r10,r10,MSR_IR|MSR_DR|MSR_RI
296 mfspr r12,SRR1 /* and SRR1 */
299 b . /* prevent spec. execution */
300 #endif /* __DISABLED__ */
302 /* KVM's trampoline code needs to be close to the interrupt handlers */
304 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
305 #include "../kvm/book3s_rmhandlers.S"
309 .globl __end_interrupts
313 * Code from here down to __end_handlers is invoked from the
314 * exception prologs above. Because the prologs assemble the
315 * addresses of these handlers using the LOAD_HANDLER macro,
316 * which uses an addi instruction, these handlers must be in
317 * the first 32k of the kernel image.
320 /*** Common interrupt handlers ***/
322 STD_EXCEPTION_COMMON(0x100, system_reset, .system_reset_exception)
325 * Machine check is different because we use a different
326 * save area: PACA_EXMC instead of PACA_EXGEN.
329 .globl machine_check_common
330 machine_check_common:
331 EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC)
335 addi r3,r1,STACK_FRAME_OVERHEAD
336 bl .machine_check_exception
339 STD_EXCEPTION_COMMON_LITE(0x900, decrementer, .timer_interrupt)
340 STD_EXCEPTION_COMMON(0xa00, trap_0a, .unknown_exception)
341 STD_EXCEPTION_COMMON(0xb00, trap_0b, .unknown_exception)
342 STD_EXCEPTION_COMMON(0xd00, single_step, .single_step_exception)
343 STD_EXCEPTION_COMMON(0xe00, trap_0e, .unknown_exception)
344 STD_EXCEPTION_COMMON_IDLE(0xf00, performance_monitor, .performance_monitor_exception)
345 STD_EXCEPTION_COMMON(0x1300, instruction_breakpoint, .instruction_breakpoint_exception)
346 #ifdef CONFIG_ALTIVEC
347 STD_EXCEPTION_COMMON(0x1700, altivec_assist, .altivec_assist_exception)
349 STD_EXCEPTION_COMMON(0x1700, altivec_assist, .unknown_exception)
351 #ifdef CONFIG_CBE_RAS
352 STD_EXCEPTION_COMMON(0x1200, cbe_system_error, .cbe_system_error_exception)
353 STD_EXCEPTION_COMMON(0x1600, cbe_maintenance, .cbe_maintenance_exception)
354 STD_EXCEPTION_COMMON(0x1800, cbe_thermal, .cbe_thermal_exception)
355 #endif /* CONFIG_CBE_RAS */
362 * Here we have detected that the kernel stack pointer is bad.
363 * R9 contains the saved CR, r13 points to the paca,
364 * r10 contains the (bad) kernel stack pointer,
365 * r11 and r12 contain the saved SRR0 and SRR1.
366 * We switch to using an emergency stack, save the registers there,
367 * and call kernel_bad_stack(), which panics.
370 ld r1,PACAEMERGSP(r13)
371 subi r1,r1,64+INT_FRAME_SIZE
392 lhz r12,PACA_TRAP_SAVE(r13)
394 addi r11,r1,INT_FRAME_SIZE
399 1: addi r3,r1,STACK_FRAME_OVERHEAD
404 * Here r13 points to the paca, r9 contains the saved CR,
405 * SRR0 and SRR1 are saved in r11 and r12,
406 * r9 - r13 are saved in paca->exgen.
409 .globl data_access_common
412 std r10,PACA_EXGEN+EX_DAR(r13)
414 stw r10,PACA_EXGEN+EX_DSISR(r13)
415 EXCEPTION_PROLOG_COMMON(0x300, PACA_EXGEN)
416 ld r3,PACA_EXGEN+EX_DAR(r13)
417 lwz r4,PACA_EXGEN+EX_DSISR(r13)
419 b .do_hash_page /* Try to handle as hpte fault */
422 .globl instruction_access_common
423 instruction_access_common:
424 EXCEPTION_PROLOG_COMMON(0x400, PACA_EXGEN)
428 b .do_hash_page /* Try to handle as hpte fault */
431 * Here is the common SLB miss user that is used when going to virtual
432 * mode for SLB misses, that is currently not used
436 .globl slb_miss_user_common
437 slb_miss_user_common:
439 std r3,PACA_EXGEN+EX_DAR(r13)
440 stw r9,PACA_EXGEN+EX_CCR(r13)
441 std r10,PACA_EXGEN+EX_LR(r13)
442 std r11,PACA_EXGEN+EX_SRR0(r13)
443 bl .slb_allocate_user
445 ld r10,PACA_EXGEN+EX_LR(r13)
446 ld r3,PACA_EXGEN+EX_R3(r13)
447 lwz r9,PACA_EXGEN+EX_CCR(r13)
448 ld r11,PACA_EXGEN+EX_SRR0(r13)
452 andi. r10,r12,MSR_RI /* check for unrecoverable exception */
453 beq- unrecov_user_slb
461 clrrdi r10,r10,2 /* clear RI before setting SRR0/1 */
467 ld r9,PACA_EXGEN+EX_R9(r13)
468 ld r10,PACA_EXGEN+EX_R10(r13)
469 ld r11,PACA_EXGEN+EX_R11(r13)
470 ld r12,PACA_EXGEN+EX_R12(r13)
471 ld r13,PACA_EXGEN+EX_R13(r13)
476 EXCEPTION_PROLOG_COMMON(0x380, PACA_EXGEN)
477 ld r4,PACA_EXGEN+EX_DAR(r13)
484 EXCEPTION_PROLOG_COMMON(0x4200, PACA_EXGEN)
487 1: addi r3,r1,STACK_FRAME_OVERHEAD
488 bl .unrecoverable_exception
491 #endif /* __DISABLED__ */
495 * r13 points to the PACA, r9 contains the saved CR,
496 * r12 contain the saved SRR1, SRR0 is still ready for return
497 * r3 has the faulting address
498 * r9 - r13 are saved in paca->exslb.
499 * r3 is saved in paca->slb_r3
500 * We assume we aren't going to take any exceptions during this procedure.
502 _GLOBAL(slb_miss_realmode)
504 #ifdef CONFIG_RELOCATABLE
508 stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */
509 std r10,PACA_EXSLB+EX_LR(r13) /* save LR */
511 bl .slb_allocate_realmode
513 /* All done -- return from exception. */
515 ld r10,PACA_EXSLB+EX_LR(r13)
516 ld r3,PACA_EXSLB+EX_R3(r13)
517 lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */
518 #ifdef CONFIG_PPC_ISERIES
520 ld r11,PACALPPACAPTR(r13)
521 ld r11,LPPACASRR0(r11) /* get SRR0 value */
522 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
523 #endif /* CONFIG_PPC_ISERIES */
527 andi. r10,r12,MSR_RI /* check for unrecoverable exception */
533 mtcrf 0x01,r9 /* slb_allocate uses cr0 and cr7 */
536 #ifdef CONFIG_PPC_ISERIES
540 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
541 #endif /* CONFIG_PPC_ISERIES */
542 ld r9,PACA_EXSLB+EX_R9(r13)
543 ld r10,PACA_EXSLB+EX_R10(r13)
544 ld r11,PACA_EXSLB+EX_R11(r13)
545 ld r12,PACA_EXSLB+EX_R12(r13)
546 ld r13,PACA_EXSLB+EX_R13(r13)
548 b . /* prevent speculative execution */
551 #ifdef CONFIG_PPC_ISERIES
554 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
555 #endif /* CONFIG_PPC_ISERIES */
557 ld r10,PACAKBASE(r13)
558 LOAD_HANDLER(r10,unrecov_slb)
566 EXCEPTION_PROLOG_COMMON(0x4100, PACA_EXSLB)
569 1: addi r3,r1,STACK_FRAME_OVERHEAD
570 bl .unrecoverable_exception
574 .globl hardware_interrupt_common
575 .globl hardware_interrupt_entry
576 hardware_interrupt_common:
577 EXCEPTION_PROLOG_COMMON(0x500, PACA_EXGEN)
579 hardware_interrupt_entry:
582 bl .ppc64_runlatch_on
583 END_FTR_SECTION_IFSET(CPU_FTR_CTRL)
584 addi r3,r1,STACK_FRAME_OVERHEAD
586 b .ret_from_except_lite
588 #ifdef CONFIG_PPC_970_NAP
591 std r9,TI_LOCAL_FLAGS(r11)
592 ld r10,_LINK(r1) /* make idle task do the */
593 std r10,_NIP(r1) /* equivalent of a blr */
598 .globl alignment_common
601 std r10,PACA_EXGEN+EX_DAR(r13)
603 stw r10,PACA_EXGEN+EX_DSISR(r13)
604 EXCEPTION_PROLOG_COMMON(0x600, PACA_EXGEN)
605 ld r3,PACA_EXGEN+EX_DAR(r13)
606 lwz r4,PACA_EXGEN+EX_DSISR(r13)
610 addi r3,r1,STACK_FRAME_OVERHEAD
612 bl .alignment_exception
616 .globl program_check_common
617 program_check_common:
618 EXCEPTION_PROLOG_COMMON(0x700, PACA_EXGEN)
620 addi r3,r1,STACK_FRAME_OVERHEAD
622 bl .program_check_exception
626 .globl fp_unavailable_common
627 fp_unavailable_common:
628 EXCEPTION_PROLOG_COMMON(0x800, PACA_EXGEN)
629 bne 1f /* if from user, just load it up */
631 addi r3,r1,STACK_FRAME_OVERHEAD
633 bl .kernel_fp_unavailable_exception
636 b fast_exception_return
639 .globl altivec_unavailable_common
640 altivec_unavailable_common:
641 EXCEPTION_PROLOG_COMMON(0xf20, PACA_EXGEN)
642 #ifdef CONFIG_ALTIVEC
646 b fast_exception_return
648 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
651 addi r3,r1,STACK_FRAME_OVERHEAD
653 bl .altivec_unavailable_exception
657 .globl vsx_unavailable_common
658 vsx_unavailable_common:
659 EXCEPTION_PROLOG_COMMON(0xf40, PACA_EXGEN)
664 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
667 addi r3,r1,STACK_FRAME_OVERHEAD
669 bl .vsx_unavailable_exception
673 .globl __end_handlers
677 * Return from an exception with minimal checks.
678 * The caller is assumed to have done EXCEPTION_PROLOG_COMMON.
679 * If interrupts have been enabled, or anything has been
680 * done that might have changed the scheduling status of
681 * any task or sent any task a signal, you should use
682 * ret_from_except or ret_from_except_lite instead of this.
684 fast_exc_return_irq: /* restores irq state too */
686 TRACE_AND_RESTORE_IRQ(r3);
688 rldicl r4,r12,49,63 /* get MSR_EE to LSB */
689 stb r4,PACAHARDIRQEN(r13) /* restore paca->hard_enabled */
692 .globl fast_exception_return
693 fast_exception_return:
696 andi. r3,r12,MSR_RI /* check if RI is set */
699 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
702 ACCOUNT_CPU_USER_EXIT(r3, r4)
718 rldicl r10,r10,48,1 /* clear EE */
719 rldicr r10,r10,16,61 /* clear RI (LE is 0 already) */
727 b . /* prevent speculative execution */
731 1: addi r3,r1,STACK_FRAME_OVERHEAD
732 bl .unrecoverable_exception
740 _STATIC(do_hash_page)
744 andis. r0,r4,0xa410 /* weird error? */
745 bne- handle_page_fault /* if not, try to insert a HPTE */
746 andis. r0,r4,DSISR_DABRMATCH@h
747 bne- handle_dabr_fault
750 andis. r0,r4,0x0020 /* Is it a segment table fault? */
751 bne- do_ste_alloc /* If so handle it */
752 END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
754 clrrdi r11,r1,THREAD_SHIFT
755 lwz r0,TI_PREEMPT(r11) /* If we're in an "NMI" */
756 andis. r0,r0,NMI_MASK@h /* (i.e. an irq when soft-disabled) */
757 bne 77f /* then don't call hash_page now */
760 * On iSeries, we soft-disable interrupts here, then
761 * hard-enable interrupts so that the hash_page code can spin on
762 * the hash_table_lock without problems on a shared processor.
767 * Currently, trace_hardirqs_off() will be called by DISABLE_INTS
768 * and will clobber volatile registers when irq tracing is enabled
769 * so we need to reload them. It may be possible to be smarter here
770 * and move the irq tracing elsewhere but let's keep it simple for
773 #ifdef CONFIG_TRACE_IRQFLAGS
779 #endif /* CONFIG_TRACE_IRQFLAGS */
781 * We need to set the _PAGE_USER bit if MSR_PR is set or if we are
782 * accessing a userspace segment (even from the kernel). We assume
783 * kernel addresses always have the high bit set.
785 rlwinm r4,r4,32-25+9,31-9,31-9 /* DSISR_STORE -> _PAGE_RW */
786 rotldi r0,r3,15 /* Move high bit into MSR_PR posn */
787 orc r0,r12,r0 /* MSR_PR | ~high_bit */
788 rlwimi r4,r0,32-13,30,30 /* becomes _PAGE_USER access bit */
789 ori r4,r4,1 /* add _PAGE_PRESENT */
790 rlwimi r4,r5,22+2,31-2,31-2 /* Set _PAGE_EXEC if trap is 0x400 */
793 * r3 contains the faulting address
794 * r4 contains the required access permissions
795 * r5 contains the trap number
797 * at return r3 = 0 for success
799 bl .hash_page /* build HPTE if possible */
800 cmpdi r3,0 /* see if hash_page succeeded */
804 * If we had interrupts soft-enabled at the point where the
805 * DSI/ISI occurred, and an interrupt came in during hash_page,
807 * We jump to ret_from_except_lite rather than fast_exception_return
808 * because ret_from_except_lite will check for and handle pending
809 * interrupts if necessary.
812 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
816 * Here we have interrupts hard-disabled, so it is sufficient
817 * to restore paca->{soft,hard}_enable and get out.
819 beq fast_exc_return_irq /* Return from exception on success */
820 END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)
822 /* For a hash failure, we don't bother re-enabling interrupts */
826 * hash_page couldn't handle it, set soft interrupt enable back
827 * to what it was before the trap. Note that .arch_local_irq_restore
828 * handles any interrupts pending at this point.
831 TRACE_AND_RESTORE_IRQ_PARTIAL(r3, 11f)
832 bl .arch_local_irq_restore
835 /* We have a data breakpoint exception - handle it */
840 addi r3,r1,STACK_FRAME_OVERHEAD
842 b .ret_from_except_lite
844 /* Here we have a page fault that hash_page can't handle. */
849 addi r3,r1,STACK_FRAME_OVERHEAD
855 addi r3,r1,STACK_FRAME_OVERHEAD
860 13: b .ret_from_except_lite
862 /* We have a page fault that hash_page could handle but HV refused
867 addi r3,r1,STACK_FRAME_OVERHEAD
873 * We come here as a result of a DSI at a point where we don't want
874 * to call hash_page, such as when we are accessing memory (possibly
875 * user memory) inside a PMU interrupt that occurred while interrupts
876 * were soft-disabled. We want to invoke the exception handler for
877 * the access, or panic if there isn't a handler.
881 addi r3,r1,STACK_FRAME_OVERHEAD
886 /* here we have a segment miss */
888 bl .ste_allocate /* try to insert stab entry */
890 bne- handle_page_fault
891 b fast_exception_return
894 * r13 points to the PACA, r9 contains the saved CR,
895 * r11 and r12 contain the saved SRR0 and SRR1.
896 * r9 - r13 are saved in paca->exslb.
897 * We assume we aren't going to take any exceptions during this procedure.
898 * We assume (DAR >> 60) == 0xc.
901 _GLOBAL(do_stab_bolted)
902 stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */
903 std r11,PACA_EXSLB+EX_SRR0(r13) /* save SRR0 in exc. frame */
905 /* Hash to the primary group */
906 ld r10,PACASTABVIRT(r13)
909 rldimi r10,r11,7,52 /* r10 = first ste of the group */
912 /* This is a kernel address, so protovsid = ESID */
913 ASM_VSID_SCRAMBLE(r11, r9, 256M)
914 rldic r9,r11,12,16 /* r9 = vsid << 12 */
916 /* Search the primary group for a free entry */
917 1: ld r11,0(r10) /* Test valid bit of the current ste */
924 /* Stick for only searching the primary group for now. */
925 /* At least for now, we use a very simple random castout scheme */
926 /* Use the TB as a random number ; OR in 1 to avoid entry 0 */
928 rldic r11,r11,4,57 /* r11 = (r11 << 4) & 0x70 */
931 /* r10 currently points to an ste one past the group of interest */
932 /* make it point to the randomly selected entry */
934 or r10,r10,r11 /* r10 is the entry to invalidate */
936 isync /* mark the entry invalid */
938 rldicl r11,r11,56,1 /* clear the valid bit */
943 clrrdi r11,r11,28 /* Get the esid part of the ste */
946 2: std r9,8(r10) /* Store the vsid part of the ste */
949 mfspr r11,SPRN_DAR /* Get the new esid */
950 clrrdi r11,r11,28 /* Permits a full 32b of ESID */
951 ori r11,r11,0x90 /* Turn on valid and kp */
952 std r11,0(r10) /* Put new entry back into the stab */
956 /* All done -- return from exception. */
957 lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */
958 ld r11,PACA_EXSLB+EX_SRR0(r13) /* get saved SRR0 */
963 mtcrf 0x80,r9 /* restore CR */
971 ld r9,PACA_EXSLB+EX_R9(r13)
972 ld r10,PACA_EXSLB+EX_R10(r13)
973 ld r11,PACA_EXSLB+EX_R11(r13)
974 ld r12,PACA_EXSLB+EX_R12(r13)
975 ld r13,PACA_EXSLB+EX_R13(r13)
977 b . /* prevent speculative execution */
980 * Space for CPU0's segment table.
982 * On iSeries, the hypervisor must fill in at least one entry before
983 * we get control (with relocate on). The address is given to the hv
984 * as a page number (see xLparMap below), so this must be at a
985 * fixed address (the linker can't compute (u64)&initial_stab >>
988 . = STAB0_OFFSET /* 0x6000 */
993 #ifdef CONFIG_PPC_PSERIES
995 * Data area reserved for FWNMI option.
996 * This address (0x7000) is fixed by the RPA.
999 .globl fwnmi_data_area
1001 #endif /* CONFIG_PPC_PSERIES */
1003 /* iSeries does not use the FWNMI stuff, so it is safe to put
1004 * this here, even if we later allow kernels that will boot on
1005 * both pSeries and iSeries */
1006 #ifdef CONFIG_PPC_ISERIES
1010 .quad HvEsidsToMap /* xNumberEsids */
1011 .quad HvRangesToMap /* xNumberRanges */
1012 .quad STAB0_PAGE /* xSegmentTableOffs */
1013 .zero 40 /* xRsvd */
1014 /* xEsids (HvEsidsToMap entries of 2 quads) */
1015 .quad PAGE_OFFSET_ESID /* xKernelEsid */
1016 .quad PAGE_OFFSET_VSID /* xKernelVsid */
1017 .quad VMALLOC_START_ESID /* xKernelEsid */
1018 .quad VMALLOC_START_VSID /* xKernelVsid */
1019 /* xRanges (HvRangesToMap entries of 3 quads) */
1020 .quad HvPagesToMap /* xPages */
1021 .quad 0 /* xOffset */
1022 .quad PAGE_OFFSET_VSID << (SID_SHIFT - HW_PAGE_SHIFT) /* xVPN */
1024 #endif /* CONFIG_PPC_ISERIES */
1026 #ifdef CONFIG_PPC_PSERIES
1028 #endif /* CONFIG_PPC_PSERIES */