2 * Copyright (C) 2004-2006 Atmel Corporation
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
10 * This file contains the low-level entry-points into the kernel, that is,
11 * exception handlers, debug trap handlers, interrupt handlers and the
12 * system call handler.
14 #include <linux/errno.h>
17 #include <asm/hardirq.h>
21 #include <asm/pgtable.h>
22 #include <asm/ptrace.h>
23 #include <asm/sysreg.h>
24 #include <asm/thread_info.h>
25 #include <asm/unistd.h>
28 # define preempt_stop mask_interrupts
31 # define fault_resume_kernel fault_restore_all
34 #define __MASK(x) ((1 << (x)) - 1)
35 #define IRQ_MASK ((__MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT) | \
36 (__MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT))
38 .section .ex.text,"ax",@progbits
45 bral do_bus_error_write
47 bral do_bus_error_read
51 bral handle_address_fault
53 bral handle_protection_fault
57 bral do_illegal_opcode_ll
59 bral do_illegal_opcode_ll
61 bral do_illegal_opcode_ll
65 bral do_illegal_opcode_ll
67 bral handle_address_fault
69 bral handle_address_fault
71 bral handle_protection_fault
73 bral handle_protection_fault
79 * r1 : Offending address
80 * r2 : Scratch register
81 * r3 : Cause (5, 12 or 13)
83 #define tlbmiss_save pushm r0-r3
84 #define tlbmiss_restore popm r0-r3
101 .global tlb_miss_common
104 mfsr r0, SYSREG_TLBEAR
107 /* Is it the vmalloc space? */
109 brcs handle_vmalloc_miss
111 /* First level lookup */
113 lsr r2, r0, PGDIR_SHIFT
115 bfextu r1, r0, PAGE_SHIFT, PGDIR_SHIFT - PAGE_SHIFT
116 bld r3, _PAGE_BIT_PRESENT
117 brcc page_table_not_present
119 /* Translate to virtual address in P1. */
123 /* Second level lookup */
125 mfsr r0, SYSREG_TLBARLO
126 bld r2, _PAGE_BIT_PRESENT
127 brcc page_not_present
129 /* Mark the page as accessed */
130 sbr r2, _PAGE_BIT_ACCESSED
133 /* Drop software flags */
134 andl r2, _PAGE_FLAGS_HARDWARE_MASK & 0xffff
135 mtsr SYSREG_TLBELO, r2
137 /* Figure out which entry we want to replace */
138 mfsr r1, SYSREG_MMUCR
141 mov r3, -1 /* All entries have been accessed, */
142 mov r2, 0 /* so start at 0 */
143 mtsr SYSREG_TLBARLO, r3 /* and reset TLBAR */
145 1: bfins r1, r2, SYSREG_DRP_OFFSET, SYSREG_DRP_SIZE
146 mtsr SYSREG_MMUCR, r1
153 /* Simply do the lookup in init's page table */
154 mov r1, lo(swapper_pg_dir)
155 orh r1, hi(swapper_pg_dir)
159 /* --- System Call --- */
163 #ifdef CONFIG_PREEMPT
166 pushm r12 /* r12_orig */
169 mfsr r0, SYSREG_RAR_SUP
170 mfsr r1, SYSREG_RSR_SUP
171 #ifdef CONFIG_PREEMPT
177 /* check for syscall tracing */
179 ld.w r1, r0[TI_flags]
180 bld r1, TIF_SYSCALL_TRACE
181 brcs syscall_trace_enter
187 lddpc lr, syscall_table_addr
189 mov r8, r5 /* 5th argument (6th is pushed by stub) */
192 .global syscall_return
195 mask_interrupts /* make sure we don't miss an interrupt
196 setting need_resched or sigpending
197 between sampling and the rets */
199 /* Store the return value so that the correct value is loaded below */
200 stdsp sp[REG_R12], r12
202 ld.w r1, r0[TI_flags]
203 andl r1, _TIF_ALLWORK_MASK, COH
204 brne syscall_exit_work
208 mtsr SYSREG_RAR_SUP, r8
209 mtsr SYSREG_RSR_SUP, r9
211 sub sp, -4 /* r12_orig */
222 .global ret_from_fork
226 /* check for syscall tracing */
228 ld.w r1, r0[TI_flags]
229 andl r1, _TIF_ALLWORK_MASK, COH
230 brne syscall_exit_work
231 rjmp syscall_exit_cont
237 rjmp syscall_trace_cont
240 bld r1, TIF_SYSCALL_TRACE
245 ld.w r1, r0[TI_flags]
247 1: bld r1, TIF_NEED_RESCHED
252 ld.w r1, r0[TI_flags]
255 2: mov r2, _TIF_SIGPENDING | _TIF_RESTORE_SIGMASK
261 rcall do_notify_resume
263 ld.w r1, r0[TI_flags]
266 3: bld r1, TIF_BREAKPOINT
267 brcc syscall_exit_cont
268 rjmp enter_monitor_mode
270 /* The slow path of the TLB miss handler */
271 page_table_not_present:
276 rcall save_full_context_ex
280 rjmp ret_from_exception
282 /* This function expects to find offending PC in SYSREG_RAR_EX */
283 .type save_full_context_ex, @function
285 save_full_context_ex:
286 mfsr r11, SYSREG_RAR_EX
287 sub r9, pc, . - debug_trampoline
288 mfsr r8, SYSREG_RSR_EX
292 andh r8, (MODE_MASK >> 16), COH
295 1: pushm r11, r12 /* PC and SR */
299 2: sub r10, sp, -(FRAME_SIZE_FULL - REG_LR)
300 stdsp sp[4], r10 /* replace saved SP */
304 * The debug handler set up a trampoline to make us
305 * automatically enter monitor mode upon return, but since
306 * we're saving the full context, we must assume that the
307 * exception handler might want to alter the return address
308 * and/or status register. So we need to restore the original
309 * context and enter monitor mode manually after the exception
312 3: get_thread_info r8
313 ld.w r11, r8[TI_rar_saved]
314 ld.w r12, r8[TI_rsr_saved]
316 .size save_full_context_ex, . - save_full_context_ex
318 /* Low-level exception handlers */
322 rcall save_full_context_ex
325 rcall do_critical_exception
327 /* We should never get here... */
329 sub r12, pc, (. - 1f)
332 1: .asciz "Return from critical exception!"
338 rcall save_full_context_ex
345 rcall save_full_context_ex
347 1: mfsr r12, SYSREG_BEAR
350 rjmp ret_from_exception
356 mfsr r9, SYSREG_RSR_NMI
357 mfsr r8, SYSREG_RAR_NMI
358 bfextu r0, r9, MODE_SHIFT, 3
361 1: pushm r8, r9 /* PC and SR */
366 mtsr SYSREG_RAR_NMI, r8
368 mtsr SYSREG_RSR_NMI, r9
372 sub sp, -4 /* skip r12_orig */
375 2: sub r10, sp, -(FRAME_SIZE_FULL - REG_LR)
376 stdsp sp[4], r10 /* replace saved SP */
380 sub sp, -4 /* skip sp */
382 sub sp, -4 /* skip r12_orig */
385 handle_address_fault:
388 rcall save_full_context_ex
391 rcall do_address_exception
392 rjmp ret_from_exception
394 handle_protection_fault:
397 rcall save_full_context_ex
401 rjmp ret_from_exception
404 do_illegal_opcode_ll:
407 rcall save_full_context_ex
410 rcall do_illegal_opcode
411 rjmp ret_from_exception
415 mfsr r1, SYSREG_TLBEAR
417 lsr r2, r1, PGDIR_SHIFT
419 lsl r1, (32 - PGDIR_SHIFT)
420 lsr r1, (32 - PGDIR_SHIFT) + PAGE_SHIFT
422 /* Translate to virtual address in P1 */
427 sbr r3, _PAGE_BIT_DIRTY
431 /* The page table is up-to-date. Update the TLB entry as well */
432 andl r0, lo(_PAGE_FLAGS_HARDWARE_MASK)
433 mtsr SYSREG_TLBELO, r0
435 /* MMUCR[DRP] is updated automatically, so let's go... */
444 rcall save_full_context_ex
449 rjmp ret_from_exception
455 andh r4, (MODE_MASK >> 16), COH
456 brne fault_resume_kernel
459 ld.w r1, r0[TI_flags]
460 andl r1, _TIF_WORK_MASK, COH
466 mtsr SYSREG_RAR_EX, r8
467 mtsr SYSREG_RSR_EX, r9
473 #ifdef CONFIG_PREEMPT
475 ld.w r2, r0[TI_preempt_count]
478 ld.w r1, r0[TI_flags]
479 bld r1, TIF_NEED_RESCHED
482 bld r4, SYSREG_GM_OFFSET
484 rcall preempt_schedule_irq
491 mtsr SYSREG_RAR_EX, r8
492 mtsr SYSREG_RSR_EX, r9
494 sub sp, -4 /* ignore SP */
496 sub sp, -4 /* ignore r12_orig */
500 /* Switch to exception mode so that we can share the same code. */
502 cbr r8, SYSREG_M0_OFFSET
503 orh r8, hi(SYSREG_BIT(M1) | SYSREG_BIT(M2))
507 ld.w r1, r0[TI_flags]
510 bld r1, TIF_NEED_RESCHED
515 ld.w r1, r0[TI_flags]
518 1: mov r2, _TIF_SIGPENDING | _TIF_RESTORE_SIGMASK
524 rcall do_notify_resume
526 ld.w r1, r0[TI_flags]
529 2: bld r1, TIF_BREAKPOINT
530 brcc fault_resume_user
531 rjmp enter_monitor_mode
533 .section .kprobes.text, "ax", @progbits
534 .type handle_debug, @function
536 sub sp, 4 /* r12_orig */
538 mfsr r8, SYSREG_RAR_DBG
539 mfsr r9, SYSREG_RSR_DBG
542 bfextu r9, r9, SYSREG_MODE_OFFSET, SYSREG_MODE_SIZE
543 brne debug_fixup_regs
546 #ifdef CONFIG_TRACE_IRQFLAGS
547 rcall trace_hardirqs_off
554 bfextu r3, r2, SYSREG_MODE_OFFSET, SYSREG_MODE_SIZE
555 brne debug_resume_kernel
558 ld.w r1, r0[TI_flags]
559 mov r2, _TIF_DBGWORK_MASK
563 bld r1, TIF_SINGLE_STEP
566 sbr r4, OCD_DC_SS_BIT
571 mtsr SYSREG_RSR_DBG, r11
572 mtsr SYSREG_RAR_DBG, r10
573 #ifdef CONFIG_TRACE_IRQFLAGS
574 rcall trace_hardirqs_on
580 .size handle_debug, . - handle_debug
582 /* Mode of the trapped context is in r9 */
583 .type debug_fixup_regs, @function
587 bfins r8, r9, SYSREG_MODE_OFFSET, SYSREG_MODE_SIZE
593 sub r8, sp, -FRAME_SIZE_FULL
595 rjmp .Ldebug_fixup_cont
596 .size debug_fixup_regs, . - debug_fixup_regs
598 .type debug_resume_kernel, @function
602 mtsr SYSREG_RAR_DBG, r10
603 mtsr SYSREG_RSR_DBG, r11
604 #ifdef CONFIG_TRACE_IRQFLAGS
605 bld r11, SYSREG_GM_OFFSET
607 rcall trace_hardirqs_on
612 bfins r2, r3, SYSREG_MODE_OFFSET, SYSREG_MODE_SIZE
618 sub sp, -4 /* skip SP */
622 .size debug_resume_kernel, . - debug_resume_kernel
624 .type debug_exit_work, @function
627 * We must return from Monitor Mode using a retd, and we must
628 * not schedule since that involves the D bit in SR getting
629 * cleared by something other than the debug hardware. This
630 * may cause undefined behaviour according to the Architecture
633 * So we fix up the return address and status and return to a
634 * stub below in Exception mode. From there, we can follow the
635 * normal exception return path.
637 * The real return address and status registers are stored on
638 * the stack in the way the exception return path understands,
639 * so no need to fix anything up there.
641 sub r8, pc, . - fault_exit_work
642 mtsr SYSREG_RAR_DBG, r8
644 orh r9, hi(SR_EM | SR_GM | MODE_EXCEPTION)
645 mtsr SYSREG_RSR_DBG, r9
648 .size debug_exit_work, . - debug_exit_work
650 .set rsr_int0, SYSREG_RSR_INT0
651 .set rsr_int1, SYSREG_RSR_INT1
652 .set rsr_int2, SYSREG_RSR_INT2
653 .set rsr_int3, SYSREG_RSR_INT3
654 .set rar_int0, SYSREG_RAR_INT0
655 .set rar_int1, SYSREG_RAR_INT1
656 .set rar_int2, SYSREG_RAR_INT2
657 .set rar_int3, SYSREG_RAR_INT3
659 .macro IRQ_LEVEL level
660 .type irq_level\level, @function
662 sub sp, 4 /* r12_orig */
664 mfsr r8, rar_int\level
665 mfsr r9, rsr_int\level
667 #ifdef CONFIG_PREEMPT
668 sub r11, pc, (. - system_call)
681 bfextu r4, r4, SYSREG_M0_OFFSET, 3
682 cp.w r4, MODE_SUPERVISOR >> SYSREG_M0_OFFSET
684 cp.w r4, MODE_USER >> SYSREG_M0_OFFSET
685 #ifdef CONFIG_PREEMPT
692 ld.w r1, r0[TI_flags]
693 andl r1, _TIF_WORK_MASK, COH
697 #ifdef CONFIG_TRACE_IRQFLAGS
698 rcall trace_hardirqs_on
701 mtsr rar_int\level, r8
702 mtsr rsr_int\level, r9
704 sub sp, -4 /* ignore r12_orig */
707 #ifdef CONFIG_PREEMPT
709 mfsr r8, rsr_int\level
711 mtsr rsr_int\level, r8
713 sub sp, -4 /* ignore r12_orig */
717 2: get_thread_info r0
718 ld.w r1, r0[TI_flags]
719 bld r1, TIF_CPU_GOING_TO_SLEEP
720 #ifdef CONFIG_PREEMPT
725 sub r1, pc, . - cpu_idle_skip_sleep
727 #ifdef CONFIG_PREEMPT
728 3: get_thread_info r0
729 ld.w r2, r0[TI_preempt_count]
732 ld.w r1, r0[TI_flags]
733 bld r1, TIF_NEED_RESCHED
736 bld r4, SYSREG_GM_OFFSET
738 rcall preempt_schedule_irq
743 .section .irq.text,"ax",@progbits
754 .section .kprobes.text, "ax", @progbits
755 .type enter_monitor_mode, @function
758 * We need to enter monitor mode to do a single step. The
759 * monitor code will alter the return address so that we
760 * return directly to the user instead of returning here.
763 rjmp breakpoint_failed
765 .size enter_monitor_mode, . - enter_monitor_mode
767 .type debug_trampoline, @function
768 .global debug_trampoline
771 * Save the registers on the stack so that the monitor code
772 * can find them easily.
774 sub sp, 4 /* r12_orig */
777 ld.w r8, r0[TI_rar_saved]
778 ld.w r9, r0[TI_rsr_saved]
782 * The monitor code will alter the return address so we don't
786 rjmp breakpoint_failed
787 .size debug_trampoline, . - debug_trampoline
789 .type breakpoint_failed, @function
792 * Something went wrong. Perhaps the debug hardware isn't
795 lda.w r12, msg_breakpoint_failed
797 mov r10, 9 /* SIGKILL */
801 msg_breakpoint_failed:
802 .asciz "Failed to enter Debug Mode"