arch/powerpc/kernel/exceptions-64s.S

   1 /*
   2  * This file contains the 64-bit "server" PowerPC variant
   3  * of the low level exception handling including exception
   4  * vectors, exception return, part of the slb and stab
   5  * handling and other fixed offset specific things.
   6  *
   7  * This file is meant to be #included from head_64.S due to
   8  * position dependent assembly.
   9  *
  10  * Most of this originates from head_64.S and thus has the same
  11  * copyright history.
  12  *
  13  */
  14
  15 #include <asm/hw_irq.h>
  16 #include <asm/exception-64s.h>
  17 #include <asm/ptrace.h>
  18
  19 /*
  20  * We layout physical memory as follows:
  21  * 0x0000 - 0x00ff : Secondary processor spin code
  22  * 0x0100 - 0x17ff : pSeries Interrupt prologs
  23  * 0x1800 - 0x4000 : interrupt support common interrupt prologs
  24  * 0x4000 - 0x5fff : pSeries interrupts with IR=1,DR=1
  25  * 0x6000 - 0x6fff : more interrupt support including for IR=1,DR=1
  26  * 0x7000 - 0x7fff : FWNMI data area
  27  * 0x8000 - 0x8fff : Initial (CPU0) segment table
  28  * 0x9000 -        : Early init and support code
  29  */
  30         /* Syscall routine is used twice, in reloc-off and reloc-on paths */
  31 #define SYSCALL_PSERIES_1                                       \
  32 BEGIN_FTR_SECTION                                               \
  33         cmpdi   r0,0x1ebe ;                                     \
  34         beq-    1f ;                                            \
  35 END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE)                          \
  36         mr      r9,r13 ;                                        \
  37         GET_PACA(r13) ;                                         \
  38         mfspr   r11,SPRN_SRR0 ;                                 \
  39 0:
  40
  41 #define SYSCALL_PSERIES_2_RFID                                  \
  42         mfspr   r12,SPRN_SRR1 ;                                 \
  43         ld      r10,PACAKBASE(r13) ;                            \
  44         LOAD_HANDLER(r10, system_call_entry) ;                  \
  45         mtspr   SPRN_SRR0,r10 ;                                 \
  46         ld      r10,PACAKMSR(r13) ;                             \
  47         mtspr   SPRN_SRR1,r10 ;                                 \
  48         rfid ;                                                  \
  49         b       . ;     /* prevent speculative execution */
  50
  51 #define SYSCALL_PSERIES_3                                       \
  52         /* Fast LE/BE switch system call */                     \
  53 1:      mfspr   r12,SPRN_SRR1 ;                                 \
  54         xori    r12,r12,MSR_LE ;                                \
  55         mtspr   SPRN_SRR1,r12 ;                                 \
  56         rfid ;          /* return to userspace */               \
  57         b       . ;                                             \
  58 2:      mfspr   r12,SPRN_SRR1 ;                                 \
  59         andi.   r12,r12,MSR_PR ;                                \
  60         bne     0b ;                                            \
  61         mtspr   SPRN_SRR0,r3 ;                                  \
  62         mtspr   SPRN_SRR1,r4 ;                                  \
  63         mtspr   SPRN_SDR1,r5 ;                                  \
  64         rfid ;                                                  \
  65         b       . ;     /* prevent speculative execution */
  66
  67 #if defined(CONFIG_RELOCATABLE)
  68         /*
  69          * We can't branch directly; in the direct case we use LR
  70          * and system_call_entry restores LR.  (We thus need to move
  71          * LR to r10 in the RFID case too.)
  72          */
  73 #define SYSCALL_PSERIES_2_DIRECT                                \
  74         mflr    r10 ;                                           \
  75         ld      r12,PACAKBASE(r13) ;                            \
  76         LOAD_HANDLER(r12, system_call_entry_direct) ;           \
  77         mtlr    r12 ;                                           \
  78         mfspr   r12,SPRN_SRR1 ;                                 \
  79         /* Re-use of r13... No spare regs to do this */ \
  80         li      r13,MSR_RI ;                                    \
  81         mtmsrd  r13,1 ;                                         \
  82         GET_PACA(r13) ; /* get r13 back */                      \
  83         blr ;
  84 #else
  85         /* We can branch directly */
  86 #define SYSCALL_PSERIES_2_DIRECT                                \
  87         mfspr   r12,SPRN_SRR1 ;                                 \
  88         li      r10,MSR_RI ;                                    \
  89         mtmsrd  r10,1 ;                 /* Set RI (EE=0) */     \
  90         b       system_call_entry_direct ;
  91 #endif
  92
  93 /*
  94  * This is the start of the interrupt handlers for pSeries
  95  * This code runs with relocation off.
  96  * Code from here to __end_interrupts gets copied down to real
  97  * address 0x100 when we are running a relocatable kernel.
  98  * Therefore any relative branches in this section must only
  99  * branch to labels in this section.
 100  */
 101         . = 0x100
 102         .globl __start_interrupts
 103 __start_interrupts:
 104
 105         .globl system_reset_pSeries;
 106 system_reset_pSeries:
 107         HMT_MEDIUM_PPR_DISCARD
 108         SET_SCRATCH0(r13)
 109 #ifdef CONFIG_PPC_P7_NAP
 110 BEGIN_FTR_SECTION
 111         /* Running native on arch 2.06 or later, check if we are
 112          * waking up from nap. We only handle no state loss and
 113          * supervisor state loss. We do -not- handle hypervisor
 114          * state loss at this time.
 115          */
 116         mfspr   r13,SPRN_SRR1
 117         rlwinm. r13,r13,47-31,30,31
 118         beq     9f
 119
 120         /* waking up from powersave (nap) state */
 121         cmpwi   cr1,r13,2
 122         /* Total loss of HV state is fatal, we could try to use the
 123          * PIR to locate a PACA, then use an emergency stack etc...
 124          * but for now, let's just stay stuck here
 125          */
 126         bgt     cr1,.
 127         GET_PACA(r13)
 128
 129 #ifdef CONFIG_KVM_BOOK3S_64_HV
 130         li      r0,KVM_HWTHREAD_IN_KERNEL
 131         stb     r0,HSTATE_HWTHREAD_STATE(r13)
 132         /* Order setting hwthread_state vs. testing hwthread_req */
 133         sync
 134         lbz     r0,HSTATE_HWTHREAD_REQ(r13)
 135         cmpwi   r0,0
 136         beq     1f
 137         b       kvm_start_guest
 138 1:
 139 #endif
 140
 141         beq     cr1,2f
 142         b       .power7_wakeup_noloss
 143 2:      b       .power7_wakeup_loss
 144 9:
 145 END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
 146 #endif /* CONFIG_PPC_P7_NAP */
 147         EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD,
 148                                  NOTEST, 0x100)
 149
 150         . = 0x200
 151 machine_check_pSeries_1:
 152         /* This is moved out of line as it can be patched by FW, but
 153          * some code path might still want to branch into the original
 154          * vector
 155          */
 156         b       machine_check_pSeries
 157
 158         . = 0x300
 159         .globl data_access_pSeries
 160 data_access_pSeries:
 161         HMT_MEDIUM_PPR_DISCARD
 162         SET_SCRATCH0(r13)
 163 BEGIN_FTR_SECTION
 164         b       data_access_check_stab
 165 data_access_not_stab:
 166 END_MMU_FTR_SECTION_IFCLR(MMU_FTR_SLB)
 167         EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, data_access_common, EXC_STD,
 168                                  KVMTEST, 0x300)
 169
 170         . = 0x380
 171         .globl data_access_slb_pSeries
 172 data_access_slb_pSeries:
 173         HMT_MEDIUM_PPR_DISCARD
 174         SET_SCRATCH0(r13)
 175         EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST, 0x380)
 176         std     r3,PACA_EXSLB+EX_R3(r13)
 177         mfspr   r3,SPRN_DAR
 178 #ifdef __DISABLED__
 179         /* Keep that around for when we re-implement dynamic VSIDs */
 180         cmpdi   r3,0
 181         bge     slb_miss_user_pseries
 182 #endif /* __DISABLED__ */
 183         mfspr   r12,SPRN_SRR1
 184 #ifndef CONFIG_RELOCATABLE
 185         b       .slb_miss_realmode
 186 #else
 187         /*
 188          * We can't just use a direct branch to .slb_miss_realmode
 189          * because the distance from here to there depends on where
 190          * the kernel ends up being put.
 191          */
 192         mfctr   r11
 193         ld      r10,PACAKBASE(r13)
 194         LOAD_HANDLER(r10, .slb_miss_realmode)
 195         mtctr   r10
 196         bctr
 197 #endif
 198
 199         STD_EXCEPTION_PSERIES(0x400, 0x400, instruction_access)
 200
 201         . = 0x480
 202         .globl instruction_access_slb_pSeries
 203 instruction_access_slb_pSeries:
 204         HMT_MEDIUM_PPR_DISCARD
 205         SET_SCRATCH0(r13)
 206         EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST_PR, 0x480)
 207         std     r3,PACA_EXSLB+EX_R3(r13)
 208         mfspr   r3,SPRN_SRR0            /* SRR0 is faulting address */
 209 #ifdef __DISABLED__
 210         /* Keep that around for when we re-implement dynamic VSIDs */
 211         cmpdi   r3,0
 212         bge     slb_miss_user_pseries
 213 #endif /* __DISABLED__ */
 214         mfspr   r12,SPRN_SRR1
 215 #ifndef CONFIG_RELOCATABLE
 216         b       .slb_miss_realmode
 217 #else
 218         mfctr   r11
 219         ld      r10,PACAKBASE(r13)
 220         LOAD_HANDLER(r10, .slb_miss_realmode)
 221         mtctr   r10
 222         bctr
 223 #endif
 224
 225         /* We open code these as we can't have a ". = x" (even with
 226          * x = "." within a feature section
 227          */
 228         . = 0x500;
 229         .globl hardware_interrupt_pSeries;
 230         .globl hardware_interrupt_hv;
 231 hardware_interrupt_pSeries:
 232 hardware_interrupt_hv:
 233         BEGIN_FTR_SECTION
 234                 _MASKABLE_EXCEPTION_PSERIES(0x502, hardware_interrupt,
 235                                             EXC_HV, SOFTEN_TEST_HV)
 236                 KVM_HANDLER(PACA_EXGEN, EXC_HV, 0x502)
 237         FTR_SECTION_ELSE
 238                 _MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt,
 239                                             EXC_STD, SOFTEN_TEST_HV_201)
 240                 KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x500)
 241         ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
 242
 243         STD_EXCEPTION_PSERIES(0x600, 0x600, alignment)
 244         KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x600)
 245
 246         STD_EXCEPTION_PSERIES(0x700, 0x700, program_check)
 247         KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x700)
 248
 249         STD_EXCEPTION_PSERIES(0x800, 0x800, fp_unavailable)
 250         KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x800)
 251
 252         MASKABLE_EXCEPTION_PSERIES(0x900, 0x900, decrementer)
 253         STD_EXCEPTION_HV(0x980, 0x982, hdecrementer)
 254
 255         MASKABLE_EXCEPTION_PSERIES(0xa00, 0xa00, doorbell_super)
 256         KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xa00)
 257
 258         STD_EXCEPTION_PSERIES(0xb00, 0xb00, trap_0b)
 259         KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xb00)
 260
 261         . = 0xc00
 262         .globl  system_call_pSeries
 263 system_call_pSeries:
 264         HMT_MEDIUM
 265 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
 266         SET_SCRATCH0(r13)
 267         GET_PACA(r13)
 268         std     r9,PACA_EXGEN+EX_R9(r13)
 269         std     r10,PACA_EXGEN+EX_R10(r13)
 270         mfcr    r9
 271         KVMTEST(0xc00)
 272         GET_SCRATCH0(r13)
 273 #endif
 274         SYSCALL_PSERIES_1
 275         SYSCALL_PSERIES_2_RFID
 276         SYSCALL_PSERIES_3
 277         KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xc00)
 278
 279         STD_EXCEPTION_PSERIES(0xd00, 0xd00, single_step)
 280         KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xd00)
 281
 282         /* At 0xe??? we have a bunch of hypervisor exceptions, we branch
 283          * out of line to handle them
 284          */
 285         . = 0xe00
 286 hv_exception_trampoline:
 287         b       h_data_storage_hv
 288         . = 0xe20
 289         b       h_instr_storage_hv
 290         . = 0xe40
 291         b       emulation_assist_hv
 292         . = 0xe50
 293         b       hmi_exception_hv
 294         . = 0xe60
 295         b       hmi_exception_hv
 296         . = 0xe80
 297         b       h_doorbell_hv
 298
 299         /* We need to deal with the Altivec unavailable exception
 300          * here which is at 0xf20, thus in the middle of the
 301          * prolog code of the PerformanceMonitor one. A little
 302          * trickery is thus necessary
 303          */
 304 performance_monitor_pSeries_1:
 305         . = 0xf00
 306         b       performance_monitor_pSeries
 307
 308 altivec_unavailable_pSeries_1:
 309         . = 0xf20
 310         b       altivec_unavailable_pSeries
 311
 312 vsx_unavailable_pSeries_1:
 313         . = 0xf40
 314         b       vsx_unavailable_pSeries
 315
 316 #ifdef CONFIG_CBE_RAS
 317         STD_EXCEPTION_HV(0x1200, 0x1202, cbe_system_error)
 318         KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1202)
 319 #endif /* CONFIG_CBE_RAS */
 320
 321         STD_EXCEPTION_PSERIES(0x1300, 0x1300, instruction_breakpoint)
 322         KVM_HANDLER_PR_SKIP(PACA_EXGEN, EXC_STD, 0x1300)
 323
 324         . = 0x1500
 325         .global denorm_exception_hv
 326 denorm_exception_hv:
 327         HMT_MEDIUM_PPR_DISCARD
 328         mtspr   SPRN_SPRG_HSCRATCH0,r13
 329         mfspr   r13,SPRN_SPRG_HPACA
 330         std     r9,PACA_EXGEN+EX_R9(r13)
 331         HMT_MEDIUM_PPR_SAVE(PACA_EXGEN, r9)
 332         std     r10,PACA_EXGEN+EX_R10(r13)
 333         std     r11,PACA_EXGEN+EX_R11(r13)
 334         std     r12,PACA_EXGEN+EX_R12(r13)
 335         mfspr   r9,SPRN_SPRG_HSCRATCH0
 336         std     r9,PACA_EXGEN+EX_R13(r13)
 337         mfcr    r9
 338
 339 #ifdef CONFIG_PPC_DENORMALISATION
 340         mfspr   r10,SPRN_HSRR1
 341         mfspr   r11,SPRN_HSRR0          /* save HSRR0 */
 342         andis.  r10,r10,(HSRR1_DENORM)@h /* denorm? */
 343         addi    r11,r11,-4              /* HSRR0 is next instruction */
 344         bne+    denorm_assist
 345 #endif
 346
 347         EXCEPTION_PROLOG_PSERIES_1(denorm_common, EXC_HV)
 348         KVM_HANDLER_SKIP(PACA_EXGEN, EXC_STD, 0x1500)
 349
 350 #ifdef CONFIG_CBE_RAS
 351         STD_EXCEPTION_HV(0x1600, 0x1602, cbe_maintenance)
 352         KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1602)
 353 #endif /* CONFIG_CBE_RAS */
 354
 355         STD_EXCEPTION_PSERIES(0x1700, 0x1700, altivec_assist)
 356         KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x1700)
 357
 358 #ifdef CONFIG_CBE_RAS
 359         STD_EXCEPTION_HV(0x1800, 0x1802, cbe_thermal)
 360         KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1802)
 361 #else
 362         . = 0x1800
 363 #endif /* CONFIG_CBE_RAS */
 364
 365
 366 /*** Out of line interrupts support ***/
 367
 368         .align  7
 369         /* moved from 0x200 */
 370 machine_check_pSeries:
 371         .globl machine_check_fwnmi
 372 machine_check_fwnmi:
 373         HMT_MEDIUM_PPR_DISCARD
 374         SET_SCRATCH0(r13)               /* save r13 */
 375         EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common,
 376                                  EXC_STD, KVMTEST, 0x200)
 377         KVM_HANDLER_SKIP(PACA_EXMC, EXC_STD, 0x200)
 378
 379         /* moved from 0x300 */
 380 data_access_check_stab:
 381         GET_PACA(r13)
 382         std     r9,PACA_EXSLB+EX_R9(r13)
 383         std     r10,PACA_EXSLB+EX_R10(r13)
 384         mfspr   r10,SPRN_DAR
 385         mfspr   r9,SPRN_DSISR
 386         srdi    r10,r10,60
 387         rlwimi  r10,r9,16,0x20
 388 #ifdef CONFIG_KVM_BOOK3S_PR
 389         lbz     r9,HSTATE_IN_GUEST(r13)
 390         rlwimi  r10,r9,8,0x300
 391 #endif
 392         mfcr    r9
 393         cmpwi   r10,0x2c
 394         beq     do_stab_bolted_pSeries
 395         mtcrf   0x80,r9
 396         ld      r9,PACA_EXSLB+EX_R9(r13)
 397         ld      r10,PACA_EXSLB+EX_R10(r13)
 398         b       data_access_not_stab
 399 do_stab_bolted_pSeries:
 400         std     r11,PACA_EXSLB+EX_R11(r13)
 401         std     r12,PACA_EXSLB+EX_R12(r13)
 402         GET_SCRATCH0(r10)
 403         std     r10,PACA_EXSLB+EX_R13(r13)
 404         EXCEPTION_PROLOG_PSERIES_1(.do_stab_bolted, EXC_STD)
 405
 406         KVM_HANDLER_SKIP(PACA_EXGEN, EXC_STD, 0x300)
 407         KVM_HANDLER_SKIP(PACA_EXSLB, EXC_STD, 0x380)
 408         KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x400)
 409         KVM_HANDLER_PR(PACA_EXSLB, EXC_STD, 0x480)
 410         KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x900)
 411         KVM_HANDLER(PACA_EXGEN, EXC_HV, 0x982)
 412
 413 #ifdef CONFIG_PPC_DENORMALISATION
 414 denorm_assist:
 415 BEGIN_FTR_SECTION
 416 /*
 417  * To denormalise we need to move a copy of the register to itself.
 418  * For POWER6 do that here for all FP regs.
 419  */
 420         mfmsr   r10
 421         ori     r10,r10,(MSR_FP|MSR_FE0|MSR_FE1)
 422         xori    r10,r10,(MSR_FE0|MSR_FE1)
 423         mtmsrd  r10
 424         sync
 425         fmr     0,0
 426         fmr     1,1
 427         fmr     2,2
 428         fmr     3,3
 429         fmr     4,4
 430         fmr     5,5
 431         fmr     6,6
 432         fmr     7,7
 433         fmr     8,8
 434         fmr     9,9
 435         fmr     10,10
 436         fmr     11,11
 437         fmr     12,12
 438         fmr     13,13
 439         fmr     14,14
 440         fmr     15,15
 441         fmr     16,16
 442         fmr     17,17
 443         fmr     18,18
 444         fmr     19,19
 445         fmr     20,20
 446         fmr     21,21
 447         fmr     22,22
 448         fmr     23,23
 449         fmr     24,24
 450         fmr     25,25
 451         fmr     26,26
 452         fmr     27,27
 453         fmr     28,28
 454         fmr     29,29
 455         fmr     30,30
 456         fmr     31,31
 457 FTR_SECTION_ELSE
 458 /*
 459  * To denormalise we need to move a copy of the register to itself.
 460  * For POWER7 do that here for the first 32 VSX registers only.
 461  */
 462         mfmsr   r10
 463         oris    r10,r10,MSR_VSX@h
 464         mtmsrd  r10
 465         sync
 466         XVCPSGNDP(0,0,0)
 467         XVCPSGNDP(1,1,1)
 468         XVCPSGNDP(2,2,2)
 469         XVCPSGNDP(3,3,3)
 470         XVCPSGNDP(4,4,4)
 471         XVCPSGNDP(5,5,5)
 472         XVCPSGNDP(6,6,6)
 473         XVCPSGNDP(7,7,7)
 474         XVCPSGNDP(8,8,8)
 475         XVCPSGNDP(9,9,9)
 476         XVCPSGNDP(10,10,10)
 477         XVCPSGNDP(11,11,11)
 478         XVCPSGNDP(12,12,12)
 479         XVCPSGNDP(13,13,13)
 480         XVCPSGNDP(14,14,14)
 481         XVCPSGNDP(15,15,15)
 482         XVCPSGNDP(16,16,16)
 483         XVCPSGNDP(17,17,17)
 484         XVCPSGNDP(18,18,18)
 485         XVCPSGNDP(19,19,19)
 486         XVCPSGNDP(20,20,20)
 487         XVCPSGNDP(21,21,21)
 488         XVCPSGNDP(22,22,22)
 489         XVCPSGNDP(23,23,23)
 490         XVCPSGNDP(24,24,24)
 491         XVCPSGNDP(25,25,25)
 492         XVCPSGNDP(26,26,26)
 493         XVCPSGNDP(27,27,27)
 494         XVCPSGNDP(28,28,28)
 495         XVCPSGNDP(29,29,29)
 496         XVCPSGNDP(30,30,30)
 497         XVCPSGNDP(31,31,31)
 498 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206)
 499         mtspr   SPRN_HSRR0,r11
 500         mtcrf   0x80,r9
 501         ld      r9,PACA_EXGEN+EX_R9(r13)
 502         RESTORE_PPR_PACA(PACA_EXGEN, r10)
 503         ld      r10,PACA_EXGEN+EX_R10(r13)
 504         ld      r11,PACA_EXGEN+EX_R11(r13)
 505         ld      r12,PACA_EXGEN+EX_R12(r13)
 506         ld      r13,PACA_EXGEN+EX_R13(r13)
 507         HRFID
 508         b       .
 509 #endif
 510
 511         .align  7
 512         /* moved from 0xe00 */
 513         STD_EXCEPTION_HV(., 0xe02, h_data_storage)
 514         KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0xe02)
 515         STD_EXCEPTION_HV(., 0xe22, h_instr_storage)
 516         KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe22)
 517         STD_EXCEPTION_HV(., 0xe42, emulation_assist)
 518         KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe42)
 519         STD_EXCEPTION_HV(., 0xe62, hmi_exception) /* need to flush cache ? */
 520         KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe62)
 521         MASKABLE_EXCEPTION_HV(., 0xe82, h_doorbell)
 522         KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe82)
 523
 524         /* moved from 0xf00 */
 525         STD_EXCEPTION_PSERIES(., 0xf00, performance_monitor)
 526         KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xf00)
 527         STD_EXCEPTION_PSERIES(., 0xf20, altivec_unavailable)
 528         KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xf20)
 529         STD_EXCEPTION_PSERIES(., 0xf40, vsx_unavailable)
 530         KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xf40)
 531
 532 /*
 533  * An interrupt came in while soft-disabled. We set paca->irq_happened, then:
 534  * - If it was a decrementer interrupt, we bump the dec to max and and return.
 535  * - If it was a doorbell we return immediately since doorbells are edge
 536  *   triggered and won't automatically refire.
 537  * - else we hard disable and return.
 538  * This is called with r10 containing the value to OR to the paca field.
 539  */
 540 #define MASKED_INTERRUPT(_H)                            \
 541 masked_##_H##interrupt:                                 \
 542         std     r11,PACA_EXGEN+EX_R11(r13);             \
 543         lbz     r11,PACAIRQHAPPENED(r13);               \
 544         or      r11,r11,r10;                            \
 545         stb     r11,PACAIRQHAPPENED(r13);               \
 546         cmpwi   r10,PACA_IRQ_DEC;                       \
 547         bne     1f;                                     \
 548         lis     r10,0x7fff;                             \
 549         ori     r10,r10,0xffff;                         \
 550         mtspr   SPRN_DEC,r10;                           \
 551         b       2f;                                     \
 552 1:      cmpwi   r10,PACA_IRQ_DBELL;                     \
 553         beq     2f;                                     \
 554         mfspr   r10,SPRN_##_H##SRR1;                    \
 555         rldicl  r10,r10,48,1; /* clear MSR_EE */        \
 556         rotldi  r10,r10,16;                             \
 557         mtspr   SPRN_##_H##SRR1,r10;                    \
 558 2:      mtcrf   0x80,r9;                                \
 559         ld      r9,PACA_EXGEN+EX_R9(r13);               \
 560         ld      r10,PACA_EXGEN+EX_R10(r13);             \
 561         ld      r11,PACA_EXGEN+EX_R11(r13);             \
 562         GET_SCRATCH0(r13);                              \
 563         ##_H##rfid;                                     \
 564         b       .
 565
 566         MASKED_INTERRUPT()
 567         MASKED_INTERRUPT(H)
 568
 569 /*
 570  * Called from arch_local_irq_enable when an interrupt needs
 571  * to be resent. r3 contains 0x500, 0x900, 0xa00 or 0xe80 to indicate
 572  * which kind of interrupt. MSR:EE is already off. We generate a
 573  * stackframe like if a real interrupt had happened.
 574  *
 575  * Note: While MSR:EE is off, we need to make sure that _MSR
 576  * in the generated frame has EE set to 1 or the exception
 577  * handler will not properly re-enable them.
 578  */
 579 _GLOBAL(__replay_interrupt)
 580         /* We are going to jump to the exception common code which
 581          * will retrieve various register values from the PACA which
 582          * we don't give a damn about, so we don't bother storing them.
 583          */
 584         mfmsr   r12
 585         mflr    r11
 586         mfcr    r9
 587         ori     r12,r12,MSR_EE
 588         cmpwi   r3,0x900
 589         beq     decrementer_common
 590         cmpwi   r3,0x500
 591         beq     hardware_interrupt_common
 592 BEGIN_FTR_SECTION
 593         cmpwi   r3,0xe80
 594         beq     h_doorbell_common
 595 FTR_SECTION_ELSE
 596         cmpwi   r3,0xa00
 597         beq     doorbell_super_common
 598 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
 599         blr
 600
 601 #ifdef CONFIG_PPC_PSERIES
 602 /*
 603  * Vectors for the FWNMI option.  Share common code.
 604  */
 605         .globl system_reset_fwnmi
 606       .align 7
 607 system_reset_fwnmi:
 608         HMT_MEDIUM_PPR_DISCARD
 609         SET_SCRATCH0(r13)               /* save r13 */
 610         EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD,
 611                                  NOTEST, 0x100)
 612
 613 #endif /* CONFIG_PPC_PSERIES */
 614
 615 #ifdef __DISABLED__
 616 /*
 617  * This is used for when the SLB miss handler has to go virtual,
 618  * which doesn't happen for now anymore but will once we re-implement
 619  * dynamic VSIDs for shared page tables
 620  */
 621 slb_miss_user_pseries:
 622         std     r10,PACA_EXGEN+EX_R10(r13)
 623         std     r11,PACA_EXGEN+EX_R11(r13)
 624         std     r12,PACA_EXGEN+EX_R12(r13)
 625         GET_SCRATCH0(r10)
 626         ld      r11,PACA_EXSLB+EX_R9(r13)
 627         ld      r12,PACA_EXSLB+EX_R3(r13)
 628         std     r10,PACA_EXGEN+EX_R13(r13)
 629         std     r11,PACA_EXGEN+EX_R9(r13)
 630         std     r12,PACA_EXGEN+EX_R3(r13)
 631         clrrdi  r12,r13,32
 632         mfmsr   r10
 633         mfspr   r11,SRR0                        /* save SRR0 */
 634         ori     r12,r12,slb_miss_user_common@l  /* virt addr of handler */
 635         ori     r10,r10,MSR_IR|MSR_DR|MSR_RI
 636         mtspr   SRR0,r12
 637         mfspr   r12,SRR1                        /* and SRR1 */
 638         mtspr   SRR1,r10
 639         rfid
 640         b       .                               /* prevent spec. execution */
 641 #endif /* __DISABLED__ */
 642
 643 /*
 644  * Code from here down to __end_handlers is invoked from the
 645  * exception prologs above.  Because the prologs assemble the
 646  * addresses of these handlers using the LOAD_HANDLER macro,
 647  * which uses an ori instruction, these handlers must be in
 648  * the first 64k of the kernel image.
 649  */
 650
 651 /*** Common interrupt handlers ***/
 652
 653         STD_EXCEPTION_COMMON(0x100, system_reset, .system_reset_exception)
 654
 655         /*
 656          * Machine check is different because we use a different
 657          * save area: PACA_EXMC instead of PACA_EXGEN.
 658          */
 659         .align  7
 660         .globl machine_check_common
 661 machine_check_common:
 662         EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC)
 663         FINISH_NAP
 664         DISABLE_INTS
 665         bl      .save_nvgprs
 666         addi    r3,r1,STACK_FRAME_OVERHEAD
 667         bl      .machine_check_exception
 668         b       .ret_from_except
 669
 670         STD_EXCEPTION_COMMON_ASYNC(0x500, hardware_interrupt, do_IRQ)
 671         STD_EXCEPTION_COMMON_ASYNC(0x900, decrementer, .timer_interrupt)
 672         STD_EXCEPTION_COMMON(0x980, hdecrementer, .hdec_interrupt)
 673 #ifdef CONFIG_PPC_DOORBELL
 674         STD_EXCEPTION_COMMON_ASYNC(0xa00, doorbell_super, .doorbell_exception)
 675 #else
 676         STD_EXCEPTION_COMMON_ASYNC(0xa00, doorbell_super, .unknown_exception)
 677 #endif
 678         STD_EXCEPTION_COMMON(0xb00, trap_0b, .unknown_exception)
 679         STD_EXCEPTION_COMMON(0xd00, single_step, .single_step_exception)
 680         STD_EXCEPTION_COMMON(0xe00, trap_0e, .unknown_exception)
 681         STD_EXCEPTION_COMMON(0xe40, emulation_assist, .program_check_exception)
 682         STD_EXCEPTION_COMMON(0xe60, hmi_exception, .unknown_exception)
 683 #ifdef CONFIG_PPC_DOORBELL
 684         STD_EXCEPTION_COMMON_ASYNC(0xe80, h_doorbell, .doorbell_exception)
 685 #else
 686         STD_EXCEPTION_COMMON_ASYNC(0xe80, h_doorbell, .unknown_exception)
 687 #endif
 688         STD_EXCEPTION_COMMON_ASYNC(0xf00, performance_monitor, .performance_monitor_exception)
 689         STD_EXCEPTION_COMMON(0x1300, instruction_breakpoint, .instruction_breakpoint_exception)
 690         STD_EXCEPTION_COMMON(0x1502, denorm, .unknown_exception)
 691 #ifdef CONFIG_ALTIVEC
 692         STD_EXCEPTION_COMMON(0x1700, altivec_assist, .altivec_assist_exception)
 693 #else
 694         STD_EXCEPTION_COMMON(0x1700, altivec_assist, .unknown_exception)
 695 #endif
 696 #ifdef CONFIG_CBE_RAS
 697         STD_EXCEPTION_COMMON(0x1200, cbe_system_error, .cbe_system_error_exception)
 698         STD_EXCEPTION_COMMON(0x1600, cbe_maintenance, .cbe_maintenance_exception)
 699         STD_EXCEPTION_COMMON(0x1800, cbe_thermal, .cbe_thermal_exception)
 700 #endif /* CONFIG_CBE_RAS */
 701
 702         /*
 703          * Relocation-on interrupts: A subset of the interrupts can be delivered
 704          * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering
 705          * it.  Addresses are the same as the original interrupt addresses, but
 706          * offset by 0xc000000000004000.
 707          * It's impossible to receive interrupts below 0x300 via this mechanism.
 708          * KVM: None of these traps are from the guest ; anything that escalated
 709          * to HV=1 from HV=0 is delivered via real mode handlers.
 710          */
 711
 712         /*
 713          * This uses the standard macro, since the original 0x300 vector
 714          * only has extra guff for STAB-based processors -- which never
 715          * come here.
 716          */
 717         STD_RELON_EXCEPTION_PSERIES(0x4300, 0x300, data_access)
 718         . = 0x4380
 719         .globl data_access_slb_relon_pSeries
 720 data_access_slb_relon_pSeries:
 721         HMT_MEDIUM_PPR_DISCARD
 722         SET_SCRATCH0(r13)
 723         EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x380)
 724         std     r3,PACA_EXSLB+EX_R3(r13)
 725         mfspr   r3,SPRN_DAR
 726         mfspr   r12,SPRN_SRR1
 727 #ifndef CONFIG_RELOCATABLE
 728         b       .slb_miss_realmode
 729 #else
 730         /*
 731          * We can't just use a direct branch to .slb_miss_realmode
 732          * because the distance from here to there depends on where
 733          * the kernel ends up being put.
 734          */
 735         mfctr   r11
 736         ld      r10,PACAKBASE(r13)
 737         LOAD_HANDLER(r10, .slb_miss_realmode)
 738         mtctr   r10
 739         bctr
 740 #endif
 741
 742         STD_RELON_EXCEPTION_PSERIES(0x4400, 0x400, instruction_access)
 743         . = 0x4480
 744         .globl instruction_access_slb_relon_pSeries
 745 instruction_access_slb_relon_pSeries:
 746         HMT_MEDIUM_PPR_DISCARD
 747         SET_SCRATCH0(r13)
 748         EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x480)
 749         std     r3,PACA_EXSLB+EX_R3(r13)
 750         mfspr   r3,SPRN_SRR0            /* SRR0 is faulting address */
 751         mfspr   r12,SPRN_SRR1
 752 #ifndef CONFIG_RELOCATABLE
 753         b       .slb_miss_realmode
 754 #else
 755         mfctr   r11
 756         ld      r10,PACAKBASE(r13)
 757         LOAD_HANDLER(r10, .slb_miss_realmode)
 758         mtctr   r10
 759         bctr
 760 #endif
 761
 762         . = 0x4500
 763         .globl hardware_interrupt_relon_pSeries;
 764         .globl hardware_interrupt_relon_hv;
 765 hardware_interrupt_relon_pSeries:
 766 hardware_interrupt_relon_hv:
 767         BEGIN_FTR_SECTION
 768                 _MASKABLE_RELON_EXCEPTION_PSERIES(0x502, hardware_interrupt, EXC_HV, SOFTEN_TEST_HV)
 769         FTR_SECTION_ELSE
 770                 _MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt, EXC_STD, SOFTEN_TEST_PR)
 771         ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_206)
 772         STD_RELON_EXCEPTION_PSERIES(0x4600, 0x600, alignment)
 773         STD_RELON_EXCEPTION_PSERIES(0x4700, 0x700, program_check)
 774         STD_RELON_EXCEPTION_PSERIES(0x4800, 0x800, fp_unavailable)
 775         MASKABLE_RELON_EXCEPTION_PSERIES(0x4900, 0x900, decrementer)
 776         STD_RELON_EXCEPTION_HV(0x4980, 0x982, hdecrementer)
 777         MASKABLE_RELON_EXCEPTION_PSERIES(0x4a00, 0xa00, doorbell_super)
 778         STD_RELON_EXCEPTION_PSERIES(0x4b00, 0xb00, trap_0b)
 779
 780         . = 0x4c00
 781         .globl system_call_relon_pSeries
 782 system_call_relon_pSeries:
 783         HMT_MEDIUM
 784         SYSCALL_PSERIES_1
 785         SYSCALL_PSERIES_2_DIRECT
 786         SYSCALL_PSERIES_3
 787
 788         STD_RELON_EXCEPTION_PSERIES(0x4d00, 0xd00, single_step)
 789
 790         . = 0x4e00
 791         b       h_data_storage_relon_hv
 792
 793         . = 0x4e20
 794         b       h_instr_storage_relon_hv
 795
 796         . = 0x4e40
 797         b       emulation_assist_relon_hv
 798
 799         . = 0x4e50
 800         b       hmi_exception_relon_hv
 801
 802         . = 0x4e60
 803         b       hmi_exception_relon_hv
 804
 805         . = 0x4e80
 806         b       h_doorbell_relon_hv
 807
 808 performance_monitor_relon_pSeries_1:
 809         . = 0x4f00
 810         b       performance_monitor_relon_pSeries
 811
 812 altivec_unavailable_relon_pSeries_1:
 813         . = 0x4f20
 814         b       altivec_unavailable_relon_pSeries
 815
 816 vsx_unavailable_relon_pSeries_1:
 817         . = 0x4f40
 818         b       vsx_unavailable_relon_pSeries
 819
 820 #ifdef CONFIG_CBE_RAS
 821         STD_RELON_EXCEPTION_HV(0x5200, 0x1202, cbe_system_error)
 822 #endif /* CONFIG_CBE_RAS */
 823         STD_RELON_EXCEPTION_PSERIES(0x5300, 0x1300, instruction_breakpoint)
 824 #ifdef CONFIG_PPC_DENORMALISATION
 825         . = 0x5500
 826         b       denorm_exception_hv
 827 #endif
 828 #ifdef CONFIG_CBE_RAS
 829         STD_RELON_EXCEPTION_HV(0x5600, 0x1602, cbe_maintenance)
 830 #else
 831 #ifdef CONFIG_HVC_SCOM
 832         STD_RELON_EXCEPTION_HV(0x5600, 0x1600, maintence_interrupt)
 833         KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1600)
 834 #endif /* CONFIG_HVC_SCOM */
 835 #endif /* CONFIG_CBE_RAS */
 836         STD_RELON_EXCEPTION_PSERIES(0x5700, 0x1700, altivec_assist)
 837 #ifdef CONFIG_CBE_RAS
 838         STD_RELON_EXCEPTION_HV(0x5800, 0x1802, cbe_thermal)
 839 #endif /* CONFIG_CBE_RAS */
 840
 841         /* Other future vectors */
 842         .align  7
 843         .globl  __end_interrupts
 844 __end_interrupts:
 845
 846         .align  7
 847 system_call_entry_direct:
 848 #if defined(CONFIG_RELOCATABLE)
 849         /* The first level prologue may have used LR to get here, saving
 850          * orig in r10.  To save hacking/ifdeffing common code, restore here.
 851          */
 852         mtlr    r10
 853 #endif
 854 system_call_entry:
 855         b       system_call_common
 856
 857 ppc64_runlatch_on_trampoline:
 858         b       .__ppc64_runlatch_on
 859
 860 /*
 861  * Here we have detected that the kernel stack pointer is bad.
 862  * R9 contains the saved CR, r13 points to the paca,
 863  * r10 contains the (bad) kernel stack pointer,
 864  * r11 and r12 contain the saved SRR0 and SRR1.
 865  * We switch to using an emergency stack, save the registers there,
 866  * and call kernel_bad_stack(), which panics.
 867  */
 868 bad_stack:
 869         ld      r1,PACAEMERGSP(r13)
 870         subi    r1,r1,64+INT_FRAME_SIZE
 871         std     r9,_CCR(r1)
 872         std     r10,GPR1(r1)
 873         std     r11,_NIP(r1)
 874         std     r12,_MSR(r1)
 875         mfspr   r11,SPRN_DAR
 876         mfspr   r12,SPRN_DSISR
 877         std     r11,_DAR(r1)
 878         std     r12,_DSISR(r1)
 879         mflr    r10
 880         mfctr   r11
 881         mfxer   r12
 882         std     r10,_LINK(r1)
 883         std     r11,_CTR(r1)
 884         std     r12,_XER(r1)
 885         SAVE_GPR(0,r1)
 886         SAVE_GPR(2,r1)
 887         ld      r10,EX_R3(r3)
 888         std     r10,GPR3(r1)
 889         SAVE_GPR(4,r1)
 890         SAVE_4GPRS(5,r1)
 891         ld      r9,EX_R9(r3)
 892         ld      r10,EX_R10(r3)
 893         SAVE_2GPRS(9,r1)
 894         ld      r9,EX_R11(r3)
 895         ld      r10,EX_R12(r3)
 896         ld      r11,EX_R13(r3)
 897         std     r9,GPR11(r1)
 898         std     r10,GPR12(r1)
 899         std     r11,GPR13(r1)
 900 BEGIN_FTR_SECTION
 901         ld      r10,EX_CFAR(r3)
 902         std     r10,ORIG_GPR3(r1)
 903 END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
 904         SAVE_8GPRS(14,r1)
 905         SAVE_10GPRS(22,r1)
 906         lhz     r12,PACA_TRAP_SAVE(r13)
 907         std     r12,_TRAP(r1)
 908         addi    r11,r1,INT_FRAME_SIZE
 909         std     r11,0(r1)
 910         li      r12,0
 911         std     r12,0(r11)
 912         ld      r2,PACATOC(r13)
 913         ld      r11,exception_marker@toc(r2)
 914         std     r12,RESULT(r1)
 915         std     r11,STACK_FRAME_OVERHEAD-16(r1)
 916 1:      addi    r3,r1,STACK_FRAME_OVERHEAD
 917         bl      .kernel_bad_stack
 918         b       1b
 919
 920 /*
 921  * Here r13 points to the paca, r9 contains the saved CR,
 922  * SRR0 and SRR1 are saved in r11 and r12,
 923  * r9 - r13 are saved in paca->exgen.
 924  */
 925         .align  7
 926         .globl data_access_common
 927 data_access_common:
 928         mfspr   r10,SPRN_DAR
 929         std     r10,PACA_EXGEN+EX_DAR(r13)
 930         mfspr   r10,SPRN_DSISR
 931         stw     r10,PACA_EXGEN+EX_DSISR(r13)
 932         EXCEPTION_PROLOG_COMMON(0x300, PACA_EXGEN)
 933         DISABLE_INTS
 934         ld      r12,_MSR(r1)
 935         ld      r3,PACA_EXGEN+EX_DAR(r13)
 936         lwz     r4,PACA_EXGEN+EX_DSISR(r13)
 937         li      r5,0x300
 938         b       .do_hash_page           /* Try to handle as hpte fault */
 939
 940         .align  7
 941         .globl  h_data_storage_common
 942 h_data_storage_common:
 943         mfspr   r10,SPRN_HDAR
 944         std     r10,PACA_EXGEN+EX_DAR(r13)
 945         mfspr   r10,SPRN_HDSISR
 946         stw     r10,PACA_EXGEN+EX_DSISR(r13)
 947         EXCEPTION_PROLOG_COMMON(0xe00, PACA_EXGEN)
 948         bl      .save_nvgprs
 949         DISABLE_INTS
 950         addi    r3,r1,STACK_FRAME_OVERHEAD
 951         bl      .unknown_exception
 952         b       .ret_from_except
 953
 954         .align  7
 955         .globl instruction_access_common
 956 instruction_access_common:
 957         EXCEPTION_PROLOG_COMMON(0x400, PACA_EXGEN)
 958         DISABLE_INTS
 959         ld      r12,_MSR(r1)
 960         ld      r3,_NIP(r1)
 961         andis.  r4,r12,0x5820
 962         li      r5,0x400
 963         b       .do_hash_page           /* Try to handle as hpte fault */
 964
 965         STD_EXCEPTION_COMMON(0xe20, h_instr_storage, .unknown_exception)
 966
 967 /*
 968  * Here is the common SLB miss user that is used when going to virtual
 969  * mode for SLB misses, that is currently not used
 970  */
 971 #ifdef __DISABLED__
 972         .align  7
 973         .globl  slb_miss_user_common
 974 slb_miss_user_common:
 975         mflr    r10
 976         std     r3,PACA_EXGEN+EX_DAR(r13)
 977         stw     r9,PACA_EXGEN+EX_CCR(r13)
 978         std     r10,PACA_EXGEN+EX_LR(r13)
 979         std     r11,PACA_EXGEN+EX_SRR0(r13)
 980         bl      .slb_allocate_user
 981
 982         ld      r10,PACA_EXGEN+EX_LR(r13)
 983         ld      r3,PACA_EXGEN+EX_R3(r13)
 984         lwz     r9,PACA_EXGEN+EX_CCR(r13)
 985         ld      r11,PACA_EXGEN+EX_SRR0(r13)
 986         mtlr    r10
 987         beq-    slb_miss_fault
 988
 989         andi.   r10,r12,MSR_RI          /* check for unrecoverable exception */
 990         beq-    unrecov_user_slb
 991         mfmsr   r10
 992
 993 .machine push
 994 .machine "power4"
 995         mtcrf   0x80,r9
 996 .machine pop
 997
 998         clrrdi  r10,r10,2               /* clear RI before setting SRR0/1 */
 999         mtmsrd  r10,1
1000
1001         mtspr   SRR0,r11
1002         mtspr   SRR1,r12
1003
1004         ld      r9,PACA_EXGEN+EX_R9(r13)
1005         ld      r10,PACA_EXGEN+EX_R10(r13)
1006         ld      r11,PACA_EXGEN+EX_R11(r13)
1007         ld      r12,PACA_EXGEN+EX_R12(r13)
1008         ld      r13,PACA_EXGEN+EX_R13(r13)
1009         rfid
1010         b       .
1011
1012 slb_miss_fault:
1013         EXCEPTION_PROLOG_COMMON(0x380, PACA_EXGEN)
1014         ld      r4,PACA_EXGEN+EX_DAR(r13)
1015         li      r5,0
1016         std     r4,_DAR(r1)
1017         std     r5,_DSISR(r1)
1018         b       handle_page_fault
1019
1020 unrecov_user_slb:
1021         EXCEPTION_PROLOG_COMMON(0x4200, PACA_EXGEN)
1022         DISABLE_INTS
1023         bl      .save_nvgprs
1024 1:      addi    r3,r1,STACK_FRAME_OVERHEAD
1025         bl      .unrecoverable_exception
1026         b       1b
1027
1028 #endif /* __DISABLED__ */
1029
1030
1031 /*
1032  * r13 points to the PACA, r9 contains the saved CR,
1033  * r12 contain the saved SRR1, SRR0 is still ready for return
1034  * r3 has the faulting address
1035  * r9 - r13 are saved in paca->exslb.
1036  * r3 is saved in paca->slb_r3
1037  * We assume we aren't going to take any exceptions during this procedure.
1038  */
1039 _GLOBAL(slb_miss_realmode)
1040         mflr    r10
1041 #ifdef CONFIG_RELOCATABLE
1042         mtctr   r11
1043 #endif
1044
1045         stw     r9,PACA_EXSLB+EX_CCR(r13)       /* save CR in exc. frame */
1046         std     r10,PACA_EXSLB+EX_LR(r13)       /* save LR */
1047
1048         bl      .slb_allocate_realmode
1049
1050         /* All done -- return from exception. */
1051
1052         ld      r10,PACA_EXSLB+EX_LR(r13)
1053         ld      r3,PACA_EXSLB+EX_R3(r13)
1054         lwz     r9,PACA_EXSLB+EX_CCR(r13)       /* get saved CR */
1055
1056         mtlr    r10
1057
1058         andi.   r10,r12,MSR_RI  /* check for unrecoverable exception */
1059         beq-    2f
1060
1061 .machine        push
1062 .machine        "power4"
1063         mtcrf   0x80,r9
1064         mtcrf   0x01,r9         /* slb_allocate uses cr0 and cr7 */
1065 .machine        pop
1066
1067         RESTORE_PPR_PACA(PACA_EXSLB, r9)
1068         ld      r9,PACA_EXSLB+EX_R9(r13)
1069         ld      r10,PACA_EXSLB+EX_R10(r13)
1070         ld      r11,PACA_EXSLB+EX_R11(r13)
1071         ld      r12,PACA_EXSLB+EX_R12(r13)
1072         ld      r13,PACA_EXSLB+EX_R13(r13)
1073         rfid
1074         b       .       /* prevent speculative execution */
1075
1076 2:      mfspr   r11,SPRN_SRR0
1077         ld      r10,PACAKBASE(r13)
1078         LOAD_HANDLER(r10,unrecov_slb)
1079         mtspr   SPRN_SRR0,r10
1080         ld      r10,PACAKMSR(r13)
1081         mtspr   SPRN_SRR1,r10
1082         rfid
1083         b       .
1084
1085 unrecov_slb:
1086         EXCEPTION_PROLOG_COMMON(0x4100, PACA_EXSLB)
1087         DISABLE_INTS
1088         bl      .save_nvgprs
1089 1:      addi    r3,r1,STACK_FRAME_OVERHEAD
1090         bl      .unrecoverable_exception
1091         b       1b
1092
1093
1094 #ifdef CONFIG_PPC_970_NAP
1095 power4_fixup_nap:
1096         andc    r9,r9,r10
1097         std     r9,TI_LOCAL_FLAGS(r11)
1098         ld      r10,_LINK(r1)           /* make idle task do the */
1099         std     r10,_NIP(r1)            /* equivalent of a blr */
1100         blr
1101 #endif
1102
1103         .align  7
1104         .globl alignment_common
1105 alignment_common:
1106         mfspr   r10,SPRN_DAR
1107         std     r10,PACA_EXGEN+EX_DAR(r13)
1108         mfspr   r10,SPRN_DSISR
1109         stw     r10,PACA_EXGEN+EX_DSISR(r13)
1110         EXCEPTION_PROLOG_COMMON(0x600, PACA_EXGEN)
1111         ld      r3,PACA_EXGEN+EX_DAR(r13)
1112         lwz     r4,PACA_EXGEN+EX_DSISR(r13)
1113         std     r3,_DAR(r1)
1114         std     r4,_DSISR(r1)
1115         bl      .save_nvgprs
1116         DISABLE_INTS
1117         addi    r3,r1,STACK_FRAME_OVERHEAD
1118         bl      .alignment_exception
1119         b       .ret_from_except
1120
1121         .align  7
1122         .globl program_check_common
1123 program_check_common:
1124         EXCEPTION_PROLOG_COMMON(0x700, PACA_EXGEN)
1125         bl      .save_nvgprs
1126         DISABLE_INTS
1127         addi    r3,r1,STACK_FRAME_OVERHEAD
1128         bl      .program_check_exception
1129         b       .ret_from_except
1130
1131         .align  7
1132         .globl fp_unavailable_common
1133 fp_unavailable_common:
1134         EXCEPTION_PROLOG_COMMON(0x800, PACA_EXGEN)
1135         bne     1f                      /* if from user, just load it up */
1136         bl      .save_nvgprs
1137         DISABLE_INTS
1138         addi    r3,r1,STACK_FRAME_OVERHEAD
1139         bl      .kernel_fp_unavailable_exception
1140         BUG_OPCODE
1141 1:      bl      .load_up_fpu
1142         b       fast_exception_return
1143
1144         .align  7
1145         .globl altivec_unavailable_common
1146 altivec_unavailable_common:
1147         EXCEPTION_PROLOG_COMMON(0xf20, PACA_EXGEN)
1148 #ifdef CONFIG_ALTIVEC
1149 BEGIN_FTR_SECTION
1150         beq     1f
1151         bl      .load_up_altivec
1152         b       fast_exception_return
1153 1:
1154 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
1155 #endif
1156         bl      .save_nvgprs
1157         DISABLE_INTS
1158         addi    r3,r1,STACK_FRAME_OVERHEAD
1159         bl      .altivec_unavailable_exception
1160         b       .ret_from_except
1161
1162         .align  7
1163         .globl vsx_unavailable_common
1164 vsx_unavailable_common:
1165         EXCEPTION_PROLOG_COMMON(0xf40, PACA_EXGEN)
1166 #ifdef CONFIG_VSX
1167 BEGIN_FTR_SECTION
1168         beq     1f
1169         b       .load_up_vsx
1170 1:
1171 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
1172 #endif
1173         bl      .save_nvgprs
1174         DISABLE_INTS
1175         addi    r3,r1,STACK_FRAME_OVERHEAD
1176         bl      .vsx_unavailable_exception
1177         b       .ret_from_except
1178
1179         .align  7
1180         .globl  __end_handlers
1181 __end_handlers:
1182
1183 /*
1184  * Hash table stuff
1185  */
1186         .align  7
1187 _STATIC(do_hash_page)
1188         std     r3,_DAR(r1)
1189         std     r4,_DSISR(r1)
1190
1191         andis.  r0,r4,0xa410            /* weird error? */
1192         bne-    handle_page_fault       /* if not, try to insert a HPTE */
1193         andis.  r0,r4,DSISR_DABRMATCH@h
1194         bne-    handle_dabr_fault
1195
1196 BEGIN_FTR_SECTION
1197         andis.  r0,r4,0x0020            /* Is it a segment table fault? */
1198         bne-    do_ste_alloc            /* If so handle it */
1199 END_MMU_FTR_SECTION_IFCLR(MMU_FTR_SLB)
1200
1201         CURRENT_THREAD_INFO(r11, r1)
1202         lwz     r0,TI_PREEMPT(r11)      /* If we're in an "NMI" */
1203         andis.  r0,r0,NMI_MASK@h        /* (i.e. an irq when soft-disabled) */
1204         bne     77f                     /* then don't call hash_page now */
1205         /*
1206          * We need to set the _PAGE_USER bit if MSR_PR is set or if we are
1207          * accessing a userspace segment (even from the kernel). We assume
1208          * kernel addresses always have the high bit set.
1209          */
1210         rlwinm  r4,r4,32-25+9,31-9,31-9 /* DSISR_STORE -> _PAGE_RW */
1211         rotldi  r0,r3,15                /* Move high bit into MSR_PR posn */
1212         orc     r0,r12,r0               /* MSR_PR | ~high_bit */
1213         rlwimi  r4,r0,32-13,30,30       /* becomes _PAGE_USER access bit */
1214         ori     r4,r4,1                 /* add _PAGE_PRESENT */
1215         rlwimi  r4,r5,22+2,31-2,31-2    /* Set _PAGE_EXEC if trap is 0x400 */
1216
1217         /*
1218          * r3 contains the faulting address
1219          * r4 contains the required access permissions
1220          * r5 contains the trap number
1221          *
1222          * at return r3 = 0 for success, 1 for page fault, negative for error
1223          */
1224         bl      .hash_page              /* build HPTE if possible */
1225         cmpdi   r3,0                    /* see if hash_page succeeded */
1226
1227         /* Success */
1228         beq     fast_exc_return_irq     /* Return from exception on success */
1229
1230         /* Error */
1231         blt-    13f
1232
1233 /* Here we have a page fault that hash_page can't handle. */
1234 handle_page_fault:
1235 11:     ld      r4,_DAR(r1)
1236         ld      r5,_DSISR(r1)
1237         addi    r3,r1,STACK_FRAME_OVERHEAD
1238         bl      .do_page_fault
1239         cmpdi   r3,0
1240         beq+    12f
1241         bl      .save_nvgprs
1242         mr      r5,r3
1243         addi    r3,r1,STACK_FRAME_OVERHEAD
1244         lwz     r4,_DAR(r1)
1245         bl      .bad_page_fault
1246         b       .ret_from_except
1247
1248 /* We have a data breakpoint exception - handle it */
1249 handle_dabr_fault:
1250         bl      .save_nvgprs
1251         ld      r4,_DAR(r1)
1252         ld      r5,_DSISR(r1)
1253         addi    r3,r1,STACK_FRAME_OVERHEAD
1254         bl      .do_dabr
1255 12:     b       .ret_from_except_lite
1256
1257
1258 /* We have a page fault that hash_page could handle but HV refused
1259  * the PTE insertion
1260  */
1261 13:     bl      .save_nvgprs
1262         mr      r5,r3
1263         addi    r3,r1,STACK_FRAME_OVERHEAD
1264         ld      r4,_DAR(r1)
1265         bl      .low_hash_fault
1266         b       .ret_from_except
1267
1268 /*
1269  * We come here as a result of a DSI at a point where we don't want
1270  * to call hash_page, such as when we are accessing memory (possibly
1271  * user memory) inside a PMU interrupt that occurred while interrupts
1272  * were soft-disabled.  We want to invoke the exception handler for
1273  * the access, or panic if there isn't a handler.
1274  */
1275 77:     bl      .save_nvgprs
1276         mr      r4,r3
1277         addi    r3,r1,STACK_FRAME_OVERHEAD
1278         li      r5,SIGSEGV
1279         bl      .bad_page_fault
1280         b       .ret_from_except
1281
1282         /* here we have a segment miss */
1283 do_ste_alloc:
1284         bl      .ste_allocate           /* try to insert stab entry */
1285         cmpdi   r3,0
1286         bne-    handle_page_fault
1287         b       fast_exception_return
1288
1289 /*
1290  * r13 points to the PACA, r9 contains the saved CR,
1291  * r11 and r12 contain the saved SRR0 and SRR1.
1292  * r9 - r13 are saved in paca->exslb.
1293  * We assume we aren't going to take any exceptions during this procedure.
1294  * We assume (DAR >> 60) == 0xc.
1295  */
1296         .align  7
1297 _GLOBAL(do_stab_bolted)
1298         stw     r9,PACA_EXSLB+EX_CCR(r13)       /* save CR in exc. frame */
1299         std     r11,PACA_EXSLB+EX_SRR0(r13)     /* save SRR0 in exc. frame */
1300
1301         /* Hash to the primary group */
1302         ld      r10,PACASTABVIRT(r13)
1303         mfspr   r11,SPRN_DAR
1304         srdi    r11,r11,28
1305         rldimi  r10,r11,7,52    /* r10 = first ste of the group */
1306
1307         /* Calculate VSID */
1308         /* This is a kernel address, so protovsid = ESID | 1 << 37 */
1309         li      r9,0x1
1310         rldimi  r11,r9,(CONTEXT_BITS + USER_ESID_BITS),0
1311         ASM_VSID_SCRAMBLE(r11, r9, 256M)
1312         rldic   r9,r11,12,16    /* r9 = vsid << 12 */
1313
1314         /* Search the primary group for a free entry */
1315 1:      ld      r11,0(r10)      /* Test valid bit of the current ste    */
1316         andi.   r11,r11,0x80
1317         beq     2f
1318         addi    r10,r10,16
1319         andi.   r11,r10,0x70
1320         bne     1b
1321
1322         /* Stick for only searching the primary group for now.          */
1323         /* At least for now, we use a very simple random castout scheme */
1324         /* Use the TB as a random number ;  OR in 1 to avoid entry 0    */
1325         mftb    r11
1326         rldic   r11,r11,4,57    /* r11 = (r11 << 4) & 0x70 */
1327         ori     r11,r11,0x10
1328
1329         /* r10 currently points to an ste one past the group of interest */
1330         /* make it point to the randomly selected entry                 */
1331         subi    r10,r10,128
1332         or      r10,r10,r11     /* r10 is the entry to invalidate       */
1333
1334         isync                   /* mark the entry invalid               */
1335         ld      r11,0(r10)
1336         rldicl  r11,r11,56,1    /* clear the valid bit */
1337         rotldi  r11,r11,8
1338         std     r11,0(r10)
1339         sync
1340
1341         clrrdi  r11,r11,28      /* Get the esid part of the ste         */
1342         slbie   r11
1343
1344 2:      std     r9,8(r10)       /* Store the vsid part of the ste       */
1345         eieio
1346
1347         mfspr   r11,SPRN_DAR            /* Get the new esid                     */
1348         clrrdi  r11,r11,28      /* Permits a full 32b of ESID           */
1349         ori     r11,r11,0x90    /* Turn on valid and kp                 */
1350         std     r11,0(r10)      /* Put new entry back into the stab     */
1351
1352         sync
1353
1354         /* All done -- return from exception. */
1355         lwz     r9,PACA_EXSLB+EX_CCR(r13)       /* get saved CR */
1356         ld      r11,PACA_EXSLB+EX_SRR0(r13)     /* get saved SRR0 */
1357
1358         andi.   r10,r12,MSR_RI
1359         beq-    unrecov_slb
1360
1361         mtcrf   0x80,r9                 /* restore CR */
1362
1363         mfmsr   r10
1364         clrrdi  r10,r10,2
1365         mtmsrd  r10,1
1366
1367         mtspr   SPRN_SRR0,r11
1368         mtspr   SPRN_SRR1,r12
1369         ld      r9,PACA_EXSLB+EX_R9(r13)
1370         ld      r10,PACA_EXSLB+EX_R10(r13)
1371         ld      r11,PACA_EXSLB+EX_R11(r13)
1372         ld      r12,PACA_EXSLB+EX_R12(r13)
1373         ld      r13,PACA_EXSLB+EX_R13(r13)
1374         rfid
1375         b       .       /* prevent speculative execution */
1376
1377
1378         /* Equivalents to the above handlers for relocation-on interrupt vectors */
1379         STD_RELON_EXCEPTION_HV(., 0xe00, h_data_storage)
1380         KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe00)
1381         STD_RELON_EXCEPTION_HV(., 0xe20, h_instr_storage)
1382         KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe20)
1383         STD_RELON_EXCEPTION_HV(., 0xe40, emulation_assist)
1384         KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe40)
1385         STD_RELON_EXCEPTION_HV(., 0xe60, hmi_exception)
1386         KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe60)
1387         MASKABLE_RELON_EXCEPTION_HV(., 0xe80, h_doorbell)
1388         KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe80)
1389
1390         STD_RELON_EXCEPTION_PSERIES(., 0xf00, performance_monitor)
1391         STD_RELON_EXCEPTION_PSERIES(., 0xf20, altivec_unavailable)
1392         STD_RELON_EXCEPTION_PSERIES(., 0xf40, vsx_unavailable)
1393
1394 #if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV)
1395 /*
1396  * Data area reserved for FWNMI option.
1397  * This address (0x7000) is fixed by the RPA.
1398  */
1399         .= 0x7000
1400         .globl fwnmi_data_area
1401 fwnmi_data_area:
1402
1403         /* pseries and powernv need to keep the whole page from
1404          * 0x7000 to 0x8000 free for use by the firmware
1405          */
1406         . = 0x8000
1407 #endif /* defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV) */
1408
1409 /* Space for CPU0's segment table */
1410         .balign 4096
1411         .globl initial_stab
1412 initial_stab:
1413         .space  4096
1414
1415 #ifdef CONFIG_PPC_POWERNV
1416 _GLOBAL(opal_mc_secondary_handler)
1417         HMT_MEDIUM_PPR_DISCARD
1418         SET_SCRATCH0(r13)
1419         GET_PACA(r13)
1420         clrldi  r3,r3,2
1421         tovirt(r3,r3)
1422         std     r3,PACA_OPAL_MC_EVT(r13)
1423         ld      r13,OPAL_MC_SRR0(r3)
1424         mtspr   SPRN_SRR0,r13
1425         ld      r13,OPAL_MC_SRR1(r3)
1426         mtspr   SPRN_SRR1,r13
1427         ld      r3,OPAL_MC_GPR3(r3)
1428         GET_SCRATCH0(r13)
1429         b       machine_check_pSeries
1430 #endif /* CONFIG_PPC_POWERNV */