arch/powerpc/mm/hash_low_64.S

   1 /*
   2  * ppc64 MMU hashtable management routines
   3  *
   4  * (c) Copyright IBM Corp. 2003, 2005
   5  *
   6  * Maintained by: Benjamin Herrenschmidt
   7  *                <benh@kernel.crashing.org>
   8  *
   9  * This file is covered by the GNU Public Licence v2 as
  10  * described in the kernel's COPYING file.
  11  */
  12
  13 #include <asm/reg.h>
  14 #include <asm/pgtable.h>
  15 #include <asm/mmu.h>
  16 #include <asm/page.h>
  17 #include <asm/types.h>
  18 #include <asm/ppc_asm.h>
  19 #include <asm/asm-offsets.h>
  20 #include <asm/cputable.h>
  21
  22         .text
  23
  24 /*
  25  * Stackframe:
  26  *
  27  *         +-> Back chain                       (SP + 256)
  28  *         |   General register save area       (SP + 112)
  29  *         |   Parameter save area              (SP + 48)
  30  *         |   TOC save area                    (SP + 40)
  31  *         |   link editor doubleword           (SP + 32)
  32  *         |   compiler doubleword              (SP + 24)
  33  *         |   LR save area                     (SP + 16)
  34  *         |   CR save area                     (SP + 8)
  35  * SP ---> +-- Back chain                       (SP + 0)
  36  */
  37 #define STACKFRAMESIZE  256
  38
  39 /* Save parameters offsets */
  40 #define STK_PARM(i)     (STACKFRAMESIZE + 48 + ((i)-3)*8)
  41
  42 /* Save non-volatile offsets */
  43 #define STK_REG(i)      (112 + ((i)-14)*8)
  44
  45
  46 #ifndef CONFIG_PPC_64K_PAGES
  47
  48 /*****************************************************************************
  49  *                                                                           *
  50  *           4K SW & 4K HW pages implementation                              *
  51  *                                                                           *
  52  *****************************************************************************/
  53
  54
  55 /*
  56  * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
  57  *               pte_t *ptep, unsigned long trap, int local, int ssize)
  58  *
  59  * Adds a 4K page to the hash table in a segment of 4K pages only
  60  */
  61
  62 _GLOBAL(__hash_page_4K)
  63         mflr    r0
  64         std     r0,16(r1)
  65         stdu    r1,-STACKFRAMESIZE(r1)
  66         /* Save all params that we need after a function call */
  67         std     r6,STK_PARM(r6)(r1)
  68         std     r8,STK_PARM(r8)(r1)
  69         std     r9,STK_PARM(r9)(r1)
  70
  71         /* Add _PAGE_PRESENT to access */
  72         ori     r4,r4,_PAGE_PRESENT
  73
  74         /* Save non-volatile registers.
  75          * r31 will hold "old PTE"
  76          * r30 is "new PTE"
  77          * r29 is "va"
  78          * r28 is a hash value
  79          * r27 is hashtab mask (maybe dynamic patched instead ?)
  80          */
  81         std     r27,STK_REG(r27)(r1)
  82         std     r28,STK_REG(r28)(r1)
  83         std     r29,STK_REG(r29)(r1)
  84         std     r30,STK_REG(r30)(r1)
  85         std     r31,STK_REG(r31)(r1)
  86
  87         /* Step 1:
  88          *
  89          * Check permissions, atomically mark the linux PTE busy
  90          * and hashed.
  91          */
  92 1:
  93         ldarx   r31,0,r6
  94         /* Check access rights (access & ~(pte_val(*ptep))) */
  95         andc.   r0,r4,r31
  96         bne-    htab_wrong_access
  97         /* Check if PTE is busy */
  98         andi.   r0,r31,_PAGE_BUSY
  99         /* If so, just bail out and refault if needed. Someone else
 100          * is changing this PTE anyway and might hash it.
 101          */
 102         bne-    htab_bail_ok
 103
 104         /* Prepare new PTE value (turn access RW into DIRTY, then
 105          * add BUSY,HASHPTE and ACCESSED)
 106          */
 107         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
 108         or      r30,r30,r31
 109         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
 110         /* Write the linux PTE atomically (setting busy) */
 111         stdcx.  r30,0,r6
 112         bne-    1b
 113         isync
 114
 115         /* Step 2:
 116          *
 117          * Insert/Update the HPTE in the hash table. At this point,
 118          * r4 (access) is re-useable, we use it for the new HPTE flags
 119          */
 120
 121 BEGIN_FTR_SECTION
 122         cmpdi   r9,0                    /* check segment size */
 123         bne     3f
 124 END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
 125         /* Calc va and put it in r29 */
 126         rldicr  r29,r5,28,63-28
 127         rldicl  r3,r3,0,36
 128         or      r29,r3,r29
 129
 130         /* Calculate hash value for primary slot and store it in r28 */
 131         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
 132         rldicl  r0,r3,64-12,48          /* (ea >> 12) & 0xffff */
 133         xor     r28,r5,r0
 134         b       4f
 135
 136 3:      /* Calc VA and hash in r29 and r28 for 1T segment */
 137         sldi    r29,r5,40               /* vsid << 40 */
 138         clrldi  r3,r3,24                /* ea & 0xffffffffff */
 139         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
 140         clrldi  r5,r5,40                /* vsid & 0xffffff */
 141         rldicl  r0,r3,64-12,36          /* (ea >> 12) & 0xfffffff */
 142         xor     r28,r28,r5
 143         or      r29,r3,r29              /* VA */
 144         xor     r28,r28,r0              /* hash */
 145
 146         /* Convert linux PTE bits into HW equivalents */
 147 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
 148         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 149         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 150         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 151         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 152         andc    r0,r30,r0               /* r0 = pte & ~r0 */
 153         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 154         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
 155
 156         /* We eventually do the icache sync here (maybe inline that
 157          * code rather than call a C function...)
 158          */
 159 BEGIN_FTR_SECTION
 160         mr      r4,r30
 161         mr      r5,r7
 162         bl      .hash_page_do_lazy_icache
 163 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 164
 165         /* At this point, r3 contains new PP bits, save them in
 166          * place of "access" in the param area (sic)
 167          */
 168         std     r3,STK_PARM(r4)(r1)
 169
 170         /* Get htab_hash_mask */
 171         ld      r4,htab_hash_mask@got(2)
 172         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 173
 174         /* Check if we may already be in the hashtable, in this case, we
 175          * go to out-of-line code to try to modify the HPTE
 176          */
 177         andi.   r0,r31,_PAGE_HASHPTE
 178         bne     htab_modify_pte
 179
 180 htab_insert_pte:
 181         /* Clear hpte bits in new pte (we also clear BUSY btw) and
 182          * add _PAGE_HASHPTE
 183          */
 184         lis     r0,_PAGE_HPTEFLAGS@h
 185         ori     r0,r0,_PAGE_HPTEFLAGS@l
 186         andc    r30,r30,r0
 187         ori     r30,r30,_PAGE_HASHPTE
 188
 189         /* physical address r5 */
 190         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 191         sldi    r5,r5,PAGE_SHIFT
 192
 193         /* Calculate primary group hash */
 194         and     r0,r28,r27
 195         rldicr  r3,r0,3,63-3            /* r3 = (hash & mask) << 3 */
 196
 197         /* Call ppc_md.hpte_insert */
 198         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 199         mr      r4,r29                  /* Retreive va */
 200         li      r7,0                    /* !bolted, !secondary */
 201         li      r8,MMU_PAGE_4K          /* page size */
 202         ld      r9,STK_PARM(r9)(r1)     /* segment size */
 203 _GLOBAL(htab_call_hpte_insert1)
 204         bl      .                       /* Patched by htab_finish_init() */
 205         cmpdi   0,r3,0
 206         bge     htab_pte_insert_ok      /* Insertion successful */
 207         cmpdi   0,r3,-2                 /* Critical failure */
 208         beq-    htab_pte_insert_failure
 209
 210         /* Now try secondary slot */
 211
 212         /* physical address r5 */
 213         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 214         sldi    r5,r5,PAGE_SHIFT
 215
 216         /* Calculate secondary group hash */
 217         andc    r0,r27,r28
 218         rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
 219
 220         /* Call ppc_md.hpte_insert */
 221         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 222         mr      r4,r29                  /* Retreive va */
 223         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 224         li      r8,MMU_PAGE_4K          /* page size */
 225         ld      r9,STK_PARM(r9)(r1)     /* segment size */
 226 _GLOBAL(htab_call_hpte_insert2)
 227         bl      .                       /* Patched by htab_finish_init() */
 228         cmpdi   0,r3,0
 229         bge+    htab_pte_insert_ok      /* Insertion successful */
 230         cmpdi   0,r3,-2                 /* Critical failure */
 231         beq-    htab_pte_insert_failure
 232
 233         /* Both are full, we need to evict something */
 234         mftb    r0
 235         /* Pick a random group based on TB */
 236         andi.   r0,r0,1
 237         mr      r5,r28
 238         bne     2f
 239         not     r5,r5
 240 2:      and     r0,r5,r27
 241         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 242         /* Call ppc_md.hpte_remove */
 243 _GLOBAL(htab_call_hpte_remove)
 244         bl      .                       /* Patched by htab_finish_init() */
 245
 246         /* Try all again */
 247         b       htab_insert_pte
 248
 249 htab_bail_ok:
 250         li      r3,0
 251         b       htab_bail
 252
 253 htab_pte_insert_ok:
 254         /* Insert slot number & secondary bit in PTE */
 255         rldimi  r30,r3,12,63-15
 256
 257         /* Write out the PTE with a normal write
 258          * (maybe add eieio may be good still ?)
 259          */
 260 htab_write_out_pte:
 261         ld      r6,STK_PARM(r6)(r1)
 262         std     r30,0(r6)
 263         li      r3, 0
 264 htab_bail:
 265         ld      r27,STK_REG(r27)(r1)
 266         ld      r28,STK_REG(r28)(r1)
 267         ld      r29,STK_REG(r29)(r1)
 268         ld      r30,STK_REG(r30)(r1)
 269         ld      r31,STK_REG(r31)(r1)
 270         addi    r1,r1,STACKFRAMESIZE
 271         ld      r0,16(r1)
 272         mtlr    r0
 273         blr
 274
 275 htab_modify_pte:
 276         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 277         mr      r4,r3
 278         rlwinm  r3,r31,32-12,29,31
 279
 280         /* Secondary group ? if yes, get a inverted hash value */
 281         mr      r5,r28
 282         andi.   r0,r31,_PAGE_SECONDARY
 283         beq     1f
 284         not     r5,r5
 285 1:
 286         /* Calculate proper slot value for ppc_md.hpte_updatepp */
 287         and     r0,r5,r27
 288         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 289         add     r3,r0,r3        /* add slot idx */
 290
 291         /* Call ppc_md.hpte_updatepp */
 292         mr      r5,r29                  /* va */
 293         li      r6,MMU_PAGE_4K          /* page size */
 294         ld      r7,STK_PARM(r9)(r1)     /* segment size */
 295         ld      r8,STK_PARM(r8)(r1)     /* get "local" param */
 296 _GLOBAL(htab_call_hpte_updatepp)
 297         bl      .                       /* Patched by htab_finish_init() */
 298
 299         /* if we failed because typically the HPTE wasn't really here
 300          * we try an insertion.
 301          */
 302         cmpdi   0,r3,-1
 303         beq-    htab_insert_pte
 304
 305         /* Clear the BUSY bit and Write out the PTE */
 306         li      r0,_PAGE_BUSY
 307         andc    r30,r30,r0
 308         b       htab_write_out_pte
 309
 310 htab_wrong_access:
 311         /* Bail out clearing reservation */
 312         stdcx.  r31,0,r6
 313         li      r3,1
 314         b       htab_bail
 315
 316 htab_pte_insert_failure:
 317         /* Bail out restoring old PTE */
 318         ld      r6,STK_PARM(r6)(r1)
 319         std     r31,0(r6)
 320         li      r3,-1
 321         b       htab_bail
 322
 323
 324 #else /* CONFIG_PPC_64K_PAGES */
 325
 326
 327 /*****************************************************************************
 328  *                                                                           *
 329  *           64K SW & 4K or 64K HW in a 4K segment pages implementation      *
 330  *                                                                           *
 331  *****************************************************************************/
 332
 333 /* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
 334  *               pte_t *ptep, unsigned long trap, int local, int ssize)
 335  */
 336
 337 /*
 338  * For now, we do NOT implement Admixed pages
 339  */
 340 _GLOBAL(__hash_page_4K)
 341         mflr    r0
 342         std     r0,16(r1)
 343         stdu    r1,-STACKFRAMESIZE(r1)
 344         /* Save all params that we need after a function call */
 345         std     r6,STK_PARM(r6)(r1)
 346         std     r8,STK_PARM(r8)(r1)
 347         std     r9,STK_PARM(r9)(r1)
 348
 349         /* Add _PAGE_PRESENT to access */
 350         ori     r4,r4,_PAGE_PRESENT
 351
 352         /* Save non-volatile registers.
 353          * r31 will hold "old PTE"
 354          * r30 is "new PTE"
 355          * r29 is "va"
 356          * r28 is a hash value
 357          * r27 is hashtab mask (maybe dynamic patched instead ?)
 358          * r26 is the hidx mask
 359          * r25 is the index in combo page
 360          */
 361         std     r25,STK_REG(r25)(r1)
 362         std     r26,STK_REG(r26)(r1)
 363         std     r27,STK_REG(r27)(r1)
 364         std     r28,STK_REG(r28)(r1)
 365         std     r29,STK_REG(r29)(r1)
 366         std     r30,STK_REG(r30)(r1)
 367         std     r31,STK_REG(r31)(r1)
 368
 369         /* Step 1:
 370          *
 371          * Check permissions, atomically mark the linux PTE busy
 372          * and hashed.
 373          */
 374 1:
 375         ldarx   r31,0,r6
 376         /* Check access rights (access & ~(pte_val(*ptep))) */
 377         andc.   r0,r4,r31
 378         bne-    htab_wrong_access
 379         /* Check if PTE is busy */
 380         andi.   r0,r31,_PAGE_BUSY
 381         /* If so, just bail out and refault if needed. Someone else
 382          * is changing this PTE anyway and might hash it.
 383          */
 384         bne-    htab_bail_ok
 385         /* Prepare new PTE value (turn access RW into DIRTY, then
 386          * add BUSY and ACCESSED)
 387          */
 388         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
 389         or      r30,r30,r31
 390         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
 391         oris    r30,r30,_PAGE_COMBO@h
 392         /* Write the linux PTE atomically (setting busy) */
 393         stdcx.  r30,0,r6
 394         bne-    1b
 395         isync
 396
 397         /* Step 2:
 398          *
 399          * Insert/Update the HPTE in the hash table. At this point,
 400          * r4 (access) is re-useable, we use it for the new HPTE flags
 401          */
 402
 403         /* Load the hidx index */
 404         rldicl  r25,r3,64-12,60
 405
 406 BEGIN_FTR_SECTION
 407         cmpdi   r9,0                    /* check segment size */
 408         bne     3f
 409 END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
 410         /* Calc va and put it in r29 */
 411         rldicr  r29,r5,28,63-28         /* r29 = (vsid << 28) */
 412         rldicl  r3,r3,0,36              /* r3 = (ea & 0x0fffffff) */
 413         or      r29,r3,r29              /* r29 = va */
 414
 415         /* Calculate hash value for primary slot and store it in r28 */
 416         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
 417         rldicl  r0,r3,64-12,48          /* (ea >> 12) & 0xffff */
 418         xor     r28,r5,r0
 419         b       4f
 420
 421 3:      /* Calc VA and hash in r29 and r28 for 1T segment */
 422         sldi    r29,r5,40               /* vsid << 40 */
 423         clrldi  r3,r3,24                /* ea & 0xffffffffff */
 424         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
 425         clrldi  r5,r5,40                /* vsid & 0xffffff */
 426         rldicl  r0,r3,64-12,36          /* (ea >> 12) & 0xfffffff */
 427         xor     r28,r28,r5
 428         or      r29,r3,r29              /* VA */
 429         xor     r28,r28,r0              /* hash */
 430
 431         /* Convert linux PTE bits into HW equivalents */
 432 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
 433         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 434         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 435         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 436         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 437         andc    r0,r30,r0               /* r0 = pte & ~r0 */
 438         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 439         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
 440
 441         /* We eventually do the icache sync here (maybe inline that
 442          * code rather than call a C function...)
 443          */
 444 BEGIN_FTR_SECTION
 445         mr      r4,r30
 446         mr      r5,r7
 447         bl      .hash_page_do_lazy_icache
 448 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 449
 450         /* At this point, r3 contains new PP bits, save them in
 451          * place of "access" in the param area (sic)
 452          */
 453         std     r3,STK_PARM(r4)(r1)
 454
 455         /* Get htab_hash_mask */
 456         ld      r4,htab_hash_mask@got(2)
 457         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 458
 459         /* Check if we may already be in the hashtable, in this case, we
 460          * go to out-of-line code to try to modify the HPTE. We look for
 461          * the bit at (1 >> (index + 32))
 462          */
 463         andi.   r0,r31,_PAGE_HASHPTE
 464         li      r26,0                   /* Default hidx */
 465         beq     htab_insert_pte
 466
 467         /*
 468          * Check if the pte was already inserted into the hash table
 469          * as a 64k HW page, and invalidate the 64k HPTE if so.
 470          */
 471         andis.  r0,r31,_PAGE_COMBO@h
 472         beq     htab_inval_old_hpte
 473
 474         ld      r6,STK_PARM(r6)(r1)
 475         ori     r26,r6,0x8000           /* Load the hidx mask */
 476         ld      r26,0(r26)
 477         addi    r5,r25,36               /* Check actual HPTE_SUB bit, this */
 478         rldcr.  r0,r31,r5,0             /* must match pgtable.h definition */
 479         bne     htab_modify_pte
 480
 481 htab_insert_pte:
 482         /* real page number in r5, PTE RPN value + index */
 483         andis.  r0,r31,_PAGE_4K_PFN@h
 484         srdi    r5,r31,PTE_RPN_SHIFT
 485         bne-    htab_special_pfn
 486         sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
 487         add     r5,r5,r25
 488 htab_special_pfn:
 489         sldi    r5,r5,HW_PAGE_SHIFT
 490
 491         /* Calculate primary group hash */
 492         and     r0,r28,r27
 493         rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
 494
 495         /* Call ppc_md.hpte_insert */
 496         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 497         mr      r4,r29                  /* Retreive va */
 498         li      r7,0                    /* !bolted, !secondary */
 499         li      r8,MMU_PAGE_4K          /* page size */
 500         ld      r9,STK_PARM(r9)(r1)     /* segment size */
 501 _GLOBAL(htab_call_hpte_insert1)
 502         bl      .                       /* patched by htab_finish_init() */
 503         cmpdi   0,r3,0
 504         bge     htab_pte_insert_ok      /* Insertion successful */
 505         cmpdi   0,r3,-2                 /* Critical failure */
 506         beq-    htab_pte_insert_failure
 507
 508         /* Now try secondary slot */
 509
 510         /* real page number in r5, PTE RPN value + index */
 511         andis.  r0,r31,_PAGE_4K_PFN@h
 512         srdi    r5,r31,PTE_RPN_SHIFT
 513         bne-    3f
 514         sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
 515         add     r5,r5,r25
 516 3:      sldi    r5,r5,HW_PAGE_SHIFT
 517
 518         /* Calculate secondary group hash */
 519         andc    r0,r27,r28
 520         rldicr  r3,r0,3,63-3            /* r0 = (~hash & mask) << 3 */
 521
 522         /* Call ppc_md.hpte_insert */
 523         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 524         mr      r4,r29                  /* Retreive va */
 525         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 526         li      r8,MMU_PAGE_4K          /* page size */
 527         ld      r9,STK_PARM(r9)(r1)     /* segment size */
 528 _GLOBAL(htab_call_hpte_insert2)
 529         bl      .                       /* patched by htab_finish_init() */
 530         cmpdi   0,r3,0
 531         bge+    htab_pte_insert_ok      /* Insertion successful */
 532         cmpdi   0,r3,-2                 /* Critical failure */
 533         beq-    htab_pte_insert_failure
 534
 535         /* Both are full, we need to evict something */
 536         mftb    r0
 537         /* Pick a random group based on TB */
 538         andi.   r0,r0,1
 539         mr      r5,r28
 540         bne     2f
 541         not     r5,r5
 542 2:      and     r0,r5,r27
 543         rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
 544         /* Call ppc_md.hpte_remove */
 545 _GLOBAL(htab_call_hpte_remove)
 546         bl      .                       /* patched by htab_finish_init() */
 547
 548         /* Try all again */
 549         b       htab_insert_pte
 550
 551         /*
 552          * Call out to C code to invalidate an 64k HW HPTE that is
 553          * useless now that the segment has been switched to 4k pages.
 554          */
 555 htab_inval_old_hpte:
 556         mr      r3,r29                  /* virtual addr */
 557         mr      r4,r31                  /* PTE.pte */
 558         li      r5,0                    /* PTE.hidx */
 559         li      r6,MMU_PAGE_64K         /* psize */
 560         ld      r7,STK_PARM(r9)(r1)     /* ssize */
 561         ld      r8,STK_PARM(r8)(r1)     /* local */
 562         bl      .flush_hash_page
 563         b       htab_insert_pte
 564
 565 htab_bail_ok:
 566         li      r3,0
 567         b       htab_bail
 568
 569 htab_pte_insert_ok:
 570         /* Insert slot number & secondary bit in PTE second half,
 571          * clear _PAGE_BUSY and set approriate HPTE slot bit
 572          */
 573         ld      r6,STK_PARM(r6)(r1)
 574         li      r0,_PAGE_BUSY
 575         andc    r30,r30,r0
 576         /* HPTE SUB bit */
 577         li      r0,1
 578         subfic  r5,r25,27               /* Must match bit position in */
 579         sld     r0,r0,r5                /* pgtable.h */
 580         or      r30,r30,r0
 581         /* hindx */
 582         sldi    r5,r25,2
 583         sld     r3,r3,r5
 584         li      r4,0xf
 585         sld     r4,r4,r5
 586         andc    r26,r26,r4
 587         or      r26,r26,r3
 588         ori     r5,r6,0x8000
 589         std     r26,0(r5)
 590         lwsync
 591         std     r30,0(r6)
 592         li      r3, 0
 593 htab_bail:
 594         ld      r25,STK_REG(r25)(r1)
 595         ld      r26,STK_REG(r26)(r1)
 596         ld      r27,STK_REG(r27)(r1)
 597         ld      r28,STK_REG(r28)(r1)
 598         ld      r29,STK_REG(r29)(r1)
 599         ld      r30,STK_REG(r30)(r1)
 600         ld      r31,STK_REG(r31)(r1)
 601         addi    r1,r1,STACKFRAMESIZE
 602         ld      r0,16(r1)
 603         mtlr    r0
 604         blr
 605
 606 htab_modify_pte:
 607         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 608         mr      r4,r3
 609         sldi    r5,r25,2
 610         srd     r3,r26,r5
 611
 612         /* Secondary group ? if yes, get a inverted hash value */
 613         mr      r5,r28
 614         andi.   r0,r3,0x8 /* page secondary ? */
 615         beq     1f
 616         not     r5,r5
 617 1:      andi.   r3,r3,0x7 /* extract idx alone */
 618
 619         /* Calculate proper slot value for ppc_md.hpte_updatepp */
 620         and     r0,r5,r27
 621         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 622         add     r3,r0,r3        /* add slot idx */
 623
 624         /* Call ppc_md.hpte_updatepp */
 625         mr      r5,r29                  /* va */
 626         li      r6,MMU_PAGE_4K          /* page size */
 627         ld      r7,STK_PARM(r9)(r1)     /* segment size */
 628         ld      r8,STK_PARM(r8)(r1)     /* get "local" param */
 629 _GLOBAL(htab_call_hpte_updatepp)
 630         bl      .                       /* patched by htab_finish_init() */
 631
 632         /* if we failed because typically the HPTE wasn't really here
 633          * we try an insertion.
 634          */
 635         cmpdi   0,r3,-1
 636         beq-    htab_insert_pte
 637
 638         /* Clear the BUSY bit and Write out the PTE */
 639         li      r0,_PAGE_BUSY
 640         andc    r30,r30,r0
 641         ld      r6,STK_PARM(r6)(r1)
 642         std     r30,0(r6)
 643         li      r3,0
 644         b       htab_bail
 645
 646 htab_wrong_access:
 647         /* Bail out clearing reservation */
 648         stdcx.  r31,0,r6
 649         li      r3,1
 650         b       htab_bail
 651
 652 htab_pte_insert_failure:
 653         /* Bail out restoring old PTE */
 654         ld      r6,STK_PARM(r6)(r1)
 655         std     r31,0(r6)
 656         li      r3,-1
 657         b       htab_bail
 658
 659 #endif /* CONFIG_PPC_64K_PAGES */
 660
 661 #ifdef CONFIG_PPC_HAS_HASH_64K
 662
 663 /*****************************************************************************
 664  *                                                                           *
 665  *           64K SW & 64K HW in a 64K segment pages implementation           *
 666  *                                                                           *
 667  *****************************************************************************/
 668
 669 _GLOBAL(__hash_page_64K)
 670         mflr    r0
 671         std     r0,16(r1)
 672         stdu    r1,-STACKFRAMESIZE(r1)
 673         /* Save all params that we need after a function call */
 674         std     r6,STK_PARM(r6)(r1)
 675         std     r8,STK_PARM(r8)(r1)
 676         std     r9,STK_PARM(r9)(r1)
 677
 678         /* Add _PAGE_PRESENT to access */
 679         ori     r4,r4,_PAGE_PRESENT
 680
 681         /* Save non-volatile registers.
 682          * r31 will hold "old PTE"
 683          * r30 is "new PTE"
 684          * r29 is "va"
 685          * r28 is a hash value
 686          * r27 is hashtab mask (maybe dynamic patched instead ?)
 687          */
 688         std     r27,STK_REG(r27)(r1)
 689         std     r28,STK_REG(r28)(r1)
 690         std     r29,STK_REG(r29)(r1)
 691         std     r30,STK_REG(r30)(r1)
 692         std     r31,STK_REG(r31)(r1)
 693
 694         /* Step 1:
 695          *
 696          * Check permissions, atomically mark the linux PTE busy
 697          * and hashed.
 698          */
 699 1:
 700         ldarx   r31,0,r6
 701         /* Check access rights (access & ~(pte_val(*ptep))) */
 702         andc.   r0,r4,r31
 703         bne-    ht64_wrong_access
 704         /* Check if PTE is busy */
 705         andi.   r0,r31,_PAGE_BUSY
 706         /* If so, just bail out and refault if needed. Someone else
 707          * is changing this PTE anyway and might hash it.
 708          */
 709         bne-    ht64_bail_ok
 710 BEGIN_FTR_SECTION
 711         /* Check if PTE has the cache-inhibit bit set */
 712         andi.   r0,r31,_PAGE_NO_CACHE
 713         /* If so, bail out and refault as a 4k page */
 714         bne-    ht64_bail_ok
 715 END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE)
 716         /* Prepare new PTE value (turn access RW into DIRTY, then
 717          * add BUSY,HASHPTE and ACCESSED)
 718          */
 719         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
 720         or      r30,r30,r31
 721         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
 722         /* Write the linux PTE atomically (setting busy) */
 723         stdcx.  r30,0,r6
 724         bne-    1b
 725         isync
 726
 727         /* Step 2:
 728          *
 729          * Insert/Update the HPTE in the hash table. At this point,
 730          * r4 (access) is re-useable, we use it for the new HPTE flags
 731          */
 732
 733 BEGIN_FTR_SECTION
 734         cmpdi   r9,0                    /* check segment size */
 735         bne     3f
 736 END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
 737         /* Calc va and put it in r29 */
 738         rldicr  r29,r5,28,63-28
 739         rldicl  r3,r3,0,36
 740         or      r29,r3,r29
 741
 742         /* Calculate hash value for primary slot and store it in r28 */
 743         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
 744         rldicl  r0,r3,64-16,52          /* (ea >> 16) & 0xfff */
 745         xor     r28,r5,r0
 746         b       4f
 747
 748 3:      /* Calc VA and hash in r29 and r28 for 1T segment */
 749         sldi    r29,r5,40               /* vsid << 40 */
 750         clrldi  r3,r3,24                /* ea & 0xffffffffff */
 751         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
 752         clrldi  r5,r5,40                /* vsid & 0xffffff */
 753         rldicl  r0,r3,64-16,40          /* (ea >> 16) & 0xffffff */
 754         xor     r28,r28,r5
 755         or      r29,r3,r29              /* VA */
 756         xor     r28,r28,r0              /* hash */
 757
 758         /* Convert linux PTE bits into HW equivalents */
 759 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
 760         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 761         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 762         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 763         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 764         andc    r0,r30,r0               /* r0 = pte & ~r0 */
 765         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 766         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
 767
 768         /* We eventually do the icache sync here (maybe inline that
 769          * code rather than call a C function...)
 770          */
 771 BEGIN_FTR_SECTION
 772         mr      r4,r30
 773         mr      r5,r7
 774         bl      .hash_page_do_lazy_icache
 775 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 776
 777         /* At this point, r3 contains new PP bits, save them in
 778          * place of "access" in the param area (sic)
 779          */
 780         std     r3,STK_PARM(r4)(r1)
 781
 782         /* Get htab_hash_mask */
 783         ld      r4,htab_hash_mask@got(2)
 784         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 785
 786         /* Check if we may already be in the hashtable, in this case, we
 787          * go to out-of-line code to try to modify the HPTE
 788          */
 789         andi.   r0,r31,_PAGE_HASHPTE
 790         bne     ht64_modify_pte
 791
 792 ht64_insert_pte:
 793         /* Clear hpte bits in new pte (we also clear BUSY btw) and
 794          * add _PAGE_HASHPTE
 795          */
 796         lis     r0,_PAGE_HPTEFLAGS@h
 797         ori     r0,r0,_PAGE_HPTEFLAGS@l
 798         andc    r30,r30,r0
 799         ori     r30,r30,_PAGE_HASHPTE
 800
 801         /* Phyical address in r5 */
 802         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 803         sldi    r5,r5,PAGE_SHIFT
 804
 805         /* Calculate primary group hash */
 806         and     r0,r28,r27
 807         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 808
 809         /* Call ppc_md.hpte_insert */
 810         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 811         mr      r4,r29                  /* Retreive va */
 812         li      r7,0                    /* !bolted, !secondary */
 813         li      r8,MMU_PAGE_64K
 814         ld      r9,STK_PARM(r9)(r1)     /* segment size */
 815 _GLOBAL(ht64_call_hpte_insert1)
 816         bl      .                       /* patched by htab_finish_init() */
 817         cmpdi   0,r3,0
 818         bge     ht64_pte_insert_ok      /* Insertion successful */
 819         cmpdi   0,r3,-2                 /* Critical failure */
 820         beq-    ht64_pte_insert_failure
 821
 822         /* Now try secondary slot */
 823
 824         /* Phyical address in r5 */
 825         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 826         sldi    r5,r5,PAGE_SHIFT
 827
 828         /* Calculate secondary group hash */
 829         andc    r0,r27,r28
 830         rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
 831
 832         /* Call ppc_md.hpte_insert */
 833         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 834         mr      r4,r29                  /* Retreive va */
 835         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 836         li      r8,MMU_PAGE_64K
 837         ld      r9,STK_PARM(r9)(r1)     /* segment size */
 838 _GLOBAL(ht64_call_hpte_insert2)
 839         bl      .                       /* patched by htab_finish_init() */
 840         cmpdi   0,r3,0
 841         bge+    ht64_pte_insert_ok      /* Insertion successful */
 842         cmpdi   0,r3,-2                 /* Critical failure */
 843         beq-    ht64_pte_insert_failure
 844
 845         /* Both are full, we need to evict something */
 846         mftb    r0
 847         /* Pick a random group based on TB */
 848         andi.   r0,r0,1
 849         mr      r5,r28
 850         bne     2f
 851         not     r5,r5
 852 2:      and     r0,r5,r27
 853         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 854         /* Call ppc_md.hpte_remove */
 855 _GLOBAL(ht64_call_hpte_remove)
 856         bl      .                       /* patched by htab_finish_init() */
 857
 858         /* Try all again */
 859         b       ht64_insert_pte
 860
 861 ht64_bail_ok:
 862         li      r3,0
 863         b       ht64_bail
 864
 865 ht64_pte_insert_ok:
 866         /* Insert slot number & secondary bit in PTE */
 867         rldimi  r30,r3,12,63-15
 868
 869         /* Write out the PTE with a normal write
 870          * (maybe add eieio may be good still ?)
 871          */
 872 ht64_write_out_pte:
 873         ld      r6,STK_PARM(r6)(r1)
 874         std     r30,0(r6)
 875         li      r3, 0
 876 ht64_bail:
 877         ld      r27,STK_REG(r27)(r1)
 878         ld      r28,STK_REG(r28)(r1)
 879         ld      r29,STK_REG(r29)(r1)
 880         ld      r30,STK_REG(r30)(r1)
 881         ld      r31,STK_REG(r31)(r1)
 882         addi    r1,r1,STACKFRAMESIZE
 883         ld      r0,16(r1)
 884         mtlr    r0
 885         blr
 886
 887 ht64_modify_pte:
 888         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 889         mr      r4,r3
 890         rlwinm  r3,r31,32-12,29,31
 891
 892         /* Secondary group ? if yes, get a inverted hash value */
 893         mr      r5,r28
 894         andi.   r0,r31,_PAGE_F_SECOND
 895         beq     1f
 896         not     r5,r5
 897 1:
 898         /* Calculate proper slot value for ppc_md.hpte_updatepp */
 899         and     r0,r5,r27
 900         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 901         add     r3,r0,r3        /* add slot idx */
 902
 903         /* Call ppc_md.hpte_updatepp */
 904         mr      r5,r29                  /* va */
 905         li      r6,MMU_PAGE_64K
 906         ld      r7,STK_PARM(r9)(r1)     /* segment size */
 907         ld      r8,STK_PARM(r8)(r1)     /* get "local" param */
 908 _GLOBAL(ht64_call_hpte_updatepp)
 909         bl      .                       /* patched by htab_finish_init() */
 910
 911         /* if we failed because typically the HPTE wasn't really here
 912          * we try an insertion.
 913          */
 914         cmpdi   0,r3,-1
 915         beq-    ht64_insert_pte
 916
 917         /* Clear the BUSY bit and Write out the PTE */
 918         li      r0,_PAGE_BUSY
 919         andc    r30,r30,r0
 920         b       ht64_write_out_pte
 921
 922 ht64_wrong_access:
 923         /* Bail out clearing reservation */
 924         stdcx.  r31,0,r6
 925         li      r3,1
 926         b       ht64_bail
 927
 928 ht64_pte_insert_failure:
 929         /* Bail out restoring old PTE */
 930         ld      r6,STK_PARM(r6)(r1)
 931         std     r31,0(r6)
 932         li      r3,-1
 933         b       ht64_bail
 934
 935
 936 #endif /* CONFIG_PPC_HAS_HASH_64K */
 937
 938
 939 /*****************************************************************************
 940  *                                                                           *
 941  *           Huge pages implementation is in hugetlbpage.c                   *
 942  *                                                                           *
 943  *****************************************************************************/