X-Git-Url: https://git.openpandora.org/cgi-bin/gitweb.cgi?p=pandora-kernel.git;a=blobdiff_plain;f=arch%2Fpowerpc%2Fmm%2Fhugetlbpage.c;h=0d12fba31bc596d13ceaf58ea8d04c23663082ac;hp=4835f73af3044b7b57d6451cfc7c777a3806f1eb;hb=5b664cb235e97afbf34db9c4d77f08ebd725335e;hpb=efa7e8673c78cc6de2d6c367eb6f50449c57ed90 diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index 4835f73af304..0d12fba31bc5 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -22,23 +22,19 @@ #include #include #include -#include #include -#include +#define HPAGE_SHIFT_64K 16 +#define HPAGE_SHIFT_16M 24 #define NUM_LOW_AREAS (0x100000000UL >> SID_SHIFT) #define NUM_HIGH_AREAS (PGTABLE_RANGE >> HTLB_AREA_SHIFT) -#ifdef CONFIG_PPC_64K_PAGES -#define HUGEPTE_INDEX_SIZE (PMD_SHIFT-HPAGE_SHIFT) -#else -#define HUGEPTE_INDEX_SIZE (PUD_SHIFT-HPAGE_SHIFT) -#endif -#define PTRS_PER_HUGEPTE (1 << HUGEPTE_INDEX_SIZE) -#define HUGEPTE_TABLE_SIZE (sizeof(pte_t) << HUGEPTE_INDEX_SIZE) +unsigned int hugepte_shift; +#define PTRS_PER_HUGEPTE (1 << hugepte_shift) +#define HUGEPTE_TABLE_SIZE (sizeof(pte_t) << hugepte_shift) -#define HUGEPD_SHIFT (HPAGE_SHIFT + HUGEPTE_INDEX_SIZE) +#define HUGEPD_SHIFT (HPAGE_SHIFT + hugepte_shift) #define HUGEPD_SIZE (1UL << HUGEPD_SHIFT) #define HUGEPD_MASK (~(HUGEPD_SIZE-1)) @@ -85,11 +81,35 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp, return 0; } +/* Base page size affects how we walk hugetlb page tables */ +#ifdef CONFIG_PPC_64K_PAGES +#define hpmd_offset(pud, addr) pmd_offset(pud, addr) +#define hpmd_alloc(mm, pud, addr) pmd_alloc(mm, pud, addr) +#else +static inline +pmd_t *hpmd_offset(pud_t *pud, unsigned long addr) +{ + if (HPAGE_SHIFT == HPAGE_SHIFT_64K) + return pmd_offset(pud, addr); + else + return (pmd_t *) pud; +} +static inline +pmd_t *hpmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long addr) +{ + if (HPAGE_SHIFT == HPAGE_SHIFT_64K) + return pmd_alloc(mm, pud, addr); + else + return (pmd_t *) pud; +} +#endif + /* Modelled after find_linux_pte() */ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) { pgd_t *pg; pud_t *pu; + pmd_t *pm; BUG_ON(get_slice_psize(mm, addr) != mmu_huge_psize); @@ -99,14 +119,9 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) if (!pgd_none(*pg)) { pu = pud_offset(pg, addr); if (!pud_none(*pu)) { -#ifdef CONFIG_PPC_64K_PAGES - pmd_t *pm; - pm = pmd_offset(pu, addr); + pm = hpmd_offset(pu, addr); if (!pmd_none(*pm)) return hugepte_offset((hugepd_t *)pm, addr); -#else - return hugepte_offset((hugepd_t *)pu, addr); -#endif } } @@ -117,6 +132,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr) { pgd_t *pg; pud_t *pu; + pmd_t *pm; hugepd_t *hpdp = NULL; BUG_ON(get_slice_psize(mm, addr) != mmu_huge_psize); @@ -127,14 +143,9 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr) pu = pud_alloc(mm, pg, addr); if (pu) { -#ifdef CONFIG_PPC_64K_PAGES - pmd_t *pm; - pm = pmd_alloc(mm, pu, addr); + pm = hpmd_alloc(mm, pu, addr); if (pm) hpdp = (hugepd_t *)pm; -#else - hpdp = (hugepd_t *)pu; -#endif } if (! hpdp) @@ -161,7 +172,6 @@ static void free_hugepte_range(struct mmu_gather *tlb, hugepd_t *hpdp) PGF_CACHENUM_MASK)); } -#ifdef CONFIG_PPC_64K_PAGES static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud, unsigned long addr, unsigned long end, unsigned long floor, unsigned long ceiling) @@ -194,7 +204,6 @@ static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud, pud_clear(pud); pmd_free_tlb(tlb, pmd); } -#endif static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, unsigned long addr, unsigned long end, @@ -213,9 +222,15 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, continue; hugetlb_free_pmd_range(tlb, pud, addr, next, floor, ceiling); #else - if (pud_none(*pud)) - continue; - free_hugepte_range(tlb, (hugepd_t *)pud); + if (HPAGE_SHIFT == HPAGE_SHIFT_64K) { + if (pud_none_or_clear_bad(pud)) + continue; + hugetlb_free_pmd_range(tlb, pud, addr, next, floor, ceiling); + } else { + if (pud_none(*pud)) + continue; + free_hugepte_range(tlb, (hugepd_t *)pud); + } #endif } while (pud++, addr = next, addr != end); @@ -406,11 +421,12 @@ int hash_huge_page(struct mm_struct *mm, unsigned long access, unsigned long va, rflags, pa; long slot; int err = 1; + int ssize = user_segment_size(ea); ptep = huge_pte_offset(mm, ea); /* Search the Linux page table for a match with va */ - va = (vsid << 28) | (ea & 0x0fffffff); + va = hpt_va(ea, vsid, ssize); /* * If no pte found or not present, send the problem up to @@ -442,8 +458,7 @@ int hash_huge_page(struct mm_struct *mm, unsigned long access, old_pte = pte_val(*ptep); if (old_pte & _PAGE_BUSY) goto out; - new_pte = old_pte | _PAGE_BUSY | - _PAGE_ACCESSED | _PAGE_HASHPTE; + new_pte = old_pte | _PAGE_BUSY | _PAGE_ACCESSED; } while(old_pte != __cmpxchg_u64((unsigned long *)ptep, old_pte, new_pte)); @@ -461,19 +476,19 @@ int hash_huge_page(struct mm_struct *mm, unsigned long access, /* There MIGHT be an HPTE for this pte */ unsigned long hash, slot; - hash = hpt_hash(va, HPAGE_SHIFT); + hash = hpt_hash(va, HPAGE_SHIFT, ssize); if (old_pte & _PAGE_F_SECOND) hash = ~hash; slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; slot += (old_pte & _PAGE_F_GIX) >> 12; if (ppc_md.hpte_updatepp(slot, rflags, va, mmu_huge_psize, - local) == -1) + ssize, local) == -1) old_pte &= ~_PAGE_HPTEFLAGS; } if (likely(!(old_pte & _PAGE_HASHPTE))) { - unsigned long hash = hpt_hash(va, HPAGE_SHIFT); + unsigned long hash = hpt_hash(va, HPAGE_SHIFT, ssize); unsigned long hpte_group; pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT; @@ -483,16 +498,18 @@ repeat: HPTES_PER_GROUP) & ~0x7UL; /* clear HPTE slot informations in new PTE */ +#ifdef CONFIG_PPC_64K_PAGES + new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HPTE_SUB0; +#else new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HASHPTE; - +#endif /* Add in WIMG bits */ - /* XXX We should store these in the pte */ - /* --BenH: I think they are ... */ - rflags |= _PAGE_COHERENT; + rflags |= (new_pte & (_PAGE_WRITETHRU | _PAGE_NO_CACHE | + _PAGE_COHERENT | _PAGE_GUARDED)); /* Insert into the hash table, primary slot */ slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, 0, - mmu_huge_psize); + mmu_huge_psize, ssize); /* Primary is full, try the secondary */ if (unlikely(slot == -1)) { @@ -500,7 +517,7 @@ repeat: HPTES_PER_GROUP) & ~0x7UL; slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, HPTE_V_SECONDARY, - mmu_huge_psize); + mmu_huge_psize, ssize); if (slot == -1) { if (mftb() & 0x1) hpte_group = ((hash & htab_hash_mask) * @@ -528,7 +545,58 @@ repeat: return err; } -static void zero_ctor(void *addr, struct kmem_cache *cache, unsigned long flags) +void set_huge_psize(int psize) +{ + /* Check that it is a page size supported by the hardware and + * that it fits within pagetable limits. */ + if (mmu_psize_defs[psize].shift && mmu_psize_defs[psize].shift < SID_SHIFT && + (mmu_psize_defs[psize].shift > MIN_HUGEPTE_SHIFT || + mmu_psize_defs[psize].shift == HPAGE_SHIFT_64K)) { + HPAGE_SHIFT = mmu_psize_defs[psize].shift; + mmu_huge_psize = psize; +#ifdef CONFIG_PPC_64K_PAGES + hugepte_shift = (PMD_SHIFT-HPAGE_SHIFT); +#else + if (HPAGE_SHIFT == HPAGE_SHIFT_64K) + hugepte_shift = (PMD_SHIFT-HPAGE_SHIFT); + else + hugepte_shift = (PUD_SHIFT-HPAGE_SHIFT); +#endif + + } else + HPAGE_SHIFT = 0; +} + +static int __init hugepage_setup_sz(char *str) +{ + unsigned long long size; + int mmu_psize = -1; + int shift; + + size = memparse(str, &str); + + shift = __ffs(size); + switch (shift) { +#ifndef CONFIG_PPC_64K_PAGES + case HPAGE_SHIFT_64K: + mmu_psize = MMU_PAGE_64K; + break; +#endif + case HPAGE_SHIFT_16M: + mmu_psize = MMU_PAGE_16M; + break; + } + + if (mmu_psize >=0 && mmu_psize_defs[mmu_psize].shift) + set_huge_psize(mmu_psize); + else + printk(KERN_WARNING "Invalid huge page size specified(%llu)\n", size); + + return 1; +} +__setup("hugepagesz=", hugepage_setup_sz); + +static void zero_ctor(struct kmem_cache *cache, void *addr) { memset(addr, 0, kmem_cache_size(cache)); }