mm/huge_memory.c: fix up "mm/huge_memory.c: respect FOLL_FORCE/FOLL_COW for thp"...

[pandora-kernel.git] / mm / huge_memory.c

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index ed0ed8a..d6e6caf 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -682,7 +682,7 @@ int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
        if (haddr >= vma->vm_start && haddr + HPAGE_PMD_SIZE <= vma->vm_end) {
                if (unlikely(anon_vma_prepare(vma)))
                        return VM_FAULT_OOM;
-               if (unlikely(khugepaged_enter(vma)))
+               if (unlikely(khugepaged_enter(vma, vma->vm_flags)))
                        return VM_FAULT_OOM;
                page = alloc_hugepage_vma(transparent_hugepage_defrag(vma),
                                          vma, haddr, numa_node_id(), 0);
@@ -964,6 +964,18 @@ out:
        return ret;
 }
 
+/*
+ * FOLL_FORCE can write to even unwritable pmd's, but only
+ * after we've gone through a COW cycle and they are dirty.
+ */
+static inline bool can_follow_write_pmd(pmd_t pmd, struct page *page,
+                                       unsigned int flags)
+{
+       return pmd_write(pmd) ||
+               ((flags & FOLL_FORCE) && (flags & FOLL_COW) &&
+                page && PageAnon(page));
+}
+
 struct page *follow_trans_huge_pmd(struct mm_struct *mm,
                                   unsigned long addr,
                                   pmd_t *pmd,
@@ -973,11 +985,12 @@ struct page *follow_trans_huge_pmd(struct mm_struct *mm,
 
        assert_spin_locked(&mm->page_table_lock);
 
-       if (flags & FOLL_WRITE && !pmd_write(*pmd))
-               goto out;
-
        page = pmd_page(*pmd);
        VM_BUG_ON(!PageHead(page));
+
+       if (flags & FOLL_WRITE && !can_follow_write_pmd(*pmd, page, flags))
+               return NULL;
+
        if (flags & FOLL_TOUCH) {
                pmd_t _pmd;
                /*
@@ -1493,7 +1506,7 @@ int hugepage_madvise(struct vm_area_struct *vma,
                 * register it here without waiting a page fault that
                 * may not happen any time soon.
                 */
-               if (unlikely(khugepaged_enter_vma_merge(vma)))
+               if (unlikely(khugepaged_enter_vma_merge(vma, *vm_flags)))
                        return -ENOMEM;
                break;
        case MADV_NOHUGEPAGE:
@@ -1625,7 +1638,8 @@ int __khugepaged_enter(struct mm_struct *mm)
        return 0;
 }
 
-int khugepaged_enter_vma_merge(struct vm_area_struct *vma)
+int khugepaged_enter_vma_merge(struct vm_area_struct *vma,
+                              unsigned long vm_flags)
 {
        unsigned long hstart, hend;
        if (!vma->anon_vma)
@@ -1634,18 +1648,18 @@ int khugepaged_enter_vma_merge(struct vm_area_struct *vma)
                 * page fault if needed.
                 */
                return 0;
-       if (vma->vm_ops)
+       if (vma->vm_ops || (vm_flags & VM_NO_THP))
                /* khugepaged not yet working on file or special mappings */
                return 0;
        /*
         * If is_pfn_mapping() is true is_learn_pfn_mapping() must be
         * true too, verify it here.
         */
-       VM_BUG_ON(is_linear_pfn_mapping(vma) || vma->vm_flags & VM_NO_THP);
+       VM_BUG_ON(is_linear_pfn_mapping(vma));
        hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
        hend = vma->vm_end & HPAGE_PMD_MASK;
        if (hstart < hend)
-               return khugepaged_enter(vma);
+               return khugepaged_enter(vma, vm_flags);
        return 0;
 }
 
@@ -1816,6 +1830,24 @@ static void __collapse_huge_page_copy(pte_t *pte, struct page *page,
        }
 }
 
+static bool hugepage_vma_check(struct vm_area_struct *vma)
+{
+       if ((!(vma->vm_flags & VM_HUGEPAGE) && !khugepaged_always()) ||
+           (vma->vm_flags & VM_NOHUGEPAGE))
+               return false;
+
+       if (!vma->anon_vma || vma->vm_ops)
+               return false;
+       if (is_vma_temporary_stack(vma))
+               return false;
+       /*
+        * If is_pfn_mapping() is true is_learn_pfn_mapping() must be
+        * true too, verify it here.
+        */
+       VM_BUG_ON(is_linear_pfn_mapping(vma));
+       return !(vma->vm_flags & VM_NO_THP);
+}
+
 static void collapse_huge_page(struct mm_struct *mm,
                               unsigned long address,
                               struct page **hpage,
@@ -1888,21 +1920,8 @@ static void collapse_huge_page(struct mm_struct *mm,
        hend = vma->vm_end & HPAGE_PMD_MASK;
        if (address < hstart || address + HPAGE_PMD_SIZE > hend)
                goto out;
-
-       if ((!(vma->vm_flags & VM_HUGEPAGE) && !khugepaged_always()) ||
-           (vma->vm_flags & VM_NOHUGEPAGE))
-               goto out;
-
-       if (!vma->anon_vma || vma->vm_ops)
+       if (!hugepage_vma_check(vma))
                goto out;
-       if (is_vma_temporary_stack(vma))
-               goto out;
-       /*
-        * If is_pfn_mapping() is true is_learn_pfn_mapping() must be
-        * true too, verify it here.
-        */
-       VM_BUG_ON(is_linear_pfn_mapping(vma) || vma->vm_flags & VM_NO_THP);
-
        pgd = pgd_offset(mm, address);
        if (!pgd_present(*pgd))
                goto out;
@@ -2132,25 +2151,11 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages,
                        progress++;
                        break;
                }
-
-               if ((!(vma->vm_flags & VM_HUGEPAGE) &&
-                    !khugepaged_always()) ||
-                   (vma->vm_flags & VM_NOHUGEPAGE)) {
-               skip:
+               if (!hugepage_vma_check(vma)) {
+skip:
                        progress++;
                        continue;
                }
-               if (!vma->anon_vma || vma->vm_ops)
-                       goto skip;
-               if (is_vma_temporary_stack(vma))
-                       goto skip;
-               /*
-                * If is_pfn_mapping() is true is_learn_pfn_mapping()
-                * must be true too, verify it here.
-                */
-               VM_BUG_ON(is_linear_pfn_mapping(vma) ||
-                         vma->vm_flags & VM_NO_THP);
-
                hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
                hend = vma->vm_end & HPAGE_PMD_MASK;
                if (hstart >= hend)