mm/rmap.c: cleanup ttu_flags

[pandora-kernel.git] / mm / rmap.c

diff --git a/mm/rmap.c b/mm/rmap.c
index 9c3e773..ea8e20d 100644 (file)
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -103,6 +103,7 @@ static inline void anon_vma_free(struct anon_vma *anon_vma)
         * LOCK should suffice since the actual taking of the lock must
         * happen _before_ what follows.
         */
+       might_sleep();
        if (rwsem_is_locked(&anon_vma->root->rwsem)) {
                anon_vma_lock_write(anon_vma);
                anon_vma_unlock_write(anon_vma);
@@ -426,8 +427,9 @@ struct anon_vma *page_get_anon_vma(struct page *page)
         * above cannot corrupt).
         */
        if (!page_mapped(page)) {
+               rcu_read_unlock();
                put_anon_vma(anon_vma);
-               anon_vma = NULL;
+               return NULL;
        }
 out:
        rcu_read_unlock();
@@ -477,9 +479,9 @@ struct anon_vma *page_lock_anon_vma_read(struct page *page)
        }
 
        if (!page_mapped(page)) {
+               rcu_read_unlock();
                put_anon_vma(anon_vma);
-               anon_vma = NULL;
-               goto out;
+               return NULL;
        }
 
        /* we pinned the anon_vma, its safe to sleep */
@@ -669,7 +671,7 @@ struct page_referenced_arg {
 /*
  * arg: page_referenced_arg will be passed
  */
-int page_referenced_one(struct page *page, struct vm_area_struct *vma,
+static int page_referenced_one(struct page *page, struct vm_area_struct *vma,
                        unsigned long address, void *arg)
 {
        struct mm_struct *mm = vma->vm_mm;
@@ -986,6 +988,12 @@ void do_page_add_anon_rmap(struct page *page,
 {
        int first = atomic_inc_and_test(&page->_mapcount);
        if (first) {
+               /*
+                * We use the irq-unsafe __{inc|mod}_zone_page_stat because
+                * these counters are not modified in interrupt context, and
+                * pte lock(a spinlock) is held, which implies preemption
+                * disabled.
+                */
                if (PageTransHuge(page))
                        __inc_zone_page_state(page,
                                              NR_ANON_TRANSPARENT_HUGEPAGES);
@@ -1024,11 +1032,25 @@ void page_add_new_anon_rmap(struct page *page,
        __mod_zone_page_state(page_zone(page), NR_ANON_PAGES,
                        hpage_nr_pages(page));
        __page_set_anon_rmap(page, vma, address, 1);
-       if (!mlocked_vma_newpage(vma, page)) {
+
+       VM_BUG_ON_PAGE(PageLRU(page), page);
+       if (likely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) != VM_LOCKED)) {
                SetPageActive(page);
                lru_cache_add(page);
-       } else
-               add_page_to_unevictable_list(page);
+               return;
+       }
+
+       if (!TestSetPageMlocked(page)) {
+               /*
+                * We use the irq-unsafe __mod_zone_page_stat because this
+                * counter is not modified from interrupt context, and the pte
+                * lock is held(spinlock), which implies preemption disabled.
+                */
+               __mod_zone_page_state(page_zone(page), NR_MLOCK,
+                                   hpage_nr_pages(page));
+               count_vm_event(UNEVICTABLE_PGMLOCKED);
+       }
+       add_page_to_unevictable_list(page);
 }
 
 /**
@@ -1077,6 +1099,11 @@ void page_remove_rmap(struct page *page)
        /*
         * Hugepages are not counted in NR_ANON_PAGES nor NR_FILE_MAPPED
         * and not charged by memcg for now.
+        *
+        * We use the irq-unsafe __{inc|mod}_zone_page_stat because
+        * these counters are not modified in interrupt context, and
+        * these counters are not modified in interrupt context, and
+        * pte lock(a spinlock) is held, which implies preemption disabled.
         */
        if (unlikely(PageHuge(page)))
                goto out;
@@ -1112,7 +1139,7 @@ out:
 /*
  * @arg: enum ttu_flags will be passed to this argument
  */
-int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
+static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
                     unsigned long address, void *arg)
 {
        struct mm_struct *mm = vma->vm_mm;
@@ -1135,7 +1162,7 @@ int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
                if (vma->vm_flags & VM_LOCKED)
                        goto out_mlock;
 
-               if (TTU_ACTION(flags) == TTU_MUNLOCK)
+               if (flags & TTU_MUNLOCK)
                        goto out_unmap;
        }
        if (!(flags & TTU_IGNORE_ACCESS)) {
@@ -1203,7 +1230,7 @@ int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
                         * pte. do_swap_page() will wait until the migration
                         * pte is removed and then restart fault handling.
                         */
-                       BUG_ON(TTU_ACTION(flags) != TTU_MIGRATION);
+                       BUG_ON(!(flags & TTU_MIGRATION));
                        entry = make_migration_entry(page, pte_write(pteval));
                }
                swp_pte = swp_entry_to_pte(entry);
@@ -1212,7 +1239,7 @@ int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
                set_pte_at(mm, address, pte, swp_pte);
                BUG_ON(pte_file(*pte));
        } else if (IS_ENABLED(CONFIG_MIGRATION) &&
-                  (TTU_ACTION(flags) == TTU_MIGRATION)) {
+                  (flags & TTU_MIGRATION)) {
                /* Establish migration entry for a file page */
                swp_entry_t entry;
                entry = make_migration_entry(page, pte_write(pteval));
@@ -1225,7 +1252,7 @@ int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
 
 out_unmap:
        pte_unmap_unlock(pte, ptl);
-       if (ret != SWAP_FAIL)
+       if (ret != SWAP_FAIL && !(flags & TTU_MUNLOCK))
                mmu_notifier_invalidate_page(mm, address);
 out:
        return ret;
@@ -1359,7 +1386,7 @@ static int try_to_unmap_cluster(unsigned long cursor, unsigned int *mapcount,
                if (page->index != linear_page_index(vma, address)) {
                        pte_t ptfile = pgoff_to_pte(page->index);
                        if (pte_soft_dirty(pteval))
-                               pte_file_mksoft_dirty(ptfile);
+                               ptfile = pte_file_mksoft_dirty(ptfile);
                        set_pte_at(mm, address, pte, ptfile);
                }
 
@@ -1512,7 +1539,7 @@ int try_to_unmap(struct page *page, enum ttu_flags flags)
         * locking requirements of exec(), migration skips
         * temporary VMAs until after exec() completes.
         */
-       if (flags & TTU_MIGRATION && !PageKsm(page) && PageAnon(page))
+       if ((flags & TTU_MIGRATION) && !PageKsm(page) && PageAnon(page))
                rwc.invalid_vma = invalid_migration_vma;
 
        ret = rmap_walk(page, &rwc);