tmpfs: convert shmem_getpage_gfp to radix-swap

Convert shmem_getpage_gfp(), the engine-room of shmem, to expect page or
swap entry returned from radix tree by find_lock_page().

Whereas the repetitive old method proceeded mainly under info->lock,
dropping and repeating whenever one of the conditions needed was not
met, now we can proceed without it, leaving shmem_add_to_page_cache() to
check for a race.

This way there is no need to preallocate a page, no need for an early
radix_tree_preload(), no need for mem_cgroup_shmem_charge_fallback().

Move the error unwinding down to the bottom instead of repeating it
throughout.  ENOSPC handling is a little different from before: there is
no longer any race between find_lock_page() and finding swap, but we can
arrive at ENOSPC before calling shmem_recalc_inode(), which might
occasionally discover freed space.

Be stricter to check i_size before returning.  info->lock is used for
little but alloced, swapped, i_blocks updates.  Move i_blocks updates
out from under the max_blocks check, so even an unlimited size=0 mount
can show accurate du.

Signed-off-by: Hugh Dickins <hughd@google.com>
Acked-by: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/mm/shmem.c b/mm/shmem.c
index 174f971..92f01d7 100644 (file)
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -166,15 +166,6 @@ static struct backing_dev_info shmem_backing_dev_info  __read_mostly = {
 static LIST_HEAD(shmem_swaplist);
 static DEFINE_MUTEX(shmem_swaplist_mutex);
 
 static LIST_HEAD(shmem_swaplist);
 static DEFINE_MUTEX(shmem_swaplist_mutex);
 

-static void shmem_free_blocks(struct inode *inode, long pages)
-{
-       struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
-       if (sbinfo->max_blocks) {
-               percpu_counter_add(&sbinfo->used_blocks, -pages);
-               inode->i_blocks -= pages*BLOCKS_PER_PAGE;
-       }
-}
-

 static int shmem_reserve_inode(struct super_block *sb)
 {
        struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
 static int shmem_reserve_inode(struct super_block *sb)
 {
        struct shmem_sb_info *sbinfo = SHMEM_SB(sb);


@@ -219,9 +210,12 @@ static void shmem_recalc_inode(struct inode *inode)
 
        freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
        if (freed > 0) {
 
        freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
        if (freed > 0) {

+               struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
+               if (sbinfo->max_blocks)
+                       percpu_counter_add(&sbinfo->used_blocks, -freed);

                info->alloced -= freed;
                info->alloced -= freed;

+               inode->i_blocks -= freed * BLOCKS_PER_PAGE;

                shmem_unacct_blocks(info->flags, freed);
                shmem_unacct_blocks(info->flags, freed);

-               shmem_free_blocks(inode, freed);

        }
 }
 
        }
 }
 


@@ -888,205 +882,180 @@ static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
        struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type)
 {
        struct address_space *mapping = inode->i_mapping;
        struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type)
 {
        struct address_space *mapping = inode->i_mapping;

-       struct shmem_inode_info *info = SHMEM_I(inode);
+       struct shmem_inode_info *info;

        struct shmem_sb_info *sbinfo;
        struct page *page;
        struct shmem_sb_info *sbinfo;
        struct page *page;

-       struct page *prealloc_page = NULL;

        swp_entry_t swap;
        int error;
        swp_entry_t swap;
        int error;

+       int once = 0;

 
        if (index > (MAX_LFS_FILESIZE >> PAGE_CACHE_SHIFT))
                return -EFBIG;
 repeat:
 
        if (index > (MAX_LFS_FILESIZE >> PAGE_CACHE_SHIFT))
                return -EFBIG;
 repeat:

+       swap.val = 0;

        page = find_lock_page(mapping, index);
        page = find_lock_page(mapping, index);

-       if (page) {
+       if (radix_tree_exceptional_entry(page)) {
+               swap = radix_to_swp_entry(page);
+               page = NULL;
+       }
+
+       if (sgp != SGP_WRITE &&
+           ((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
+               error = -EINVAL;
+               goto failed;
+       }
+
+       if (page || (sgp == SGP_READ && !swap.val)) {

                /*
                 * Once we can get the page lock, it must be uptodate:
                 * if there were an error in reading back from swap,
                 * the page would not be inserted into the filecache.
                 */
                /*
                 * Once we can get the page lock, it must be uptodate:
                 * if there were an error in reading back from swap,
                 * the page would not be inserted into the filecache.
                 */

-               BUG_ON(!PageUptodate(page));
-               goto done;
+               BUG_ON(page && !PageUptodate(page));
+               *pagep = page;
+               return 0;

        }
 
        /*
        }
 
        /*

-        * Try to preload while we can wait, to not make a habit of
-        * draining atomic reserves; but don't latch on to this cpu.
+        * Fast cache lookup did not find it:
+        * bring it back from swap or allocate.

         */
         */

-       error = radix_tree_preload(gfp & GFP_RECLAIM_MASK);
-       if (error)
-               goto out;
-       radix_tree_preload_end();
-
-       if (sgp != SGP_READ && !prealloc_page) {
-               prealloc_page = shmem_alloc_page(gfp, info, index);
-               if (prealloc_page) {
-                       SetPageSwapBacked(prealloc_page);
-                       if (mem_cgroup_cache_charge(prealloc_page,
-                                       current->mm, GFP_KERNEL)) {
-                               page_cache_release(prealloc_page);
-                               prealloc_page = NULL;
-                       }
-               }
-       }
+       info = SHMEM_I(inode);
+       sbinfo = SHMEM_SB(inode->i_sb);

 
 

-       spin_lock(&info->lock);
-       shmem_recalc_inode(inode);
-       swap = shmem_get_swap(info, index);

        if (swap.val) {
                /* Look it up and read it in.. */
                page = lookup_swap_cache(swap);
                if (!page) {
        if (swap.val) {
                /* Look it up and read it in.. */
                page = lookup_swap_cache(swap);
                if (!page) {

-                       spin_unlock(&info->lock);

                        /* here we actually do the io */
                        if (fault_type)
                                *fault_type |= VM_FAULT_MAJOR;
                        page = shmem_swapin(swap, gfp, info, index);
                        if (!page) {
                        /* here we actually do the io */
                        if (fault_type)
                                *fault_type |= VM_FAULT_MAJOR;
                        page = shmem_swapin(swap, gfp, info, index);
                        if (!page) {

-                               swp_entry_t nswap = shmem_get_swap(info, index);
-                               if (nswap.val == swap.val) {
-                                       error = -ENOMEM;
-                                       goto out;
-                               }
-                               goto repeat;
+                               error = -ENOMEM;
+                               goto failed;

                        }
                        }

-                       wait_on_page_locked(page);
-                       page_cache_release(page);
-                       goto repeat;

                }
 
                /* We have to do this with page locked to prevent races */
                }
 
                /* We have to do this with page locked to prevent races */

-               if (!trylock_page(page)) {
-                       spin_unlock(&info->lock);
-                       wait_on_page_locked(page);
-                       page_cache_release(page);
-                       goto repeat;
-               }
-               if (PageWriteback(page)) {
-                       spin_unlock(&info->lock);
-                       wait_on_page_writeback(page);
-                       unlock_page(page);
-                       page_cache_release(page);
-                       goto repeat;
-               }
+               lock_page(page);

                if (!PageUptodate(page)) {
                if (!PageUptodate(page)) {

-                       spin_unlock(&info->lock);
-                       unlock_page(page);
-                       page_cache_release(page);

                        error = -EIO;
                        error = -EIO;

-                       goto out;
+                       goto failed;

                }
                }

-
-               error = add_to_page_cache_locked(page, mapping,
-                                                index, GFP_NOWAIT);
-               if (error) {
-                       spin_unlock(&info->lock);
-                       if (error == -ENOMEM) {
-                               /*
-                                * reclaim from proper memory cgroup and
-                                * call memcg's OOM if needed.
-                                */
-                               error = mem_cgroup_shmem_charge_fallback(
-                                               page, current->mm, gfp);
-                               if (error) {
-                                       unlock_page(page);
-                                       page_cache_release(page);
-                                       goto out;
-                               }
-                       }
-                       unlock_page(page);
-                       page_cache_release(page);
-                       goto repeat;
+               wait_on_page_writeback(page);
+
+               /* Someone may have already done it for us */
+               if (page->mapping) {
+                       if (page->mapping == mapping &&
+                           page->index == index)
+                               goto done;
+                       error = -EEXIST;
+                       goto failed;

                }
 
                }
 

-               delete_from_swap_cache(page);
-               shmem_put_swap(info, index, (swp_entry_t){0});
+               error = shmem_add_to_page_cache(page, mapping, index,
+                                       gfp, swp_to_radix_entry(swap));
+               if (error)
+                       goto failed;
+
+               spin_lock(&info->lock);

                info->swapped--;
                info->swapped--;

+               shmem_recalc_inode(inode);

                spin_unlock(&info->lock);
                spin_unlock(&info->lock);

+
+               delete_from_swap_cache(page);

                set_page_dirty(page);
                swap_free(swap);
 
                set_page_dirty(page);
                swap_free(swap);
 

-       } else if (sgp == SGP_READ) {
-               page = find_get_page(mapping, index);
-               if (page && !trylock_page(page)) {
-                       spin_unlock(&info->lock);
-                       wait_on_page_locked(page);
-                       page_cache_release(page);
-                       goto repeat;
+       } else {
+               if (shmem_acct_block(info->flags)) {
+                       error = -ENOSPC;
+                       goto failed;

                }
                }

-               spin_unlock(&info->lock);
-
-       } else if (prealloc_page) {
-               sbinfo = SHMEM_SB(inode->i_sb);

                if (sbinfo->max_blocks) {
                        if (percpu_counter_compare(&sbinfo->used_blocks,
                if (sbinfo->max_blocks) {
                        if (percpu_counter_compare(&sbinfo->used_blocks,

-                                               sbinfo->max_blocks) >= 0 ||
-                           shmem_acct_block(info->flags))
-                               goto nospace;
+                                               sbinfo->max_blocks) >= 0) {
+                               error = -ENOSPC;
+                               goto unacct;
+                       }

                        percpu_counter_inc(&sbinfo->used_blocks);
                        percpu_counter_inc(&sbinfo->used_blocks);

-                       inode->i_blocks += BLOCKS_PER_PAGE;
-               } else if (shmem_acct_block(info->flags))
-                       goto nospace;
-
-               page = prealloc_page;
-               prealloc_page = NULL;
+               }

 
 

-               swap = shmem_get_swap(info, index);
-               if (swap.val)
-                       mem_cgroup_uncharge_cache_page(page);
-               else
-                       error = add_to_page_cache_lru(page, mapping,
-                                               index, GFP_NOWAIT);
-               /*
-                * At add_to_page_cache_lru() failure,
-                * uncharge will be done automatically.
-                */
-               if (swap.val || error) {
-                       shmem_unacct_blocks(info->flags, 1);
-                       shmem_free_blocks(inode, 1);
-                       spin_unlock(&info->lock);
-                       page_cache_release(page);
-                       goto repeat;
+               page = shmem_alloc_page(gfp, info, index);
+               if (!page) {
+                       error = -ENOMEM;
+                       goto decused;

                }
 
                }
 

+               SetPageSwapBacked(page);
+               __set_page_locked(page);
+               error = shmem_add_to_page_cache(page, mapping, index,
+                                                               gfp, NULL);
+               if (error)
+                       goto decused;
+               lru_cache_add_anon(page);
+
+               spin_lock(&info->lock);

                info->alloced++;
                info->alloced++;

+               inode->i_blocks += BLOCKS_PER_PAGE;
+               shmem_recalc_inode(inode);

                spin_unlock(&info->lock);
                spin_unlock(&info->lock);

+

                clear_highpage(page);
                flush_dcache_page(page);
                SetPageUptodate(page);
                if (sgp == SGP_DIRTY)
                        set_page_dirty(page);
                clear_highpage(page);
                flush_dcache_page(page);
                SetPageUptodate(page);
                if (sgp == SGP_DIRTY)
                        set_page_dirty(page);

-
-       } else {
-               spin_unlock(&info->lock);
-               error = -ENOMEM;
-               goto out;

        }
 done:
        }
 done:

-       *pagep = page;
-       error = 0;
-out:
-       if (prealloc_page) {
-               mem_cgroup_uncharge_cache_page(prealloc_page);
-               page_cache_release(prealloc_page);
+       /* Perhaps the file has been truncated since we checked */
+       if (sgp != SGP_WRITE &&
+           ((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
+               error = -EINVAL;
+               goto trunc;

        }
        }

-       return error;
+       *pagep = page;
+       return 0;

 
 

-nospace:

        /*
        /*

-        * Perhaps the page was brought in from swap between find_lock_page
-        * and taking info->lock?  We allow for that at add_to_page_cache_lru,
-        * but must also avoid reporting a spurious ENOSPC while working on a
-        * full tmpfs.
+        * Error recovery.

         */
         */

-       page = find_get_page(mapping, index);
+trunc:
+       ClearPageDirty(page);
+       delete_from_page_cache(page);
+       spin_lock(&info->lock);
+       info->alloced--;
+       inode->i_blocks -= BLOCKS_PER_PAGE;

        spin_unlock(&info->lock);
        spin_unlock(&info->lock);

+decused:
+       if (sbinfo->max_blocks)
+               percpu_counter_add(&sbinfo->used_blocks, -1);
+unacct:
+       shmem_unacct_blocks(info->flags, 1);
+failed:
+       if (swap.val && error != -EINVAL) {
+               struct page *test = find_get_page(mapping, index);
+               if (test && !radix_tree_exceptional_entry(test))
+                       page_cache_release(test);
+               /* Have another try if the entry has changed */
+               if (test != swp_to_radix_entry(swap))
+                       error = -EEXIST;
+       }

        if (page) {
        if (page) {

+               unlock_page(page);

                page_cache_release(page);
                page_cache_release(page);

+       }
+       if (error == -ENOSPC && !once++) {
+               info = SHMEM_I(inode);
+               spin_lock(&info->lock);
+               shmem_recalc_inode(inode);
+               spin_unlock(&info->lock);

                goto repeat;
        }
                goto repeat;
        }

-       error = -ENOSPC;
-       goto out;
+       if (error == -EEXIST)
+               goto repeat;
+       return error;

 }
 
 static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 }
 
 static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)


@@ -1095,9 +1064,6 @@ static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
        int error;
        int ret = VM_FAULT_LOCKED;
 
        int error;
        int ret = VM_FAULT_LOCKED;
 

-       if (((loff_t)vmf->pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode))
-               return VM_FAULT_SIGBUS;
-

        error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret);
        if (error)
                return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
        error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret);
        if (error)
                return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);


@@ -2164,8 +2130,7 @@ static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
        if (config.max_inodes < inodes)
                goto out;
        /*
        if (config.max_inodes < inodes)
                goto out;
        /*

-        * Those tests also disallow limited->unlimited while any are in
-        * use, so i_blocks will always be zero when max_blocks is zero;
+        * Those tests disallow limited->unlimited while any are in use;

         * but we must separately disallow unlimited->limited, because
         * in that case we have no record of how much is already in use.
         */
         * but we must separately disallow unlimited->limited, because
         * in that case we have no record of how much is already in use.
         */