net/mlx4_en: Fix mixed PFC and Global pause user control requests

[pandora-kernel.git] / mm / rmap.c

diff --git a/mm/rmap.c b/mm/rmap.c
index ae27d95..98f0bf7 100644 (file)
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -72,6 +72,8 @@ static inline struct anon_vma *anon_vma_alloc(void)
        anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL);
        if (anon_vma) {
                atomic_set(&anon_vma->refcount, 1);
+               anon_vma->degree = 1;   /* Reference for first vma */
+               anon_vma->parent = anon_vma;
                /*
                 * Initialise the anon_vma root to point to itself. If called
                 * from fork, the root will be reset to the parents anon_vma.
@@ -181,6 +183,8 @@ int anon_vma_prepare(struct vm_area_struct *vma)
                        avc->vma = vma;
                        list_add(&avc->same_vma, &vma->anon_vma_chain);
                        list_add_tail(&avc->same_anon_vma, &anon_vma->head);
+                       /* vma reference or self-parent link for new root */
+                       anon_vma->degree++;
                        allocated = NULL;
                        avc = NULL;
                }
@@ -244,6 +248,14 @@ static void anon_vma_chain_link(struct vm_area_struct *vma,
 /*
  * Attach the anon_vmas from src to dst.
  * Returns 0 on success, -ENOMEM on failure.
+ *
+ * If dst->anon_vma is NULL this function tries to find and reuse existing
+ * anon_vma which has no vmas and only one child anon_vma. This prevents
+ * degradation of anon_vma hierarchy to endless linear chain in case of
+ * constantly forking task. On the other hand, an anon_vma with more than one
+ * child isn't reused even if there was no alive vma, thus rmap walker has a
+ * good chance of avoiding scanning the whole hierarchy when it searches where
+ * page is mapped.
  */
 int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src)
 {
@@ -264,11 +276,32 @@ int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src)
                anon_vma = pavc->anon_vma;
                root = lock_anon_vma_root(root, anon_vma);
                anon_vma_chain_link(dst, avc, anon_vma);
+
+               /*
+                * Reuse existing anon_vma if its degree lower than two,
+                * that means it has no vma and only one anon_vma child.
+                *
+                * Do not chose parent anon_vma, otherwise first child
+                * will always reuse it. Root anon_vma is never reused:
+                * it has self-parent reference and at least one child.
+                */
+               if (!dst->anon_vma && anon_vma != src->anon_vma &&
+                               anon_vma->degree < 2)
+                       dst->anon_vma = anon_vma;
        }
+       if (dst->anon_vma)
+               dst->anon_vma->degree++;
        unlock_anon_vma_root(root);
        return 0;
 
  enomem_failure:
+       /*
+        * dst->anon_vma is dropped here otherwise its degree can be incorrectly
+        * decremented in unlink_anon_vmas().
+        * We can safely do this because callers of anon_vma_clone() don't care
+        * about dst->anon_vma if anon_vma_clone() failed.
+        */
+       dst->anon_vma = NULL;
        unlink_anon_vmas(dst);
        return -ENOMEM;
 }
@@ -287,6 +320,9 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
        if (!pvma->anon_vma)
                return 0;
 
+       /* Drop inherited anon_vma, we'll reuse existing or allocate new. */
+       vma->anon_vma = NULL;
+
        /*
         * First, attach the new VMA to the parent VMA's anon_vmas,
         * so rmap can find non-COWed pages in child processes.
@@ -294,6 +330,10 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
        if (anon_vma_clone(vma, pvma))
                return -ENOMEM;
 
+       /* An existing anon_vma has been reused, all done then. */
+       if (vma->anon_vma)
+               return 0;
+
        /* Then add our own anon_vma. */
        anon_vma = anon_vma_alloc();
        if (!anon_vma)
@@ -307,6 +347,7 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
         * lock any of the anon_vmas in this anon_vma tree.
         */
        anon_vma->root = pvma->anon_vma->root;
+       anon_vma->parent = pvma->anon_vma;
        /*
         * With refcounts, an anon_vma can stay around longer than the
         * process it belongs to. The root anon_vma needs to be pinned until
@@ -317,6 +358,7 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
        vma->anon_vma = anon_vma;
        anon_vma_lock(anon_vma);
        anon_vma_chain_link(vma, avc, anon_vma);
+       anon_vma->parent->degree++;
        anon_vma_unlock(anon_vma);
 
        return 0;
@@ -347,12 +389,16 @@ void unlink_anon_vmas(struct vm_area_struct *vma)
                 * Leave empty anon_vmas on the list - we'll need
                 * to free them outside the lock.
                 */
-               if (list_empty(&anon_vma->head))
+               if (list_empty(&anon_vma->head)) {
+                       anon_vma->parent->degree--;
                        continue;
+               }
 
                list_del(&avc->same_vma);
                anon_vma_chain_free(avc);
        }
+       if (vma->anon_vma)
+               vma->anon_vma->degree--;
        unlock_anon_vma_root(root);
 
        /*
@@ -363,6 +409,7 @@ void unlink_anon_vmas(struct vm_area_struct *vma)
        list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
                struct anon_vma *anon_vma = avc->anon_vma;
 
+               BUG_ON(anon_vma->degree);
                put_anon_vma(anon_vma);
 
                list_del(&avc->same_vma);
@@ -1671,10 +1718,9 @@ void __put_anon_vma(struct anon_vma *anon_vma)
 {
        struct anon_vma *root = anon_vma->root;
 
+       anon_vma_free(anon_vma);
        if (root != anon_vma && atomic_dec_and_test(&root->refcount))
                anon_vma_free(root);
-
-       anon_vma_free(anon_vma);
 }
 
 #ifdef CONFIG_MIGRATION