Merge branch '3.1-fixes-for-rmk' of git://linux-arm.org/linux-2.6-wd into fixes

[pandora-kernel.git] / mm / swap.c

diff --git a/mm/swap.c b/mm/swap.c
index 5602f1a..3a442f1 100644 (file)
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -272,14 +272,10 @@ static void update_page_reclaim_stat(struct zone *zone, struct page *page,
                memcg_reclaim_stat->recent_rotated[file]++;
 }
 
-/*
- * FIXME: speed this up?
- */
-void activate_page(struct page *page)
+static void __activate_page(struct page *page, void *arg)
 {
        struct zone *zone = page_zone(page);
 
-       spin_lock_irq(&zone->lru_lock);
        if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
                int file = page_is_file_cache(page);
                int lru = page_lru_base_type(page);
@@ -292,8 +288,45 @@ void activate_page(struct page *page)
 
                update_page_reclaim_stat(zone, page, file, 1);
        }
+}
+
+#ifdef CONFIG_SMP
+static DEFINE_PER_CPU(struct pagevec, activate_page_pvecs);
+
+static void activate_page_drain(int cpu)
+{
+       struct pagevec *pvec = &per_cpu(activate_page_pvecs, cpu);
+
+       if (pagevec_count(pvec))
+               pagevec_lru_move_fn(pvec, __activate_page, NULL);
+}
+
+void activate_page(struct page *page)
+{
+       if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
+               struct pagevec *pvec = &get_cpu_var(activate_page_pvecs);
+
+               page_cache_get(page);
+               if (!pagevec_add(pvec, page))
+                       pagevec_lru_move_fn(pvec, __activate_page, NULL);
+               put_cpu_var(activate_page_pvecs);
+       }
+}
+
+#else
+static inline void activate_page_drain(int cpu)
+{
+}
+
+void activate_page(struct page *page)
+{
+       struct zone *zone = page_zone(page);
+
+       spin_lock_irq(&zone->lru_lock);
+       __activate_page(page, NULL);
        spin_unlock_irq(&zone->lru_lock);
 }
+#endif
 
 /*
  * Mark a page as having seen activity.
@@ -464,6 +497,8 @@ static void drain_cpu_pagevecs(int cpu)
        pvec = &per_cpu(lru_deactivate_pvecs, cpu);
        if (pagevec_count(pvec))
                pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
+
+       activate_page_drain(cpu);
 }
 
 /**
@@ -476,6 +511,13 @@ static void drain_cpu_pagevecs(int cpu)
  */
 void deactivate_page(struct page *page)
 {
+       /*
+        * In a workload with many unevictable page such as mprotect, unevictable
+        * page deactivation for accelerating reclaim is pointless.
+        */
+       if (PageUnevictable(page))
+               return;
+
        if (likely(get_page_unless_zero(page))) {
                struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs);