net: gro: selective flush of packets

[pandora-kernel.git] / net / core / dev.c

diff --git a/net/core/dev.c b/net/core/dev.c
index de2bad7..d44668f 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -3471,17 +3471,31 @@ out:
        return netif_receive_skb(skb);
 }
 
-inline void napi_gro_flush(struct napi_struct *napi)
+/* napi->gro_list contains packets ordered by age.
+ * youngest packets at the head of it.
+ * Complete skbs in reverse order to reduce latencies.
+ */
+void napi_gro_flush(struct napi_struct *napi, bool flush_old)
 {
-       struct sk_buff *skb, *next;
+       struct sk_buff *skb, *prev = NULL;
 
-       for (skb = napi->gro_list; skb; skb = next) {
-               next = skb->next;
+       /* scan list and build reverse chain */
+       for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
+               skb->prev = prev;
+               prev = skb;
+       }
+
+       for (skb = prev; skb; skb = prev) {
                skb->next = NULL;
+
+               if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
+                       return;
+
+               prev = skb->prev;
                napi_gro_complete(skb);
+               napi->gro_count--;
        }
 
-       napi->gro_count = 0;
        napi->gro_list = NULL;
 }
 EXPORT_SYMBOL(napi_gro_flush);
@@ -3542,6 +3556,7 @@ enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
 
        napi->gro_count++;
        NAPI_GRO_CB(skb)->count = 1;
+       NAPI_GRO_CB(skb)->age = jiffies;
        skb_shinfo(skb)->gso_size = skb_gro_len(skb);
        skb->next = napi->gro_list;
        napi->gro_list = skb;
@@ -3878,7 +3893,7 @@ void napi_complete(struct napi_struct *n)
        if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
                return;
 
-       napi_gro_flush(n);
+       napi_gro_flush(n, false);
        local_irq_save(flags);
        __napi_complete(n);
        local_irq_restore(flags);
@@ -3983,8 +3998,17 @@ static void net_rx_action(struct softirq_action *h)
                                local_irq_enable();
                                napi_complete(n);
                                local_irq_disable();
-                       } else
+                       } else {
+                               if (n->gro_list) {
+                                       /* flush too old packets
+                                        * If HZ < 1000, flush all packets.
+                                        */
+                                       local_irq_enable();
+                                       napi_gro_flush(n, HZ >= 1000);
+                                       local_irq_disable();
+                               }
                                list_move_tail(&n->poll_list, &sd->poll_list);
+                       }
                }
 
                netpoll_poll_unlock(have);