net: allow better page reuse in splice(sock -> pipe)

[pandora-kernel.git] / net / core / skbuff.c

diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index baf8d28..bf257de 100644 (file)
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -67,8 +67,7 @@
 
 #include <asm/uaccess.h>
 #include <trace/events/skb.h>
-
-#include "kmap_skb.h"
+#include <linux/highmem.h>
 
 static struct kmem_cache *skbuff_head_cache __read_mostly;
 static struct kmem_cache *skbuff_fclone_cache __read_mostly;
@@ -707,10 +706,10 @@ int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
                        }
                        return -ENOMEM;
                }
-               vaddr = kmap_skb_frag(&skb_shinfo(skb)->frags[i]);
+               vaddr = kmap_atomic(skb_frag_page(f));
                memcpy(page_address(page),
                       vaddr + f->page_offset, skb_frag_size(f));
-               kunmap_skb_frag(vaddr);
+               kunmap_atomic(vaddr);
                page->private = (unsigned long)head;
                head = page;
        }
@@ -952,9 +951,11 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
                goto adjust_others;
        }
 
-       data = kmalloc(size + sizeof(struct skb_shared_info), gfp_mask);
+       data = kmalloc(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
+                      gfp_mask);
        if (!data)
                goto nodata;
+       size = SKB_WITH_OVERHEAD(ksize(data));
 
        /* Copy only real data... and, alas, header. This should be
         * optimized for the cases when header is void.
@@ -1273,7 +1274,7 @@ drop_pages:
                                return -ENOMEM;
 
                        nfrag->next = frag->next;
-                       kfree_skb(frag);
+                       consume_skb(frag);
                        frag = nfrag;
                        *fragp = frag;
                }
@@ -1485,21 +1486,22 @@ int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
 
        for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
                int end;
+               skb_frag_t *f = &skb_shinfo(skb)->frags[i];
 
                WARN_ON(start > offset + len);
 
-               end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
+               end = start + skb_frag_size(f);
                if ((copy = end - offset) > 0) {
                        u8 *vaddr;
 
                        if (copy > len)
                                copy = len;
 
-                       vaddr = kmap_skb_frag(&skb_shinfo(skb)->frags[i]);
+                       vaddr = kmap_atomic(skb_frag_page(f));
                        memcpy(to,
-                              vaddr + skb_shinfo(skb)->frags[i].page_offset+
-                              offset - start, copy);
-                       kunmap_skb_frag(vaddr);
+                              vaddr + f->page_offset + offset - start,
+                              copy);
+                       kunmap_atomic(vaddr);
 
                        if ((len -= copy) == 0)
                                return 0;
@@ -1563,6 +1565,9 @@ new_page:
        } else {
                unsigned int mlen;
 
+               /* If we are the only user of the page, we can reset offset */
+               if (page_count(p) == 1)
+                       sk->sk_sndmsg_off = 0;
                off = sk->sk_sndmsg_off;
                mlen = PAGE_SIZE - off;
                if (mlen < 64 && mlen < *len) {
@@ -1804,10 +1809,10 @@ int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
                        if (copy > len)
                                copy = len;
 
-                       vaddr = kmap_skb_frag(frag);
+                       vaddr = kmap_atomic(skb_frag_page(frag));
                        memcpy(vaddr + frag->page_offset + offset - start,
                               from, copy);
-                       kunmap_skb_frag(vaddr);
+                       kunmap_atomic(vaddr);
 
                        if ((len -= copy) == 0)
                                return 0;
@@ -1867,21 +1872,21 @@ __wsum skb_checksum(const struct sk_buff *skb, int offset,
 
        for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
                int end;
+               skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 
                WARN_ON(start > offset + len);
 
-               end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
+               end = start + skb_frag_size(frag);
                if ((copy = end - offset) > 0) {
                        __wsum csum2;
                        u8 *vaddr;
-                       skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 
                        if (copy > len)
                                copy = len;
-                       vaddr = kmap_skb_frag(frag);
+                       vaddr = kmap_atomic(skb_frag_page(frag));
                        csum2 = csum_partial(vaddr + frag->page_offset +
                                             offset - start, copy, 0);
-                       kunmap_skb_frag(vaddr);
+                       kunmap_atomic(vaddr);
                        csum = csum_block_add(csum, csum2, pos);
                        if (!(len -= copy))
                                return csum;
@@ -1953,12 +1958,12 @@ __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
 
                        if (copy > len)
                                copy = len;
-                       vaddr = kmap_skb_frag(frag);
+                       vaddr = kmap_atomic(skb_frag_page(frag));
                        csum2 = csum_partial_copy_nocheck(vaddr +
                                                          frag->page_offset +
                                                          offset - start, to,
                                                          copy, 0);
-                       kunmap_skb_frag(vaddr);
+                       kunmap_atomic(vaddr);
                        csum = csum_block_add(csum, csum2, pos);
                        if (!(len -= copy))
                                return csum;
@@ -2478,7 +2483,7 @@ next_skb:
 
                if (abs_offset < block_limit) {
                        if (!st->frag_data)
-                               st->frag_data = kmap_skb_frag(frag);
+                               st->frag_data = kmap_atomic(skb_frag_page(frag));
 
                        *data = (u8 *) st->frag_data + frag->page_offset +
                                (abs_offset - st->stepped_offset);
@@ -2487,7 +2492,7 @@ next_skb:
                }
 
                if (st->frag_data) {
-                       kunmap_skb_frag(st->frag_data);
+                       kunmap_atomic(st->frag_data);
                        st->frag_data = NULL;
                }
 
@@ -2496,7 +2501,7 @@ next_skb:
        }
 
        if (st->frag_data) {
-               kunmap_skb_frag(st->frag_data);
+               kunmap_atomic(st->frag_data);
                st->frag_data = NULL;
        }
 
@@ -2524,7 +2529,7 @@ EXPORT_SYMBOL(skb_seq_read);
 void skb_abort_seq_read(struct skb_seq_state *st)
 {
        if (st->frag_data)
-               kunmap_skb_frag(st->frag_data);
+               kunmap_atomic(st->frag_data);
 }
 EXPORT_SYMBOL(skb_abort_seq_read);
 
@@ -3164,7 +3169,7 @@ int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
        int len = skb->len;
 
        if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
-           (unsigned)sk->sk_rcvbuf)
+           (unsigned int)sk->sk_rcvbuf)
                return -ENOMEM;
 
        skb_orphan(skb);