Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/net-next

[pandora-kernel.git] / net / core / skbuff.c

diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 46cbd28..2beda82 100644 (file)
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -329,6 +329,18 @@ static void skb_release_data(struct sk_buff *skb)
                                put_page(skb_shinfo(skb)->frags[i].page);
                }
 
+               /*
+                * If skb buf is from userspace, we need to notify the caller
+                * the lower device DMA has done;
+                */
+               if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
+                       struct ubuf_info *uarg;
+
+                       uarg = skb_shinfo(skb)->destructor_arg;
+                       if (uarg->callback)
+                               uarg->callback(uarg);
+               }
+
                if (skb_has_frag_list(skb))
                        skb_drop_fraglist(skb);
 
@@ -481,6 +493,9 @@ bool skb_recycle_check(struct sk_buff *skb, int skb_size)
        if (irqs_disabled())
                return false;
 
+       if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY)
+               return false;
+
        if (skb_is_nonlinear(skb) || skb->fclone != SKB_FCLONE_UNAVAILABLE)
                return false;
 
@@ -596,6 +611,51 @@ struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
 }
 EXPORT_SYMBOL_GPL(skb_morph);
 
+/* skb frags copy userspace buffers to kernel */
+static int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
+{
+       int i;
+       int num_frags = skb_shinfo(skb)->nr_frags;
+       struct page *page, *head = NULL;
+       struct ubuf_info *uarg = skb_shinfo(skb)->destructor_arg;
+
+       for (i = 0; i < num_frags; i++) {
+               u8 *vaddr;
+               skb_frag_t *f = &skb_shinfo(skb)->frags[i];
+
+               page = alloc_page(GFP_ATOMIC);
+               if (!page) {
+                       while (head) {
+                               struct page *next = (struct page *)head->private;
+                               put_page(head);
+                               head = next;
+                       }
+                       return -ENOMEM;
+               }
+               vaddr = kmap_skb_frag(&skb_shinfo(skb)->frags[i]);
+               memcpy(page_address(page),
+                      vaddr + f->page_offset, f->size);
+               kunmap_skb_frag(vaddr);
+               page->private = (unsigned long)head;
+               head = page;
+       }
+
+       /* skb frags release userspace buffers */
+       for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
+               put_page(skb_shinfo(skb)->frags[i].page);
+
+       uarg->callback(uarg);
+
+       /* skb frags point to kernel buffers */
+       for (i = skb_shinfo(skb)->nr_frags; i > 0; i--) {
+               skb_shinfo(skb)->frags[i - 1].page_offset = 0;
+               skb_shinfo(skb)->frags[i - 1].page = head;
+               head = (struct page *)head->private;
+       }
+       return 0;
+}
+
+
 /**
  *     skb_clone       -       duplicate an sk_buff
  *     @skb: buffer to clone
@@ -614,6 +674,12 @@ struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
 {
        struct sk_buff *n;
 
+       if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
+               if (skb_copy_ubufs(skb, gfp_mask))
+                       return NULL;
+               skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY;
+       }
+
        n = skb + 1;
        if (skb->fclone == SKB_FCLONE_ORIG &&
            n->fclone == SKB_FCLONE_UNAVAILABLE) {
@@ -731,6 +797,14 @@ struct sk_buff *pskb_copy(struct sk_buff *skb, gfp_t gfp_mask)
        if (skb_shinfo(skb)->nr_frags) {
                int i;
 
+               if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
+                       if (skb_copy_ubufs(skb, gfp_mask)) {
+                               kfree_skb(n);
+                               n = NULL;
+                               goto out;
+                       }
+                       skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY;
+               }
                for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
                        skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
                        get_page(skb_shinfo(n)->frags[i].page);
@@ -788,7 +862,6 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
                fastpath = true;
        else {
                int delta = skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1;
-
                fastpath = atomic_read(&skb_shinfo(skb)->dataref) == delta;
        }
 
@@ -819,6 +892,12 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
        if (fastpath) {
                kfree(skb->head);
        } else {
+               /* copy this zero copy skb frags */
+               if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
+                       if (skb_copy_ubufs(skb, gfp_mask))
+                               goto nofrags;
+                       skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY;
+               }
                for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
                        get_page(skb_shinfo(skb)->frags[i].page);
 
@@ -853,6 +932,8 @@ adjust_others:
        atomic_set(&skb_shinfo(skb)->dataref, 1);
        return 0;
 
+nofrags:
+       kfree(data);
 nodata:
        return -ENOMEM;
 }
@@ -1354,6 +1435,7 @@ int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
                }
                start = end;
        }
+
        if (!len)
                return 0;