net/ipv4/ip_output.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              The Internet Protocol (IP) output module.
   7  *
   8  * Version:     $Id: ip_output.c,v 1.100 2002/02/01 22:01:03 davem Exp $
   9  *
  10  * Authors:     Ross Biro
  11  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  12  *              Donald Becker, <becker@super.org>
  13  *              Alan Cox, <Alan.Cox@linux.org>
  14  *              Richard Underwood
  15  *              Stefan Becker, <stefanb@yello.ping.de>
  16  *              Jorge Cwik, <jorge@laser.satlink.net>
  17  *              Arnt Gulbrandsen, <agulbra@nvg.unit.no>
  18  *              Hirokazu Takahashi, <taka@valinux.co.jp>
  19  *
  20  *      See ip_input.c for original log
  21  *
  22  *      Fixes:
  23  *              Alan Cox        :       Missing nonblock feature in ip_build_xmit.
  24  *              Mike Kilburn    :       htons() missing in ip_build_xmit.
  25  *              Bradford Johnson:       Fix faulty handling of some frames when
  26  *                                      no route is found.
  27  *              Alexander Demenshin:    Missing sk/skb free in ip_queue_xmit
  28  *                                      (in case if packet not accepted by
  29  *                                      output firewall rules)
  30  *              Mike McLagan    :       Routing by source
  31  *              Alexey Kuznetsov:       use new route cache
  32  *              Andi Kleen:             Fix broken PMTU recovery and remove
  33  *                                      some redundant tests.
  34  *      Vitaly E. Lavrov        :       Transparent proxy revived after year coma.
  35  *              Andi Kleen      :       Replace ip_reply with ip_send_reply.
  36  *              Andi Kleen      :       Split fast and slow ip_build_xmit path
  37  *                                      for decreased register pressure on x86
  38  *                                      and more readibility.
  39  *              Marc Boucher    :       When call_out_firewall returns FW_QUEUE,
  40  *                                      silently drop skb instead of failing with -EPERM.
  41  *              Detlev Wengorz  :       Copy protocol for fragments.
  42  *              Hirokazu Takahashi:     HW checksumming for outgoing UDP
  43  *                                      datagrams.
  44  *              Hirokazu Takahashi:     sendfile() on UDP works now.
  45  */
  46
  47 #include <asm/uaccess.h>
  48 #include <asm/system.h>
  49 #include <linux/module.h>
  50 #include <linux/types.h>
  51 #include <linux/kernel.h>
  52 #include <linux/sched.h>
  53 #include <linux/mm.h>
  54 #include <linux/string.h>
  55 #include <linux/errno.h>
  56
  57 #include <linux/socket.h>
  58 #include <linux/sockios.h>
  59 #include <linux/in.h>
  60 #include <linux/inet.h>
  61 #include <linux/netdevice.h>
  62 #include <linux/etherdevice.h>
  63 #include <linux/proc_fs.h>
  64 #include <linux/stat.h>
  65 #include <linux/init.h>
  66
  67 #include <net/snmp.h>
  68 #include <net/ip.h>
  69 #include <net/protocol.h>
  70 #include <net/route.h>
  71 #include <net/xfrm.h>
  72 #include <linux/skbuff.h>
  73 #include <net/sock.h>
  74 #include <net/arp.h>
  75 #include <net/icmp.h>
  76 #include <net/checksum.h>
  77 #include <net/inetpeer.h>
  78 #include <net/checksum.h>
  79 #include <linux/igmp.h>
  80 #include <linux/netfilter_ipv4.h>
  81 #include <linux/netfilter_bridge.h>
  82 #include <linux/mroute.h>
  83 #include <linux/netlink.h>
  84 #include <linux/tcp.h>
  85
  86 int sysctl_ip_default_ttl = IPDEFTTL;
  87
  88 /* Generate a checksum for an outgoing IP datagram. */
  89 __inline__ void ip_send_check(struct iphdr *iph)
  90 {
  91         iph->check = 0;
  92         iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
  93 }
  94
  95 /* dev_loopback_xmit for use with netfilter. */
  96 static int ip_dev_loopback_xmit(struct sk_buff *newskb)
  97 {
  98         newskb->mac.raw = newskb->data;
  99         __skb_pull(newskb, newskb->nh.raw - newskb->data);
 100         newskb->pkt_type = PACKET_LOOPBACK;
 101         newskb->ip_summed = CHECKSUM_UNNECESSARY;
 102         BUG_TRAP(newskb->dst);
 103         netif_rx(newskb);
 104         return 0;
 105 }
 106
 107 static inline int ip_select_ttl(struct inet_sock *inet, struct dst_entry *dst)
 108 {
 109         int ttl = inet->uc_ttl;
 110
 111         if (ttl < 0)
 112                 ttl = dst_metric(dst, RTAX_HOPLIMIT);
 113         return ttl;
 114 }
 115
 116 /*
 117  *              Add an ip header to a skbuff and send it out.
 118  *
 119  */
 120 int ip_build_and_send_pkt(struct sk_buff *skb, struct sock *sk,
 121                           u32 saddr, u32 daddr, struct ip_options *opt)
 122 {
 123         struct inet_sock *inet = inet_sk(sk);
 124         struct rtable *rt = (struct rtable *)skb->dst;
 125         struct iphdr *iph;
 126
 127         /* Build the IP header. */
 128         if (opt)
 129                 iph=(struct iphdr *)skb_push(skb,sizeof(struct iphdr) + opt->optlen);
 130         else
 131                 iph=(struct iphdr *)skb_push(skb,sizeof(struct iphdr));
 132
 133         iph->version  = 4;
 134         iph->ihl      = 5;
 135         iph->tos      = inet->tos;
 136         if (ip_dont_fragment(sk, &rt->u.dst))
 137                 iph->frag_off = htons(IP_DF);
 138         else
 139                 iph->frag_off = 0;
 140         iph->ttl      = ip_select_ttl(inet, &rt->u.dst);
 141         iph->daddr    = rt->rt_dst;
 142         iph->saddr    = rt->rt_src;
 143         iph->protocol = sk->sk_protocol;
 144         iph->tot_len  = htons(skb->len);
 145         ip_select_ident(iph, &rt->u.dst, sk);
 146         skb->nh.iph   = iph;
 147
 148         if (opt && opt->optlen) {
 149                 iph->ihl += opt->optlen>>2;
 150                 ip_options_build(skb, opt, daddr, rt, 0);
 151         }
 152         ip_send_check(iph);
 153
 154         skb->priority = sk->sk_priority;
 155
 156         /* Send it out. */
 157         return NF_HOOK(PF_INET, NF_IP_LOCAL_OUT, skb, NULL, rt->u.dst.dev,
 158                        dst_output);
 159 }
 160
 161 EXPORT_SYMBOL_GPL(ip_build_and_send_pkt);
 162
 163 static inline int ip_finish_output2(struct sk_buff *skb)
 164 {
 165         struct dst_entry *dst = skb->dst;
 166         struct hh_cache *hh = dst->hh;
 167         struct net_device *dev = dst->dev;
 168         int hh_len = LL_RESERVED_SPACE(dev);
 169
 170         /* Be paranoid, rather than too clever. */
 171         if (unlikely(skb_headroom(skb) < hh_len && dev->hard_header)) {
 172                 struct sk_buff *skb2;
 173
 174                 skb2 = skb_realloc_headroom(skb, LL_RESERVED_SPACE(dev));
 175                 if (skb2 == NULL) {
 176                         kfree_skb(skb);
 177                         return -ENOMEM;
 178                 }
 179                 if (skb->sk)
 180                         skb_set_owner_w(skb2, skb->sk);
 181                 kfree_skb(skb);
 182                 skb = skb2;
 183         }
 184
 185         if (hh) {
 186                 int hh_alen;
 187
 188                 read_lock_bh(&hh->hh_lock);
 189                 hh_alen = HH_DATA_ALIGN(hh->hh_len);
 190                 memcpy(skb->data - hh_alen, hh->hh_data, hh_alen);
 191                 read_unlock_bh(&hh->hh_lock);
 192                 skb_push(skb, hh->hh_len);
 193                 return hh->hh_output(skb);
 194         } else if (dst->neighbour)
 195                 return dst->neighbour->output(skb);
 196
 197         if (net_ratelimit())
 198                 printk(KERN_DEBUG "ip_finish_output2: No header cache and no neighbour!\n");
 199         kfree_skb(skb);
 200         return -EINVAL;
 201 }
 202
 203 static inline int ip_finish_output(struct sk_buff *skb)
 204 {
 205 #if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM)
 206         /* Policy lookup after SNAT yielded a new policy */
 207         if (skb->dst->xfrm != NULL) {
 208                 IPCB(skb)->flags |= IPSKB_REROUTED;
 209                 return dst_output(skb);
 210         }
 211 #endif
 212         if (skb->len > dst_mtu(skb->dst) && !skb_is_gso(skb))
 213                 return ip_fragment(skb, ip_finish_output2);
 214         else
 215                 return ip_finish_output2(skb);
 216 }
 217
 218 int ip_mc_output(struct sk_buff *skb)
 219 {
 220         struct sock *sk = skb->sk;
 221         struct rtable *rt = (struct rtable*)skb->dst;
 222         struct net_device *dev = rt->u.dst.dev;
 223
 224         /*
 225          *      If the indicated interface is up and running, send the packet.
 226          */
 227         IP_INC_STATS(IPSTATS_MIB_OUTREQUESTS);
 228
 229         skb->dev = dev;
 230         skb->protocol = htons(ETH_P_IP);
 231
 232         /*
 233          *      Multicasts are looped back for other local users
 234          */
 235
 236         if (rt->rt_flags&RTCF_MULTICAST) {
 237                 if ((!sk || inet_sk(sk)->mc_loop)
 238 #ifdef CONFIG_IP_MROUTE
 239                 /* Small optimization: do not loopback not local frames,
 240                    which returned after forwarding; they will be  dropped
 241                    by ip_mr_input in any case.
 242                    Note, that local frames are looped back to be delivered
 243                    to local recipients.
 244
 245                    This check is duplicated in ip_mr_input at the moment.
 246                  */
 247                     && ((rt->rt_flags&RTCF_LOCAL) || !(IPCB(skb)->flags&IPSKB_FORWARDED))
 248 #endif
 249                 ) {
 250                         struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
 251                         if (newskb)
 252                                 NF_HOOK(PF_INET, NF_IP_POST_ROUTING, newskb, NULL,
 253                                         newskb->dev,
 254                                         ip_dev_loopback_xmit);
 255                 }
 256
 257                 /* Multicasts with ttl 0 must not go beyond the host */
 258
 259                 if (skb->nh.iph->ttl == 0) {
 260                         kfree_skb(skb);
 261                         return 0;
 262                 }
 263         }
 264
 265         if (rt->rt_flags&RTCF_BROADCAST) {
 266                 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
 267                 if (newskb)
 268                         NF_HOOK(PF_INET, NF_IP_POST_ROUTING, newskb, NULL,
 269                                 newskb->dev, ip_dev_loopback_xmit);
 270         }
 271
 272         return NF_HOOK_COND(PF_INET, NF_IP_POST_ROUTING, skb, NULL, skb->dev,
 273                             ip_finish_output,
 274                             !(IPCB(skb)->flags & IPSKB_REROUTED));
 275 }
 276
 277 int ip_output(struct sk_buff *skb)
 278 {
 279         struct net_device *dev = skb->dst->dev;
 280
 281         IP_INC_STATS(IPSTATS_MIB_OUTREQUESTS);
 282
 283         skb->dev = dev;
 284         skb->protocol = htons(ETH_P_IP);
 285
 286         return NF_HOOK_COND(PF_INET, NF_IP_POST_ROUTING, skb, NULL, dev,
 287                             ip_finish_output,
 288                             !(IPCB(skb)->flags & IPSKB_REROUTED));
 289 }
 290
 291 int ip_queue_xmit(struct sk_buff *skb, int ipfragok)
 292 {
 293         struct sock *sk = skb->sk;
 294         struct inet_sock *inet = inet_sk(sk);
 295         struct ip_options *opt = inet->opt;
 296         struct rtable *rt;
 297         struct iphdr *iph;
 298
 299         /* Skip all of this if the packet is already routed,
 300          * f.e. by something like SCTP.
 301          */
 302         rt = (struct rtable *) skb->dst;
 303         if (rt != NULL)
 304                 goto packet_routed;
 305
 306         /* Make sure we can route this packet. */
 307         rt = (struct rtable *)__sk_dst_check(sk, 0);
 308         if (rt == NULL) {
 309                 u32 daddr;
 310
 311                 /* Use correct destination address if we have options. */
 312                 daddr = inet->daddr;
 313                 if(opt && opt->srr)
 314                         daddr = opt->faddr;
 315
 316                 {
 317                         struct flowi fl = { .oif = sk->sk_bound_dev_if,
 318                                             .nl_u = { .ip4_u =
 319                                                       { .daddr = daddr,
 320                                                         .saddr = inet->saddr,
 321                                                         .tos = RT_CONN_FLAGS(sk) } },
 322                                             .proto = sk->sk_protocol,
 323                                             .uli_u = { .ports =
 324                                                        { .sport = inet->sport,
 325                                                          .dport = inet->dport } } };
 326
 327                         /* If this fails, retransmit mechanism of transport layer will
 328                          * keep trying until route appears or the connection times
 329                          * itself out.
 330                          */
 331                         if (ip_route_output_flow(&rt, &fl, sk, 0))
 332                                 goto no_route;
 333                 }
 334                 sk_setup_caps(sk, &rt->u.dst);
 335         }
 336         skb->dst = dst_clone(&rt->u.dst);
 337
 338 packet_routed:
 339         if (opt && opt->is_strictroute && rt->rt_dst != rt->rt_gateway)
 340                 goto no_route;
 341
 342         /* OK, we know where to send it, allocate and build IP header. */
 343         iph = (struct iphdr *) skb_push(skb, sizeof(struct iphdr) + (opt ? opt->optlen : 0));
 344         *((__u16 *)iph) = htons((4 << 12) | (5 << 8) | (inet->tos & 0xff));
 345         iph->tot_len = htons(skb->len);
 346         if (ip_dont_fragment(sk, &rt->u.dst) && !ipfragok)
 347                 iph->frag_off = htons(IP_DF);
 348         else
 349                 iph->frag_off = 0;
 350         iph->ttl      = ip_select_ttl(inet, &rt->u.dst);
 351         iph->protocol = sk->sk_protocol;
 352         iph->saddr    = rt->rt_src;
 353         iph->daddr    = rt->rt_dst;
 354         skb->nh.iph   = iph;
 355         /* Transport layer set skb->h.foo itself. */
 356
 357         if (opt && opt->optlen) {
 358                 iph->ihl += opt->optlen >> 2;
 359                 ip_options_build(skb, opt, inet->daddr, rt, 0);
 360         }
 361
 362         ip_select_ident_more(iph, &rt->u.dst, sk,
 363                              (skb_shinfo(skb)->gso_segs ?: 1) - 1);
 364
 365         /* Add an IP checksum. */
 366         ip_send_check(iph);
 367
 368         skb->priority = sk->sk_priority;
 369
 370         return NF_HOOK(PF_INET, NF_IP_LOCAL_OUT, skb, NULL, rt->u.dst.dev,
 371                        dst_output);
 372
 373 no_route:
 374         IP_INC_STATS(IPSTATS_MIB_OUTNOROUTES);
 375         kfree_skb(skb);
 376         return -EHOSTUNREACH;
 377 }
 378
 379
 380 static void ip_copy_metadata(struct sk_buff *to, struct sk_buff *from)
 381 {
 382         to->pkt_type = from->pkt_type;
 383         to->priority = from->priority;
 384         to->protocol = from->protocol;
 385         dst_release(to->dst);
 386         to->dst = dst_clone(from->dst);
 387         to->dev = from->dev;
 388
 389         /* Copy the flags to each fragment. */
 390         IPCB(to)->flags = IPCB(from)->flags;
 391
 392 #ifdef CONFIG_NET_SCHED
 393         to->tc_index = from->tc_index;
 394 #endif
 395 #ifdef CONFIG_NETFILTER
 396         to->nfmark = from->nfmark;
 397         /* Connection association is same as pre-frag packet */
 398         nf_conntrack_put(to->nfct);
 399         to->nfct = from->nfct;
 400         nf_conntrack_get(to->nfct);
 401         to->nfctinfo = from->nfctinfo;
 402 #if defined(CONFIG_IP_VS) || defined(CONFIG_IP_VS_MODULE)
 403         to->ipvs_property = from->ipvs_property;
 404 #endif
 405 #ifdef CONFIG_BRIDGE_NETFILTER
 406         nf_bridge_put(to->nf_bridge);
 407         to->nf_bridge = from->nf_bridge;
 408         nf_bridge_get(to->nf_bridge);
 409 #endif
 410 #endif
 411         skb_copy_secmark(to, from);
 412 }
 413
 414 /*
 415  *      This IP datagram is too large to be sent in one piece.  Break it up into
 416  *      smaller pieces (each of size equal to IP header plus
 417  *      a block of the data of the original IP data part) that will yet fit in a
 418  *      single device frame, and queue such a frame for sending.
 419  */
 420
 421 int ip_fragment(struct sk_buff *skb, int (*output)(struct sk_buff*))
 422 {
 423         struct iphdr *iph;
 424         int raw = 0;
 425         int ptr;
 426         struct net_device *dev;
 427         struct sk_buff *skb2;
 428         unsigned int mtu, hlen, left, len, ll_rs;
 429         int offset;
 430         __be16 not_last_frag;
 431         struct rtable *rt = (struct rtable*)skb->dst;
 432         int err = 0;
 433
 434         dev = rt->u.dst.dev;
 435
 436         /*
 437          *      Point into the IP datagram header.
 438          */
 439
 440         iph = skb->nh.iph;
 441
 442         if (unlikely((iph->frag_off & htons(IP_DF)) && !skb->local_df)) {
 443                 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED,
 444                           htonl(dst_mtu(&rt->u.dst)));
 445                 kfree_skb(skb);
 446                 return -EMSGSIZE;
 447         }
 448
 449         /*
 450          *      Setup starting values.
 451          */
 452
 453         hlen = iph->ihl * 4;
 454         mtu = dst_mtu(&rt->u.dst) - hlen;       /* Size of data space */
 455         IPCB(skb)->flags |= IPSKB_FRAG_COMPLETE;
 456
 457         /* When frag_list is given, use it. First, check its validity:
 458          * some transformers could create wrong frag_list or break existing
 459          * one, it is not prohibited. In this case fall back to copying.
 460          *
 461          * LATER: this step can be merged to real generation of fragments,
 462          * we can switch to copy when see the first bad fragment.
 463          */
 464         if (skb_shinfo(skb)->frag_list) {
 465                 struct sk_buff *frag;
 466                 int first_len = skb_pagelen(skb);
 467
 468                 if (first_len - hlen > mtu ||
 469                     ((first_len - hlen) & 7) ||
 470                     (iph->frag_off & htons(IP_MF|IP_OFFSET)) ||
 471                     skb_cloned(skb))
 472                         goto slow_path;
 473
 474                 for (frag = skb_shinfo(skb)->frag_list; frag; frag = frag->next) {
 475                         /* Correct geometry. */
 476                         if (frag->len > mtu ||
 477                             ((frag->len & 7) && frag->next) ||
 478                             skb_headroom(frag) < hlen)
 479                             goto slow_path;
 480
 481                         /* Partially cloned skb? */
 482                         if (skb_shared(frag))
 483                                 goto slow_path;
 484
 485                         BUG_ON(frag->sk);
 486                         if (skb->sk) {
 487                                 sock_hold(skb->sk);
 488                                 frag->sk = skb->sk;
 489                                 frag->destructor = sock_wfree;
 490                                 skb->truesize -= frag->truesize;
 491                         }
 492                 }
 493
 494                 /* Everything is OK. Generate! */
 495
 496                 err = 0;
 497                 offset = 0;
 498                 frag = skb_shinfo(skb)->frag_list;
 499                 skb_shinfo(skb)->frag_list = NULL;
 500                 skb->data_len = first_len - skb_headlen(skb);
 501                 skb->len = first_len;
 502                 iph->tot_len = htons(first_len);
 503                 iph->frag_off = htons(IP_MF);
 504                 ip_send_check(iph);
 505
 506                 for (;;) {
 507                         /* Prepare header of the next frame,
 508                          * before previous one went down. */
 509                         if (frag) {
 510                                 frag->ip_summed = CHECKSUM_NONE;
 511                                 frag->h.raw = frag->data;
 512                                 frag->nh.raw = __skb_push(frag, hlen);
 513                                 memcpy(frag->nh.raw, iph, hlen);
 514                                 iph = frag->nh.iph;
 515                                 iph->tot_len = htons(frag->len);
 516                                 ip_copy_metadata(frag, skb);
 517                                 if (offset == 0)
 518                                         ip_options_fragment(frag);
 519                                 offset += skb->len - hlen;
 520                                 iph->frag_off = htons(offset>>3);
 521                                 if (frag->next != NULL)
 522                                         iph->frag_off |= htons(IP_MF);
 523                                 /* Ready, complete checksum */
 524                                 ip_send_check(iph);
 525                         }
 526
 527                         err = output(skb);
 528
 529                         if (err || !frag)
 530                                 break;
 531
 532                         skb = frag;
 533                         frag = skb->next;
 534                         skb->next = NULL;
 535                 }
 536
 537                 if (err == 0) {
 538                         IP_INC_STATS(IPSTATS_MIB_FRAGOKS);
 539                         return 0;
 540                 }
 541
 542                 while (frag) {
 543                         skb = frag->next;
 544                         kfree_skb(frag);
 545                         frag = skb;
 546                 }
 547                 IP_INC_STATS(IPSTATS_MIB_FRAGFAILS);
 548                 return err;
 549         }
 550
 551 slow_path:
 552         left = skb->len - hlen;         /* Space per frame */
 553         ptr = raw + hlen;               /* Where to start from */
 554
 555 #ifdef CONFIG_BRIDGE_NETFILTER
 556         /* for bridged IP traffic encapsulated inside f.e. a vlan header,
 557          * we need to make room for the encapsulating header */
 558         ll_rs = LL_RESERVED_SPACE_EXTRA(rt->u.dst.dev, nf_bridge_pad(skb));
 559         mtu -= nf_bridge_pad(skb);
 560 #else
 561         ll_rs = LL_RESERVED_SPACE(rt->u.dst.dev);
 562 #endif
 563         /*
 564          *      Fragment the datagram.
 565          */
 566
 567         offset = (ntohs(iph->frag_off) & IP_OFFSET) << 3;
 568         not_last_frag = iph->frag_off & htons(IP_MF);
 569
 570         /*
 571          *      Keep copying data until we run out.
 572          */
 573
 574         while(left > 0) {
 575                 len = left;
 576                 /* IF: it doesn't fit, use 'mtu' - the data space left */
 577                 if (len > mtu)
 578                         len = mtu;
 579                 /* IF: we are not sending upto and including the packet end
 580                    then align the next start on an eight byte boundary */
 581                 if (len < left) {
 582                         len &= ~7;
 583                 }
 584                 /*
 585                  *      Allocate buffer.
 586                  */
 587
 588                 if ((skb2 = alloc_skb(len+hlen+ll_rs, GFP_ATOMIC)) == NULL) {
 589                         NETDEBUG(KERN_INFO "IP: frag: no memory for new fragment!\n");
 590                         err = -ENOMEM;
 591                         goto fail;
 592                 }
 593
 594                 /*
 595                  *      Set up data on packet
 596                  */
 597
 598                 ip_copy_metadata(skb2, skb);
 599                 skb_reserve(skb2, ll_rs);
 600                 skb_put(skb2, len + hlen);
 601                 skb2->nh.raw = skb2->data;
 602                 skb2->h.raw = skb2->data + hlen;
 603
 604                 /*
 605                  *      Charge the memory for the fragment to any owner
 606                  *      it might possess
 607                  */
 608
 609                 if (skb->sk)
 610                         skb_set_owner_w(skb2, skb->sk);
 611
 612                 /*
 613                  *      Copy the packet header into the new buffer.
 614                  */
 615
 616                 memcpy(skb2->nh.raw, skb->data, hlen);
 617
 618                 /*
 619                  *      Copy a block of the IP datagram.
 620                  */
 621                 if (skb_copy_bits(skb, ptr, skb2->h.raw, len))
 622                         BUG();
 623                 left -= len;
 624
 625                 /*
 626                  *      Fill in the new header fields.
 627                  */
 628                 iph = skb2->nh.iph;
 629                 iph->frag_off = htons((offset >> 3));
 630
 631                 /* ANK: dirty, but effective trick. Upgrade options only if
 632                  * the segment to be fragmented was THE FIRST (otherwise,
 633                  * options are already fixed) and make it ONCE
 634                  * on the initial skb, so that all the following fragments
 635                  * will inherit fixed options.
 636                  */
 637                 if (offset == 0)
 638                         ip_options_fragment(skb);
 639
 640                 /*
 641                  *      Added AC : If we are fragmenting a fragment that's not the
 642                  *                 last fragment then keep MF on each bit
 643                  */
 644                 if (left > 0 || not_last_frag)
 645                         iph->frag_off |= htons(IP_MF);
 646                 ptr += len;
 647                 offset += len;
 648
 649                 /*
 650                  *      Put this fragment into the sending queue.
 651                  */
 652
 653                 IP_INC_STATS(IPSTATS_MIB_FRAGCREATES);
 654
 655                 iph->tot_len = htons(len + hlen);
 656
 657                 ip_send_check(iph);
 658
 659                 err = output(skb2);
 660                 if (err)
 661                         goto fail;
 662         }
 663         kfree_skb(skb);
 664         IP_INC_STATS(IPSTATS_MIB_FRAGOKS);
 665         return err;
 666
 667 fail:
 668         kfree_skb(skb);
 669         IP_INC_STATS(IPSTATS_MIB_FRAGFAILS);
 670         return err;
 671 }
 672
 673 EXPORT_SYMBOL(ip_fragment);
 674
 675 int
 676 ip_generic_getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb)
 677 {
 678         struct iovec *iov = from;
 679
 680         if (skb->ip_summed == CHECKSUM_HW) {
 681                 if (memcpy_fromiovecend(to, iov, offset, len) < 0)
 682                         return -EFAULT;
 683         } else {
 684                 unsigned int csum = 0;
 685                 if (csum_partial_copy_fromiovecend(to, iov, offset, len, &csum) < 0)
 686                         return -EFAULT;
 687                 skb->csum = csum_block_add(skb->csum, csum, odd);
 688         }
 689         return 0;
 690 }
 691
 692 static inline unsigned int
 693 csum_page(struct page *page, int offset, int copy)
 694 {
 695         char *kaddr;
 696         unsigned int csum;
 697         kaddr = kmap(page);
 698         csum = csum_partial(kaddr + offset, copy, 0);
 699         kunmap(page);
 700         return csum;
 701 }
 702
 703 static inline int ip_ufo_append_data(struct sock *sk,
 704                         int getfrag(void *from, char *to, int offset, int len,
 705                                int odd, struct sk_buff *skb),
 706                         void *from, int length, int hh_len, int fragheaderlen,
 707                         int transhdrlen, int mtu,unsigned int flags)
 708 {
 709         struct sk_buff *skb;
 710         int err;
 711
 712         /* There is support for UDP fragmentation offload by network
 713          * device, so create one single skb packet containing complete
 714          * udp datagram
 715          */
 716         if ((skb = skb_peek_tail(&sk->sk_write_queue)) == NULL) {
 717                 skb = sock_alloc_send_skb(sk,
 718                         hh_len + fragheaderlen + transhdrlen + 20,
 719                         (flags & MSG_DONTWAIT), &err);
 720
 721                 if (skb == NULL)
 722                         return err;
 723
 724                 /* reserve space for Hardware header */
 725                 skb_reserve(skb, hh_len);
 726
 727                 /* create space for UDP/IP header */
 728                 skb_put(skb,fragheaderlen + transhdrlen);
 729
 730                 /* initialize network header pointer */
 731                 skb->nh.raw = skb->data;
 732
 733                 /* initialize protocol header pointer */
 734                 skb->h.raw = skb->data + fragheaderlen;
 735
 736                 skb->ip_summed = CHECKSUM_HW;
 737                 skb->csum = 0;
 738                 sk->sk_sndmsg_off = 0;
 739         }
 740
 741         err = skb_append_datato_frags(sk,skb, getfrag, from,
 742                                (length - transhdrlen));
 743         if (!err) {
 744                 /* specify the length of each IP datagram fragment*/
 745                 skb_shinfo(skb)->gso_size = mtu - fragheaderlen;
 746                 skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
 747                 __skb_queue_tail(&sk->sk_write_queue, skb);
 748
 749                 return 0;
 750         }
 751         /* There is not enough support do UFO ,
 752          * so follow normal path
 753          */
 754         kfree_skb(skb);
 755         return err;
 756 }
 757
 758 /*
 759  *      ip_append_data() and ip_append_page() can make one large IP datagram
 760  *      from many pieces of data. Each pieces will be holded on the socket
 761  *      until ip_push_pending_frames() is called. Each piece can be a page
 762  *      or non-page data.
 763  *
 764  *      Not only UDP, other transport protocols - e.g. raw sockets - can use
 765  *      this interface potentially.
 766  *
 767  *      LATER: length must be adjusted by pad at tail, when it is required.
 768  */
 769 int ip_append_data(struct sock *sk,
 770                    int getfrag(void *from, char *to, int offset, int len,
 771                                int odd, struct sk_buff *skb),
 772                    void *from, int length, int transhdrlen,
 773                    struct ipcm_cookie *ipc, struct rtable *rt,
 774                    unsigned int flags)
 775 {
 776         struct inet_sock *inet = inet_sk(sk);
 777         struct sk_buff *skb;
 778
 779         struct ip_options *opt = NULL;
 780         int hh_len;
 781         int exthdrlen;
 782         int mtu;
 783         int copy;
 784         int err;
 785         int offset = 0;
 786         unsigned int maxfraglen, fragheaderlen;
 787         int csummode = CHECKSUM_NONE;
 788
 789         if (flags&MSG_PROBE)
 790                 return 0;
 791
 792         if (skb_queue_empty(&sk->sk_write_queue)) {
 793                 /*
 794                  * setup for corking.
 795                  */
 796                 opt = ipc->opt;
 797                 if (opt) {
 798                         if (inet->cork.opt == NULL) {
 799                                 inet->cork.opt = kmalloc(sizeof(struct ip_options) + 40, sk->sk_allocation);
 800                                 if (unlikely(inet->cork.opt == NULL))
 801                                         return -ENOBUFS;
 802                         }
 803                         memcpy(inet->cork.opt, opt, sizeof(struct ip_options)+opt->optlen);
 804                         inet->cork.flags |= IPCORK_OPT;
 805                         inet->cork.addr = ipc->addr;
 806                 }
 807                 dst_hold(&rt->u.dst);
 808                 inet->cork.fragsize = mtu = dst_mtu(rt->u.dst.path);
 809                 inet->cork.rt = rt;
 810                 inet->cork.length = 0;
 811                 sk->sk_sndmsg_page = NULL;
 812                 sk->sk_sndmsg_off = 0;
 813                 if ((exthdrlen = rt->u.dst.header_len) != 0) {
 814                         length += exthdrlen;
 815                         transhdrlen += exthdrlen;
 816                 }
 817         } else {
 818                 rt = inet->cork.rt;
 819                 if (inet->cork.flags & IPCORK_OPT)
 820                         opt = inet->cork.opt;
 821
 822                 transhdrlen = 0;
 823                 exthdrlen = 0;
 824                 mtu = inet->cork.fragsize;
 825         }
 826         hh_len = LL_RESERVED_SPACE(rt->u.dst.dev);
 827
 828         fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
 829         maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen;
 830
 831         if (inet->cork.length + length > 0xFFFF - fragheaderlen) {
 832                 ip_local_error(sk, EMSGSIZE, rt->rt_dst, inet->dport, mtu-exthdrlen);
 833                 return -EMSGSIZE;
 834         }
 835
 836         /*
 837          * transhdrlen > 0 means that this is the first fragment and we wish
 838          * it won't be fragmented in the future.
 839          */
 840         if (transhdrlen &&
 841             length + fragheaderlen <= mtu &&
 842             rt->u.dst.dev->features & NETIF_F_ALL_CSUM &&
 843             !exthdrlen)
 844                 csummode = CHECKSUM_HW;
 845
 846         inet->cork.length += length;
 847         if (((length > mtu) && (sk->sk_protocol == IPPROTO_UDP)) &&
 848                         (rt->u.dst.dev->features & NETIF_F_UFO)) {
 849
 850                 err = ip_ufo_append_data(sk, getfrag, from, length, hh_len,
 851                                          fragheaderlen, transhdrlen, mtu,
 852                                          flags);
 853                 if (err)
 854                         goto error;
 855                 return 0;
 856         }
 857
 858         /* So, what's going on in the loop below?
 859          *
 860          * We use calculated fragment length to generate chained skb,
 861          * each of segments is IP fragment ready for sending to network after
 862          * adding appropriate IP header.
 863          */
 864
 865         if ((skb = skb_peek_tail(&sk->sk_write_queue)) == NULL)
 866                 goto alloc_new_skb;
 867
 868         while (length > 0) {
 869                 /* Check if the remaining data fits into current packet. */
 870                 copy = mtu - skb->len;
 871                 if (copy < length)
 872                         copy = maxfraglen - skb->len;
 873                 if (copy <= 0) {
 874                         char *data;
 875                         unsigned int datalen;
 876                         unsigned int fraglen;
 877                         unsigned int fraggap;
 878                         unsigned int alloclen;
 879                         struct sk_buff *skb_prev;
 880 alloc_new_skb:
 881                         skb_prev = skb;
 882                         if (skb_prev)
 883                                 fraggap = skb_prev->len - maxfraglen;
 884                         else
 885                                 fraggap = 0;
 886
 887                         /*
 888                          * If remaining data exceeds the mtu,
 889                          * we know we need more fragment(s).
 890                          */
 891                         datalen = length + fraggap;
 892                         if (datalen > mtu - fragheaderlen)
 893                                 datalen = maxfraglen - fragheaderlen;
 894                         fraglen = datalen + fragheaderlen;
 895
 896                         if ((flags & MSG_MORE) &&
 897                             !(rt->u.dst.dev->features&NETIF_F_SG))
 898                                 alloclen = mtu;
 899                         else
 900                                 alloclen = datalen + fragheaderlen;
 901
 902                         /* The last fragment gets additional space at tail.
 903                          * Note, with MSG_MORE we overallocate on fragments,
 904                          * because we have no idea what fragment will be
 905                          * the last.
 906                          */
 907                         if (datalen == length + fraggap)
 908                                 alloclen += rt->u.dst.trailer_len;
 909
 910                         if (transhdrlen) {
 911                                 skb = sock_alloc_send_skb(sk,
 912                                                 alloclen + hh_len + 15,
 913                                                 (flags & MSG_DONTWAIT), &err);
 914                         } else {
 915                                 skb = NULL;
 916                                 if (atomic_read(&sk->sk_wmem_alloc) <=
 917                                     2 * sk->sk_sndbuf)
 918                                         skb = sock_wmalloc(sk,
 919                                                            alloclen + hh_len + 15, 1,
 920                                                            sk->sk_allocation);
 921                                 if (unlikely(skb == NULL))
 922                                         err = -ENOBUFS;
 923                         }
 924                         if (skb == NULL)
 925                                 goto error;
 926
 927                         /*
 928                          *      Fill in the control structures
 929                          */
 930                         skb->ip_summed = csummode;
 931                         skb->csum = 0;
 932                         skb_reserve(skb, hh_len);
 933
 934                         /*
 935                          *      Find where to start putting bytes.
 936                          */
 937                         data = skb_put(skb, fraglen);
 938                         skb->nh.raw = data + exthdrlen;
 939                         data += fragheaderlen;
 940                         skb->h.raw = data + exthdrlen;
 941
 942                         if (fraggap) {
 943                                 skb->csum = skb_copy_and_csum_bits(
 944                                         skb_prev, maxfraglen,
 945                                         data + transhdrlen, fraggap, 0);
 946                                 skb_prev->csum = csum_sub(skb_prev->csum,
 947                                                           skb->csum);
 948                                 data += fraggap;
 949                                 skb_trim(skb_prev, maxfraglen);
 950                         }
 951
 952                         copy = datalen - transhdrlen - fraggap;
 953                         if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) {
 954                                 err = -EFAULT;
 955                                 kfree_skb(skb);
 956                                 goto error;
 957                         }
 958
 959                         offset += copy;
 960                         length -= datalen - fraggap;
 961                         transhdrlen = 0;
 962                         exthdrlen = 0;
 963                         csummode = CHECKSUM_NONE;
 964
 965                         /*
 966                          * Put the packet on the pending queue.
 967                          */
 968                         __skb_queue_tail(&sk->sk_write_queue, skb);
 969                         continue;
 970                 }
 971
 972                 if (copy > length)
 973                         copy = length;
 974
 975                 if (!(rt->u.dst.dev->features&NETIF_F_SG)) {
 976                         unsigned int off;
 977
 978                         off = skb->len;
 979                         if (getfrag(from, skb_put(skb, copy),
 980                                         offset, copy, off, skb) < 0) {
 981                                 __skb_trim(skb, off);
 982                                 err = -EFAULT;
 983                                 goto error;
 984                         }
 985                 } else {
 986                         int i = skb_shinfo(skb)->nr_frags;
 987                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i-1];
 988                         struct page *page = sk->sk_sndmsg_page;
 989                         int off = sk->sk_sndmsg_off;
 990                         unsigned int left;
 991
 992                         if (page && (left = PAGE_SIZE - off) > 0) {
 993                                 if (copy >= left)
 994                                         copy = left;
 995                                 if (page != frag->page) {
 996                                         if (i == MAX_SKB_FRAGS) {
 997                                                 err = -EMSGSIZE;
 998                                                 goto error;
 999                                         }
1000                                         get_page(page);
1001                                         skb_fill_page_desc(skb, i, page, sk->sk_sndmsg_off, 0);
1002                                         frag = &skb_shinfo(skb)->frags[i];
1003                                 }
1004                         } else if (i < MAX_SKB_FRAGS) {
1005                                 if (copy > PAGE_SIZE)
1006                                         copy = PAGE_SIZE;
1007                                 page = alloc_pages(sk->sk_allocation, 0);
1008                                 if (page == NULL)  {
1009                                         err = -ENOMEM;
1010                                         goto error;
1011                                 }
1012                                 sk->sk_sndmsg_page = page;
1013                                 sk->sk_sndmsg_off = 0;
1014
1015                                 skb_fill_page_desc(skb, i, page, 0, 0);
1016                                 frag = &skb_shinfo(skb)->frags[i];
1017                                 skb->truesize += PAGE_SIZE;
1018                                 atomic_add(PAGE_SIZE, &sk->sk_wmem_alloc);
1019                         } else {
1020                                 err = -EMSGSIZE;
1021                                 goto error;
1022                         }
1023                         if (getfrag(from, page_address(frag->page)+frag->page_offset+frag->size, offset, copy, skb->len, skb) < 0) {
1024                                 err = -EFAULT;
1025                                 goto error;
1026                         }
1027                         sk->sk_sndmsg_off += copy;
1028                         frag->size += copy;
1029                         skb->len += copy;
1030                         skb->data_len += copy;
1031                 }
1032                 offset += copy;
1033                 length -= copy;
1034         }
1035
1036         return 0;
1037
1038 error:
1039         inet->cork.length -= length;
1040         IP_INC_STATS(IPSTATS_MIB_OUTDISCARDS);
1041         return err;
1042 }
1043
1044 ssize_t ip_append_page(struct sock *sk, struct page *page,
1045                        int offset, size_t size, int flags)
1046 {
1047         struct inet_sock *inet = inet_sk(sk);
1048         struct sk_buff *skb;
1049         struct rtable *rt;
1050         struct ip_options *opt = NULL;
1051         int hh_len;
1052         int mtu;
1053         int len;
1054         int err;
1055         unsigned int maxfraglen, fragheaderlen, fraggap;
1056
1057         if (inet->hdrincl)
1058                 return -EPERM;
1059
1060         if (flags&MSG_PROBE)
1061                 return 0;
1062
1063         if (skb_queue_empty(&sk->sk_write_queue))
1064                 return -EINVAL;
1065
1066         rt = inet->cork.rt;
1067         if (inet->cork.flags & IPCORK_OPT)
1068                 opt = inet->cork.opt;
1069
1070         if (!(rt->u.dst.dev->features&NETIF_F_SG))
1071                 return -EOPNOTSUPP;
1072
1073         hh_len = LL_RESERVED_SPACE(rt->u.dst.dev);
1074         mtu = inet->cork.fragsize;
1075
1076         fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
1077         maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen;
1078
1079         if (inet->cork.length + size > 0xFFFF - fragheaderlen) {
1080                 ip_local_error(sk, EMSGSIZE, rt->rt_dst, inet->dport, mtu);
1081                 return -EMSGSIZE;
1082         }
1083
1084         if ((skb = skb_peek_tail(&sk->sk_write_queue)) == NULL)
1085                 return -EINVAL;
1086
1087         inet->cork.length += size;
1088         if ((sk->sk_protocol == IPPROTO_UDP) &&
1089             (rt->u.dst.dev->features & NETIF_F_UFO)) {
1090                 skb_shinfo(skb)->gso_size = mtu - fragheaderlen;
1091                 skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
1092         }
1093
1094
1095         while (size > 0) {
1096                 int i;
1097
1098                 if (skb_is_gso(skb))
1099                         len = size;
1100                 else {
1101
1102                         /* Check if the remaining data fits into current packet. */
1103                         len = mtu - skb->len;
1104                         if (len < size)
1105                                 len = maxfraglen - skb->len;
1106                 }
1107                 if (len <= 0) {
1108                         struct sk_buff *skb_prev;
1109                         char *data;
1110                         struct iphdr *iph;
1111                         int alloclen;
1112
1113                         skb_prev = skb;
1114                         fraggap = skb_prev->len - maxfraglen;
1115
1116                         alloclen = fragheaderlen + hh_len + fraggap + 15;
1117                         skb = sock_wmalloc(sk, alloclen, 1, sk->sk_allocation);
1118                         if (unlikely(!skb)) {
1119                                 err = -ENOBUFS;
1120                                 goto error;
1121                         }
1122
1123                         /*
1124                          *      Fill in the control structures
1125                          */
1126                         skb->ip_summed = CHECKSUM_NONE;
1127                         skb->csum = 0;
1128                         skb_reserve(skb, hh_len);
1129
1130                         /*
1131                          *      Find where to start putting bytes.
1132                          */
1133                         data = skb_put(skb, fragheaderlen + fraggap);
1134                         skb->nh.iph = iph = (struct iphdr *)data;
1135                         data += fragheaderlen;
1136                         skb->h.raw = data;
1137
1138                         if (fraggap) {
1139                                 skb->csum = skb_copy_and_csum_bits(
1140                                         skb_prev, maxfraglen,
1141                                         data, fraggap, 0);
1142                                 skb_prev->csum = csum_sub(skb_prev->csum,
1143                                                           skb->csum);
1144                                 skb_trim(skb_prev, maxfraglen);
1145                         }
1146
1147                         /*
1148                          * Put the packet on the pending queue.
1149                          */
1150                         __skb_queue_tail(&sk->sk_write_queue, skb);
1151                         continue;
1152                 }
1153
1154                 i = skb_shinfo(skb)->nr_frags;
1155                 if (len > size)
1156                         len = size;
1157                 if (skb_can_coalesce(skb, i, page, offset)) {
1158                         skb_shinfo(skb)->frags[i-1].size += len;
1159                 } else if (i < MAX_SKB_FRAGS) {
1160                         get_page(page);
1161                         skb_fill_page_desc(skb, i, page, offset, len);
1162                 } else {
1163                         err = -EMSGSIZE;
1164                         goto error;
1165                 }
1166
1167                 if (skb->ip_summed == CHECKSUM_NONE) {
1168                         unsigned int csum;
1169                         csum = csum_page(page, offset, len);
1170                         skb->csum = csum_block_add(skb->csum, csum, skb->len);
1171                 }
1172
1173                 skb->len += len;
1174                 skb->data_len += len;
1175                 offset += len;
1176                 size -= len;
1177         }
1178         return 0;
1179
1180 error:
1181         inet->cork.length -= size;
1182         IP_INC_STATS(IPSTATS_MIB_OUTDISCARDS);
1183         return err;
1184 }
1185
1186 /*
1187  *      Combined all pending IP fragments on the socket as one IP datagram
1188  *      and push them out.
1189  */
1190 int ip_push_pending_frames(struct sock *sk)
1191 {
1192         struct sk_buff *skb, *tmp_skb;
1193         struct sk_buff **tail_skb;
1194         struct inet_sock *inet = inet_sk(sk);
1195         struct ip_options *opt = NULL;
1196         struct rtable *rt = inet->cork.rt;
1197         struct iphdr *iph;
1198         __be16 df = 0;
1199         __u8 ttl;
1200         int err = 0;
1201
1202         if ((skb = __skb_dequeue(&sk->sk_write_queue)) == NULL)
1203                 goto out;
1204         tail_skb = &(skb_shinfo(skb)->frag_list);
1205
1206         /* move skb->data to ip header from ext header */
1207         if (skb->data < skb->nh.raw)
1208                 __skb_pull(skb, skb->nh.raw - skb->data);
1209         while ((tmp_skb = __skb_dequeue(&sk->sk_write_queue)) != NULL) {
1210                 __skb_pull(tmp_skb, skb->h.raw - skb->nh.raw);
1211                 *tail_skb = tmp_skb;
1212                 tail_skb = &(tmp_skb->next);
1213                 skb->len += tmp_skb->len;
1214                 skb->data_len += tmp_skb->len;
1215                 skb->truesize += tmp_skb->truesize;
1216                 __sock_put(tmp_skb->sk);
1217                 tmp_skb->destructor = NULL;
1218                 tmp_skb->sk = NULL;
1219         }
1220
1221         /* Unless user demanded real pmtu discovery (IP_PMTUDISC_DO), we allow
1222          * to fragment the frame generated here. No matter, what transforms
1223          * how transforms change size of the packet, it will come out.
1224          */
1225         if (inet->pmtudisc != IP_PMTUDISC_DO)
1226                 skb->local_df = 1;
1227
1228         /* DF bit is set when we want to see DF on outgoing frames.
1229          * If local_df is set too, we still allow to fragment this frame
1230          * locally. */
1231         if (inet->pmtudisc == IP_PMTUDISC_DO ||
1232             (skb->len <= dst_mtu(&rt->u.dst) &&
1233              ip_dont_fragment(sk, &rt->u.dst)))
1234                 df = htons(IP_DF);
1235
1236         if (inet->cork.flags & IPCORK_OPT)
1237                 opt = inet->cork.opt;
1238
1239         if (rt->rt_type == RTN_MULTICAST)
1240                 ttl = inet->mc_ttl;
1241         else
1242                 ttl = ip_select_ttl(inet, &rt->u.dst);
1243
1244         iph = (struct iphdr *)skb->data;
1245         iph->version = 4;
1246         iph->ihl = 5;
1247         if (opt) {
1248                 iph->ihl += opt->optlen>>2;
1249                 ip_options_build(skb, opt, inet->cork.addr, rt, 0);
1250         }
1251         iph->tos = inet->tos;
1252         iph->tot_len = htons(skb->len);
1253         iph->frag_off = df;
1254         ip_select_ident(iph, &rt->u.dst, sk);
1255         iph->ttl = ttl;
1256         iph->protocol = sk->sk_protocol;
1257         iph->saddr = rt->rt_src;
1258         iph->daddr = rt->rt_dst;
1259         ip_send_check(iph);
1260
1261         skb->priority = sk->sk_priority;
1262         skb->dst = dst_clone(&rt->u.dst);
1263
1264         /* Netfilter gets whole the not fragmented skb. */
1265         err = NF_HOOK(PF_INET, NF_IP_LOCAL_OUT, skb, NULL,
1266                       skb->dst->dev, dst_output);
1267         if (err) {
1268                 if (err > 0)
1269                         err = inet->recverr ? net_xmit_errno(err) : 0;
1270                 if (err)
1271                         goto error;
1272         }
1273
1274 out:
1275         inet->cork.flags &= ~IPCORK_OPT;
1276         kfree(inet->cork.opt);
1277         inet->cork.opt = NULL;
1278         if (inet->cork.rt) {
1279                 ip_rt_put(inet->cork.rt);
1280                 inet->cork.rt = NULL;
1281         }
1282         return err;
1283
1284 error:
1285         IP_INC_STATS(IPSTATS_MIB_OUTDISCARDS);
1286         goto out;
1287 }
1288
1289 /*
1290  *      Throw away all pending data on the socket.
1291  */
1292 void ip_flush_pending_frames(struct sock *sk)
1293 {
1294         struct inet_sock *inet = inet_sk(sk);
1295         struct sk_buff *skb;
1296
1297         while ((skb = __skb_dequeue_tail(&sk->sk_write_queue)) != NULL)
1298                 kfree_skb(skb);
1299
1300         inet->cork.flags &= ~IPCORK_OPT;
1301         kfree(inet->cork.opt);
1302         inet->cork.opt = NULL;
1303         if (inet->cork.rt) {
1304                 ip_rt_put(inet->cork.rt);
1305                 inet->cork.rt = NULL;
1306         }
1307 }
1308
1309
1310 /*
1311  *      Fetch data from kernel space and fill in checksum if needed.
1312  */
1313 static int ip_reply_glue_bits(void *dptr, char *to, int offset,
1314                               int len, int odd, struct sk_buff *skb)
1315 {
1316         unsigned int csum;
1317
1318         csum = csum_partial_copy_nocheck(dptr+offset, to, len, 0);
1319         skb->csum = csum_block_add(skb->csum, csum, odd);
1320         return 0;
1321 }
1322
1323 /*
1324  *      Generic function to send a packet as reply to another packet.
1325  *      Used to send TCP resets so far. ICMP should use this function too.
1326  *
1327  *      Should run single threaded per socket because it uses the sock
1328  *      structure to pass arguments.
1329  *
1330  *      LATER: switch from ip_build_xmit to ip_append_*
1331  */
1332 void ip_send_reply(struct sock *sk, struct sk_buff *skb, struct ip_reply_arg *arg,
1333                    unsigned int len)
1334 {
1335         struct inet_sock *inet = inet_sk(sk);
1336         struct {
1337                 struct ip_options       opt;
1338                 char                    data[40];
1339         } replyopts;
1340         struct ipcm_cookie ipc;
1341         u32 daddr;
1342         struct rtable *rt = (struct rtable*)skb->dst;
1343
1344         if (ip_options_echo(&replyopts.opt, skb))
1345                 return;
1346
1347         daddr = ipc.addr = rt->rt_src;
1348         ipc.opt = NULL;
1349
1350         if (replyopts.opt.optlen) {
1351                 ipc.opt = &replyopts.opt;
1352
1353                 if (ipc.opt->srr)
1354                         daddr = replyopts.opt.faddr;
1355         }
1356
1357         {
1358                 struct flowi fl = { .nl_u = { .ip4_u =
1359                                               { .daddr = daddr,
1360                                                 .saddr = rt->rt_spec_dst,
1361                                                 .tos = RT_TOS(skb->nh.iph->tos) } },
1362                                     /* Not quite clean, but right. */
1363                                     .uli_u = { .ports =
1364                                                { .sport = skb->h.th->dest,
1365                                                  .dport = skb->h.th->source } },
1366                                     .proto = sk->sk_protocol };
1367                 if (ip_route_output_key(&rt, &fl))
1368                         return;
1369         }
1370
1371         /* And let IP do all the hard work.
1372
1373            This chunk is not reenterable, hence spinlock.
1374            Note that it uses the fact, that this function is called
1375            with locally disabled BH and that sk cannot be already spinlocked.
1376          */
1377         bh_lock_sock(sk);
1378         inet->tos = skb->nh.iph->tos;
1379         sk->sk_priority = skb->priority;
1380         sk->sk_protocol = skb->nh.iph->protocol;
1381         ip_append_data(sk, ip_reply_glue_bits, arg->iov->iov_base, len, 0,
1382                        &ipc, rt, MSG_DONTWAIT);
1383         if ((skb = skb_peek(&sk->sk_write_queue)) != NULL) {
1384                 if (arg->csumoffset >= 0)
1385                         *((u16 *)skb->h.raw + arg->csumoffset) = csum_fold(csum_add(skb->csum, arg->csum));
1386                 skb->ip_summed = CHECKSUM_NONE;
1387                 ip_push_pending_frames(sk);
1388         }
1389
1390         bh_unlock_sock(sk);
1391
1392         ip_rt_put(rt);
1393 }
1394
1395 void __init ip_init(void)
1396 {
1397         ip_rt_init();
1398         inet_initpeers();
1399
1400 #if defined(CONFIG_IP_MULTICAST) && defined(CONFIG_PROC_FS)
1401         igmp_mc_proc_init();
1402 #endif
1403 }
1404
1405 EXPORT_SYMBOL(ip_generic_getfrag);
1406 EXPORT_SYMBOL(ip_queue_xmit);
1407 EXPORT_SYMBOL(ip_send_check);