net/ipv4/tcp.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              Implementation of the Transmission Control Protocol(TCP).
   7  *
   8  * Version:     $Id: tcp.c,v 1.216 2002/02/01 22:01:04 davem Exp $
   9  *
  10  * Authors:     Ross Biro
  11  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  12  *              Mark Evans, <evansmp@uhura.aston.ac.uk>
  13  *              Corey Minyard <wf-rch!minyard@relay.EU.net>
  14  *              Florian La Roche, <flla@stud.uni-sb.de>
  15  *              Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
  16  *              Linus Torvalds, <torvalds@cs.helsinki.fi>
  17  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  18  *              Matthew Dillon, <dillon@apollo.west.oic.com>
  19  *              Arnt Gulbrandsen, <agulbra@nvg.unit.no>
  20  *              Jorge Cwik, <jorge@laser.satlink.net>
  21  *
  22  * Fixes:
  23  *              Alan Cox        :       Numerous verify_area() calls
  24  *              Alan Cox        :       Set the ACK bit on a reset
  25  *              Alan Cox        :       Stopped it crashing if it closed while
  26  *                                      sk->inuse=1 and was trying to connect
  27  *                                      (tcp_err()).
  28  *              Alan Cox        :       All icmp error handling was broken
  29  *                                      pointers passed where wrong and the
  30  *                                      socket was looked up backwards. Nobody
  31  *                                      tested any icmp error code obviously.
  32  *              Alan Cox        :       tcp_err() now handled properly. It
  33  *                                      wakes people on errors. poll
  34  *                                      behaves and the icmp error race
  35  *                                      has gone by moving it into sock.c
  36  *              Alan Cox        :       tcp_send_reset() fixed to work for
  37  *                                      everything not just packets for
  38  *                                      unknown sockets.
  39  *              Alan Cox        :       tcp option processing.
  40  *              Alan Cox        :       Reset tweaked (still not 100%) [Had
  41  *                                      syn rule wrong]
  42  *              Herp Rosmanith  :       More reset fixes
  43  *              Alan Cox        :       No longer acks invalid rst frames.
  44  *                                      Acking any kind of RST is right out.
  45  *              Alan Cox        :       Sets an ignore me flag on an rst
  46  *                                      receive otherwise odd bits of prattle
  47  *                                      escape still
  48  *              Alan Cox        :       Fixed another acking RST frame bug.
  49  *                                      Should stop LAN workplace lockups.
  50  *              Alan Cox        :       Some tidyups using the new skb list
  51  *                                      facilities
  52  *              Alan Cox        :       sk->keepopen now seems to work
  53  *              Alan Cox        :       Pulls options out correctly on accepts
  54  *              Alan Cox        :       Fixed assorted sk->rqueue->next errors
  55  *              Alan Cox        :       PSH doesn't end a TCP read. Switched a
  56  *                                      bit to skb ops.
  57  *              Alan Cox        :       Tidied tcp_data to avoid a potential
  58  *                                      nasty.
  59  *              Alan Cox        :       Added some better commenting, as the
  60  *                                      tcp is hard to follow
  61  *              Alan Cox        :       Removed incorrect check for 20 * psh
  62  *      Michael O'Reilly        :       ack < copied bug fix.
  63  *      Johannes Stille         :       Misc tcp fixes (not all in yet).
  64  *              Alan Cox        :       FIN with no memory -> CRASH
  65  *              Alan Cox        :       Added socket option proto entries.
  66  *                                      Also added awareness of them to accept.
  67  *              Alan Cox        :       Added TCP options (SOL_TCP)
  68  *              Alan Cox        :       Switched wakeup calls to callbacks,
  69  *                                      so the kernel can layer network
  70  *                                      sockets.
  71  *              Alan Cox        :       Use ip_tos/ip_ttl settings.
  72  *              Alan Cox        :       Handle FIN (more) properly (we hope).
  73  *              Alan Cox        :       RST frames sent on unsynchronised
  74  *                                      state ack error.
  75  *              Alan Cox        :       Put in missing check for SYN bit.
  76  *              Alan Cox        :       Added tcp_select_window() aka NET2E
  77  *                                      window non shrink trick.
  78  *              Alan Cox        :       Added a couple of small NET2E timer
  79  *                                      fixes
  80  *              Charles Hedrick :       TCP fixes
  81  *              Toomas Tamm     :       TCP window fixes
  82  *              Alan Cox        :       Small URG fix to rlogin ^C ack fight
  83  *              Charles Hedrick :       Rewrote most of it to actually work
  84  *              Linus           :       Rewrote tcp_read() and URG handling
  85  *                                      completely
  86  *              Gerhard Koerting:       Fixed some missing timer handling
  87  *              Matthew Dillon  :       Reworked TCP machine states as per RFC
  88  *              Gerhard Koerting:       PC/TCP workarounds
  89  *              Adam Caldwell   :       Assorted timer/timing errors
  90  *              Matthew Dillon  :       Fixed another RST bug
  91  *              Alan Cox        :       Move to kernel side addressing changes.
  92  *              Alan Cox        :       Beginning work on TCP fastpathing
  93  *                                      (not yet usable)
  94  *              Arnt Gulbrandsen:       Turbocharged tcp_check() routine.
  95  *              Alan Cox        :       TCP fast path debugging
  96  *              Alan Cox        :       Window clamping
  97  *              Michael Riepe   :       Bug in tcp_check()
  98  *              Matt Dillon     :       More TCP improvements and RST bug fixes
  99  *              Matt Dillon     :       Yet more small nasties remove from the
 100  *                                      TCP code (Be very nice to this man if
 101  *                                      tcp finally works 100%) 8)
 102  *              Alan Cox        :       BSD accept semantics.
 103  *              Alan Cox        :       Reset on closedown bug.
 104  *      Peter De Schrijver      :       ENOTCONN check missing in tcp_sendto().
 105  *              Michael Pall    :       Handle poll() after URG properly in
 106  *                                      all cases.
 107  *              Michael Pall    :       Undo the last fix in tcp_read_urg()
 108  *                                      (multi URG PUSH broke rlogin).
 109  *              Michael Pall    :       Fix the multi URG PUSH problem in
 110  *                                      tcp_readable(), poll() after URG
 111  *                                      works now.
 112  *              Michael Pall    :       recv(...,MSG_OOB) never blocks in the
 113  *                                      BSD api.
 114  *              Alan Cox        :       Changed the semantics of sk->socket to
 115  *                                      fix a race and a signal problem with
 116  *                                      accept() and async I/O.
 117  *              Alan Cox        :       Relaxed the rules on tcp_sendto().
 118  *              Yury Shevchuk   :       Really fixed accept() blocking problem.
 119  *              Craig I. Hagan  :       Allow for BSD compatible TIME_WAIT for
 120  *                                      clients/servers which listen in on
 121  *                                      fixed ports.
 122  *              Alan Cox        :       Cleaned the above up and shrank it to
 123  *                                      a sensible code size.
 124  *              Alan Cox        :       Self connect lockup fix.
 125  *              Alan Cox        :       No connect to multicast.
 126  *              Ross Biro       :       Close unaccepted children on master
 127  *                                      socket close.
 128  *              Alan Cox        :       Reset tracing code.
 129  *              Alan Cox        :       Spurious resets on shutdown.
 130  *              Alan Cox        :       Giant 15 minute/60 second timer error
 131  *              Alan Cox        :       Small whoops in polling before an
 132  *                                      accept.
 133  *              Alan Cox        :       Kept the state trace facility since
 134  *                                      it's handy for debugging.
 135  *              Alan Cox        :       More reset handler fixes.
 136  *              Alan Cox        :       Started rewriting the code based on
 137  *                                      the RFC's for other useful protocol
 138  *                                      references see: Comer, KA9Q NOS, and
 139  *                                      for a reference on the difference
 140  *                                      between specifications and how BSD
 141  *                                      works see the 4.4lite source.
 142  *              A.N.Kuznetsov   :       Don't time wait on completion of tidy
 143  *                                      close.
 144  *              Linus Torvalds  :       Fin/Shutdown & copied_seq changes.
 145  *              Linus Torvalds  :       Fixed BSD port reuse to work first syn
 146  *              Alan Cox        :       Reimplemented timers as per the RFC
 147  *                                      and using multiple timers for sanity.
 148  *              Alan Cox        :       Small bug fixes, and a lot of new
 149  *                                      comments.
 150  *              Alan Cox        :       Fixed dual reader crash by locking
 151  *                                      the buffers (much like datagram.c)
 152  *              Alan Cox        :       Fixed stuck sockets in probe. A probe
 153  *                                      now gets fed up of retrying without
 154  *                                      (even a no space) answer.
 155  *              Alan Cox        :       Extracted closing code better
 156  *              Alan Cox        :       Fixed the closing state machine to
 157  *                                      resemble the RFC.
 158  *              Alan Cox        :       More 'per spec' fixes.
 159  *              Jorge Cwik      :       Even faster checksumming.
 160  *              Alan Cox        :       tcp_data() doesn't ack illegal PSH
 161  *                                      only frames. At least one pc tcp stack
 162  *                                      generates them.
 163  *              Alan Cox        :       Cache last socket.
 164  *              Alan Cox        :       Per route irtt.
 165  *              Matt Day        :       poll()->select() match BSD precisely on error
 166  *              Alan Cox        :       New buffers
 167  *              Marc Tamsky     :       Various sk->prot->retransmits and
 168  *                                      sk->retransmits misupdating fixed.
 169  *                                      Fixed tcp_write_timeout: stuck close,
 170  *                                      and TCP syn retries gets used now.
 171  *              Mark Yarvis     :       In tcp_read_wakeup(), don't send an
 172  *                                      ack if state is TCP_CLOSED.
 173  *              Alan Cox        :       Look up device on a retransmit - routes may
 174  *                                      change. Doesn't yet cope with MSS shrink right
 175  *                                      but it's a start!
 176  *              Marc Tamsky     :       Closing in closing fixes.
 177  *              Mike Shaver     :       RFC1122 verifications.
 178  *              Alan Cox        :       rcv_saddr errors.
 179  *              Alan Cox        :       Block double connect().
 180  *              Alan Cox        :       Small hooks for enSKIP.
 181  *              Alexey Kuznetsov:       Path MTU discovery.
 182  *              Alan Cox        :       Support soft errors.
 183  *              Alan Cox        :       Fix MTU discovery pathological case
 184  *                                      when the remote claims no mtu!
 185  *              Marc Tamsky     :       TCP_CLOSE fix.
 186  *              Colin (G3TNE)   :       Send a reset on syn ack replies in
 187  *                                      window but wrong (fixes NT lpd problems)
 188  *              Pedro Roque     :       Better TCP window handling, delayed ack.
 189  *              Joerg Reuter    :       No modification of locked buffers in
 190  *                                      tcp_do_retransmit()
 191  *              Eric Schenk     :       Changed receiver side silly window
 192  *                                      avoidance algorithm to BSD style
 193  *                                      algorithm. This doubles throughput
 194  *                                      against machines running Solaris,
 195  *                                      and seems to result in general
 196  *                                      improvement.
 197  *      Stefan Magdalinski      :       adjusted tcp_readable() to fix FIONREAD
 198  *      Willy Konynenberg       :       Transparent proxying support.
 199  *      Mike McLagan            :       Routing by source
 200  *              Keith Owens     :       Do proper merging with partial SKB's in
 201  *                                      tcp_do_sendmsg to avoid burstiness.
 202  *              Eric Schenk     :       Fix fast close down bug with
 203  *                                      shutdown() followed by close().
 204  *              Andi Kleen      :       Make poll agree with SIGIO
 205  *      Salvatore Sanfilippo    :       Support SO_LINGER with linger == 1 and
 206  *                                      lingertime == 0 (RFC 793 ABORT Call)
 207  *      Hirokazu Takahashi      :       Use copy_from_user() instead of
 208  *                                      csum_and_copy_from_user() if possible.
 209  *
 210  *              This program is free software; you can redistribute it and/or
 211  *              modify it under the terms of the GNU General Public License
 212  *              as published by the Free Software Foundation; either version
 213  *              2 of the License, or(at your option) any later version.
 214  *
 215  * Description of States:
 216  *
 217  *      TCP_SYN_SENT            sent a connection request, waiting for ack
 218  *
 219  *      TCP_SYN_RECV            received a connection request, sent ack,
 220  *                              waiting for final ack in three-way handshake.
 221  *
 222  *      TCP_ESTABLISHED         connection established
 223  *
 224  *      TCP_FIN_WAIT1           our side has shutdown, waiting to complete
 225  *                              transmission of remaining buffered data
 226  *
 227  *      TCP_FIN_WAIT2           all buffered data sent, waiting for remote
 228  *                              to shutdown
 229  *
 230  *      TCP_CLOSING             both sides have shutdown but we still have
 231  *                              data we have to finish sending
 232  *
 233  *      TCP_TIME_WAIT           timeout to catch resent junk before entering
 234  *                              closed, can only be entered from FIN_WAIT2
 235  *                              or CLOSING.  Required because the other end
 236  *                              may not have gotten our last ACK causing it
 237  *                              to retransmit the data packet (which we ignore)
 238  *
 239  *      TCP_CLOSE_WAIT          remote side has shutdown and is waiting for
 240  *                              us to finish writing our data and to shutdown
 241  *                              (we have to close() to move on to LAST_ACK)
 242  *
 243  *      TCP_LAST_ACK            out side has shutdown after remote has
 244  *                              shutdown.  There may still be data in our
 245  *                              buffer that we have to finish sending
 246  *
 247  *      TCP_CLOSE               socket is finished
 248  */
 249
 250 #include <linux/kernel.h>
 251 #include <linux/module.h>
 252 #include <linux/types.h>
 253 #include <linux/fcntl.h>
 254 #include <linux/poll.h>
 255 #include <linux/init.h>
 256 #include <linux/fs.h>
 257 #include <linux/skbuff.h>
 258 #include <linux/scatterlist.h>
 259 #include <linux/splice.h>
 260 #include <linux/net.h>
 261 #include <linux/socket.h>
 262 #include <linux/random.h>
 263 #include <linux/bootmem.h>
 264 #include <linux/highmem.h>
 265 #include <linux/swap.h>
 266 #include <linux/cache.h>
 267 #include <linux/err.h>
 268 #include <linux/crypto.h>
 269
 270 #include <net/icmp.h>
 271 #include <net/tcp.h>
 272 #include <net/xfrm.h>
 273 #include <net/ip.h>
 274 #include <net/netdma.h>
 275 #include <net/sock.h>
 276
 277 #include <asm/uaccess.h>
 278 #include <asm/ioctls.h>
 279
 280 int sysctl_tcp_fin_timeout __read_mostly = TCP_FIN_TIMEOUT;
 281
 282 DEFINE_SNMP_STAT(struct tcp_mib, tcp_statistics) __read_mostly;
 283
 284 atomic_t tcp_orphan_count = ATOMIC_INIT(0);
 285
 286 EXPORT_SYMBOL_GPL(tcp_orphan_count);
 287
 288 int sysctl_tcp_mem[3] __read_mostly;
 289 int sysctl_tcp_wmem[3] __read_mostly;
 290 int sysctl_tcp_rmem[3] __read_mostly;
 291
 292 EXPORT_SYMBOL(sysctl_tcp_mem);
 293 EXPORT_SYMBOL(sysctl_tcp_rmem);
 294 EXPORT_SYMBOL(sysctl_tcp_wmem);
 295
 296 atomic_t tcp_memory_allocated;  /* Current allocated memory. */
 297 atomic_t tcp_sockets_allocated; /* Current number of TCP sockets. */
 298
 299 EXPORT_SYMBOL(tcp_memory_allocated);
 300 EXPORT_SYMBOL(tcp_sockets_allocated);
 301
 302 /*
 303  * TCP splice context
 304  */
 305 struct tcp_splice_state {
 306         struct pipe_inode_info *pipe;
 307         size_t len;
 308         unsigned int flags;
 309 };
 310
 311 /*
 312  * Pressure flag: try to collapse.
 313  * Technical note: it is used by multiple contexts non atomically.
 314  * All the __sk_mem_schedule() is of this nature: accounting
 315  * is strict, actions are advisory and have some latency.
 316  */
 317 int tcp_memory_pressure __read_mostly;
 318
 319 EXPORT_SYMBOL(tcp_memory_pressure);
 320
 321 void tcp_enter_memory_pressure(void)
 322 {
 323         if (!tcp_memory_pressure) {
 324                 NET_INC_STATS(LINUX_MIB_TCPMEMORYPRESSURES);
 325                 tcp_memory_pressure = 1;
 326         }
 327 }
 328
 329 EXPORT_SYMBOL(tcp_enter_memory_pressure);
 330
 331 /*
 332  *      Wait for a TCP event.
 333  *
 334  *      Note that we don't need to lock the socket, as the upper poll layers
 335  *      take care of normal races (between the test and the event) and we don't
 336  *      go look at any of the socket buffers directly.
 337  */
 338 unsigned int tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
 339 {
 340         unsigned int mask;
 341         struct sock *sk = sock->sk;
 342         struct tcp_sock *tp = tcp_sk(sk);
 343
 344         poll_wait(file, sk->sk_sleep, wait);
 345         if (sk->sk_state == TCP_LISTEN)
 346                 return inet_csk_listen_poll(sk);
 347
 348         /* Socket is not locked. We are protected from async events
 349            by poll logic and correct handling of state changes
 350            made by another threads is impossible in any case.
 351          */
 352
 353         mask = 0;
 354         if (sk->sk_err)
 355                 mask = POLLERR;
 356
 357         /*
 358          * POLLHUP is certainly not done right. But poll() doesn't
 359          * have a notion of HUP in just one direction, and for a
 360          * socket the read side is more interesting.
 361          *
 362          * Some poll() documentation says that POLLHUP is incompatible
 363          * with the POLLOUT/POLLWR flags, so somebody should check this
 364          * all. But careful, it tends to be safer to return too many
 365          * bits than too few, and you can easily break real applications
 366          * if you don't tell them that something has hung up!
 367          *
 368          * Check-me.
 369          *
 370          * Check number 1. POLLHUP is _UNMASKABLE_ event (see UNIX98 and
 371          * our fs/select.c). It means that after we received EOF,
 372          * poll always returns immediately, making impossible poll() on write()
 373          * in state CLOSE_WAIT. One solution is evident --- to set POLLHUP
 374          * if and only if shutdown has been made in both directions.
 375          * Actually, it is interesting to look how Solaris and DUX
 376          * solve this dilemma. I would prefer, if PULLHUP were maskable,
 377          * then we could set it on SND_SHUTDOWN. BTW examples given
 378          * in Stevens' books assume exactly this behaviour, it explains
 379          * why PULLHUP is incompatible with POLLOUT.    --ANK
 380          *
 381          * NOTE. Check for TCP_CLOSE is added. The goal is to prevent
 382          * blocking on fresh not-connected or disconnected socket. --ANK
 383          */
 384         if (sk->sk_shutdown == SHUTDOWN_MASK || sk->sk_state == TCP_CLOSE)
 385                 mask |= POLLHUP;
 386         if (sk->sk_shutdown & RCV_SHUTDOWN)
 387                 mask |= POLLIN | POLLRDNORM | POLLRDHUP;
 388
 389         /* Connected? */
 390         if ((1 << sk->sk_state) & ~(TCPF_SYN_SENT | TCPF_SYN_RECV)) {
 391                 /* Potential race condition. If read of tp below will
 392                  * escape above sk->sk_state, we can be illegally awaken
 393                  * in SYN_* states. */
 394                 if ((tp->rcv_nxt != tp->copied_seq) &&
 395                     (tp->urg_seq != tp->copied_seq ||
 396                      tp->rcv_nxt != tp->copied_seq + 1 ||
 397                      sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data))
 398                         mask |= POLLIN | POLLRDNORM;
 399
 400                 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
 401                         if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk)) {
 402                                 mask |= POLLOUT | POLLWRNORM;
 403                         } else {  /* send SIGIO later */
 404                                 set_bit(SOCK_ASYNC_NOSPACE,
 405                                         &sk->sk_socket->flags);
 406                                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 407
 408                                 /* Race breaker. If space is freed after
 409                                  * wspace test but before the flags are set,
 410                                  * IO signal will be lost.
 411                                  */
 412                                 if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk))
 413                                         mask |= POLLOUT | POLLWRNORM;
 414                         }
 415                 }
 416
 417                 if (tp->urg_data & TCP_URG_VALID)
 418                         mask |= POLLPRI;
 419         }
 420         return mask;
 421 }
 422
 423 int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg)
 424 {
 425         struct tcp_sock *tp = tcp_sk(sk);
 426         int answ;
 427
 428         switch (cmd) {
 429         case SIOCINQ:
 430                 if (sk->sk_state == TCP_LISTEN)
 431                         return -EINVAL;
 432
 433                 lock_sock(sk);
 434                 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
 435                         answ = 0;
 436                 else if (sock_flag(sk, SOCK_URGINLINE) ||
 437                          !tp->urg_data ||
 438                          before(tp->urg_seq, tp->copied_seq) ||
 439                          !before(tp->urg_seq, tp->rcv_nxt)) {
 440                         answ = tp->rcv_nxt - tp->copied_seq;
 441
 442                         /* Subtract 1, if FIN is in queue. */
 443                         if (answ && !skb_queue_empty(&sk->sk_receive_queue))
 444                                 answ -=
 445                        tcp_hdr((struct sk_buff *)sk->sk_receive_queue.prev)->fin;
 446                 } else
 447                         answ = tp->urg_seq - tp->copied_seq;
 448                 release_sock(sk);
 449                 break;
 450         case SIOCATMARK:
 451                 answ = tp->urg_data && tp->urg_seq == tp->copied_seq;
 452                 break;
 453         case SIOCOUTQ:
 454                 if (sk->sk_state == TCP_LISTEN)
 455                         return -EINVAL;
 456
 457                 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
 458                         answ = 0;
 459                 else
 460                         answ = tp->write_seq - tp->snd_una;
 461                 break;
 462         default:
 463                 return -ENOIOCTLCMD;
 464         }
 465
 466         return put_user(answ, (int __user *)arg);
 467 }
 468
 469 static inline void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb)
 470 {
 471         TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_PSH;
 472         tp->pushed_seq = tp->write_seq;
 473 }
 474
 475 static inline int forced_push(struct tcp_sock *tp)
 476 {
 477         return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1));
 478 }
 479
 480 static inline void skb_entail(struct sock *sk, struct sk_buff *skb)
 481 {
 482         struct tcp_sock *tp = tcp_sk(sk);
 483         struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
 484
 485         skb->csum    = 0;
 486         tcb->seq     = tcb->end_seq = tp->write_seq;
 487         tcb->flags   = TCPCB_FLAG_ACK;
 488         tcb->sacked  = 0;
 489         skb_header_release(skb);
 490         tcp_add_write_queue_tail(sk, skb);
 491         sk->sk_wmem_queued += skb->truesize;
 492         sk_mem_charge(sk, skb->truesize);
 493         if (tp->nonagle & TCP_NAGLE_PUSH)
 494                 tp->nonagle &= ~TCP_NAGLE_PUSH;
 495 }
 496
 497 static inline void tcp_mark_urg(struct tcp_sock *tp, int flags,
 498                                 struct sk_buff *skb)
 499 {
 500         if (flags & MSG_OOB) {
 501                 tp->urg_mode = 1;
 502                 tp->snd_up = tp->write_seq;
 503         }
 504 }
 505
 506 static inline void tcp_push(struct sock *sk, int flags, int mss_now,
 507                             int nonagle)
 508 {
 509         struct tcp_sock *tp = tcp_sk(sk);
 510
 511         if (tcp_send_head(sk)) {
 512                 struct sk_buff *skb = tcp_write_queue_tail(sk);
 513                 if (!(flags & MSG_MORE) || forced_push(tp))
 514                         tcp_mark_push(tp, skb);
 515                 tcp_mark_urg(tp, flags, skb);
 516                 __tcp_push_pending_frames(sk, mss_now,
 517                                           (flags & MSG_MORE) ? TCP_NAGLE_CORK : nonagle);
 518         }
 519 }
 520
 521 static int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
 522                                 unsigned int offset, size_t len)
 523 {
 524         struct tcp_splice_state *tss = rd_desc->arg.data;
 525
 526         return skb_splice_bits(skb, offset, tss->pipe, tss->len, tss->flags);
 527 }
 528
 529 static int __tcp_splice_read(struct sock *sk, struct tcp_splice_state *tss)
 530 {
 531         /* Store TCP splice context information in read_descriptor_t. */
 532         read_descriptor_t rd_desc = {
 533                 .arg.data = tss,
 534         };
 535
 536         return tcp_read_sock(sk, &rd_desc, tcp_splice_data_recv);
 537 }
 538
 539 /**
 540  *  tcp_splice_read - splice data from TCP socket to a pipe
 541  * @sock:       socket to splice from
 542  * @ppos:       position (not valid)
 543  * @pipe:       pipe to splice to
 544  * @len:        number of bytes to splice
 545  * @flags:      splice modifier flags
 546  *
 547  * Description:
 548  *    Will read pages from given socket and fill them into a pipe.
 549  *
 550  **/
 551 ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,
 552                         struct pipe_inode_info *pipe, size_t len,
 553                         unsigned int flags)
 554 {
 555         struct sock *sk = sock->sk;
 556         struct tcp_splice_state tss = {
 557                 .pipe = pipe,
 558                 .len = len,
 559                 .flags = flags,
 560         };
 561         long timeo;
 562         ssize_t spliced;
 563         int ret;
 564
 565         /*
 566          * We can't seek on a socket input
 567          */
 568         if (unlikely(*ppos))
 569                 return -ESPIPE;
 570
 571         ret = spliced = 0;
 572
 573         lock_sock(sk);
 574
 575         timeo = sock_rcvtimeo(sk, flags & SPLICE_F_NONBLOCK);
 576         while (tss.len) {
 577                 ret = __tcp_splice_read(sk, &tss);
 578                 if (ret < 0)
 579                         break;
 580                 else if (!ret) {
 581                         if (spliced)
 582                                 break;
 583                         if (flags & SPLICE_F_NONBLOCK) {
 584                                 ret = -EAGAIN;
 585                                 break;
 586                         }
 587                         if (sock_flag(sk, SOCK_DONE))
 588                                 break;
 589                         if (sk->sk_err) {
 590                                 ret = sock_error(sk);
 591                                 break;
 592                         }
 593                         if (sk->sk_shutdown & RCV_SHUTDOWN)
 594                                 break;
 595                         if (sk->sk_state == TCP_CLOSE) {
 596                                 /*
 597                                  * This occurs when user tries to read
 598                                  * from never connected socket.
 599                                  */
 600                                 if (!sock_flag(sk, SOCK_DONE))
 601                                         ret = -ENOTCONN;
 602                                 break;
 603                         }
 604                         if (!timeo) {
 605                                 ret = -EAGAIN;
 606                                 break;
 607                         }
 608                         sk_wait_data(sk, &timeo);
 609                         if (signal_pending(current)) {
 610                                 ret = sock_intr_errno(timeo);
 611                                 break;
 612                         }
 613                         continue;
 614                 }
 615                 tss.len -= ret;
 616                 spliced += ret;
 617
 618                 release_sock(sk);
 619                 lock_sock(sk);
 620
 621                 if (sk->sk_err || sk->sk_state == TCP_CLOSE ||
 622                     (sk->sk_shutdown & RCV_SHUTDOWN) || !timeo ||
 623                     signal_pending(current))
 624                         break;
 625         }
 626
 627         release_sock(sk);
 628
 629         if (spliced)
 630                 return spliced;
 631
 632         return ret;
 633 }
 634
 635 struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp)
 636 {
 637         struct sk_buff *skb;
 638
 639         /* The TCP header must be at least 32-bit aligned.  */
 640         size = ALIGN(size, 4);
 641
 642         skb = alloc_skb_fclone(size + sk->sk_prot->max_header, gfp);
 643         if (skb) {
 644                 if (sk_wmem_schedule(sk, skb->truesize)) {
 645                         /*
 646                          * Make sure that we have exactly size bytes
 647                          * available to the caller, no more, no less.
 648                          */
 649                         skb_reserve(skb, skb_tailroom(skb) - size);
 650                         return skb;
 651                 }
 652                 __kfree_skb(skb);
 653         } else {
 654                 sk->sk_prot->enter_memory_pressure();
 655                 sk_stream_moderate_sndbuf(sk);
 656         }
 657         return NULL;
 658 }
 659
 660 static ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffset,
 661                          size_t psize, int flags)
 662 {
 663         struct tcp_sock *tp = tcp_sk(sk);
 664         int mss_now, size_goal;
 665         int err;
 666         ssize_t copied;
 667         long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
 668
 669         /* Wait for a connection to finish. */
 670         if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT))
 671                 if ((err = sk_stream_wait_connect(sk, &timeo)) != 0)
 672                         goto out_err;
 673
 674         clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
 675
 676         mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
 677         size_goal = tp->xmit_size_goal;
 678         copied = 0;
 679
 680         err = -EPIPE;
 681         if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
 682                 goto do_error;
 683
 684         while (psize > 0) {
 685                 struct sk_buff *skb = tcp_write_queue_tail(sk);
 686                 struct page *page = pages[poffset / PAGE_SIZE];
 687                 int copy, i, can_coalesce;
 688                 int offset = poffset % PAGE_SIZE;
 689                 int size = min_t(size_t, psize, PAGE_SIZE - offset);
 690
 691                 if (!tcp_send_head(sk) || (copy = size_goal - skb->len) <= 0) {
 692 new_segment:
 693                         if (!sk_stream_memory_free(sk))
 694                                 goto wait_for_sndbuf;
 695
 696                         skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation);
 697                         if (!skb)
 698                                 goto wait_for_memory;
 699
 700                         skb_entail(sk, skb);
 701                         copy = size_goal;
 702                 }
 703
 704                 if (copy > size)
 705                         copy = size;
 706
 707                 i = skb_shinfo(skb)->nr_frags;
 708                 can_coalesce = skb_can_coalesce(skb, i, page, offset);
 709                 if (!can_coalesce && i >= MAX_SKB_FRAGS) {
 710                         tcp_mark_push(tp, skb);
 711                         goto new_segment;
 712                 }
 713                 if (!sk_wmem_schedule(sk, copy))
 714                         goto wait_for_memory;
 715
 716                 if (can_coalesce) {
 717                         skb_shinfo(skb)->frags[i - 1].size += copy;
 718                 } else {
 719                         get_page(page);
 720                         skb_fill_page_desc(skb, i, page, offset, copy);
 721                 }
 722
 723                 skb->len += copy;
 724                 skb->data_len += copy;
 725                 skb->truesize += copy;
 726                 sk->sk_wmem_queued += copy;
 727                 sk_mem_charge(sk, copy);
 728                 skb->ip_summed = CHECKSUM_PARTIAL;
 729                 tp->write_seq += copy;
 730                 TCP_SKB_CB(skb)->end_seq += copy;
 731                 skb_shinfo(skb)->gso_segs = 0;
 732
 733                 if (!copied)
 734                         TCP_SKB_CB(skb)->flags &= ~TCPCB_FLAG_PSH;
 735
 736                 copied += copy;
 737                 poffset += copy;
 738                 if (!(psize -= copy))
 739                         goto out;
 740
 741                 if (skb->len < size_goal || (flags & MSG_OOB))
 742                         continue;
 743
 744                 if (forced_push(tp)) {
 745                         tcp_mark_push(tp, skb);
 746                         __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
 747                 } else if (skb == tcp_send_head(sk))
 748                         tcp_push_one(sk, mss_now);
 749                 continue;
 750
 751 wait_for_sndbuf:
 752                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 753 wait_for_memory:
 754                 if (copied)
 755                         tcp_push(sk, flags & ~MSG_MORE, mss_now, TCP_NAGLE_PUSH);
 756
 757                 if ((err = sk_stream_wait_memory(sk, &timeo)) != 0)
 758                         goto do_error;
 759
 760                 mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
 761                 size_goal = tp->xmit_size_goal;
 762         }
 763
 764 out:
 765         if (copied)
 766                 tcp_push(sk, flags, mss_now, tp->nonagle);
 767         return copied;
 768
 769 do_error:
 770         if (copied)
 771                 goto out;
 772 out_err:
 773         return sk_stream_error(sk, flags, err);
 774 }
 775
 776 ssize_t tcp_sendpage(struct socket *sock, struct page *page, int offset,
 777                      size_t size, int flags)
 778 {
 779         ssize_t res;
 780         struct sock *sk = sock->sk;
 781
 782         if (!(sk->sk_route_caps & NETIF_F_SG) ||
 783             !(sk->sk_route_caps & NETIF_F_ALL_CSUM))
 784                 return sock_no_sendpage(sock, page, offset, size, flags);
 785
 786         lock_sock(sk);
 787         TCP_CHECK_TIMER(sk);
 788         res = do_tcp_sendpages(sk, &page, offset, size, flags);
 789         TCP_CHECK_TIMER(sk);
 790         release_sock(sk);
 791         return res;
 792 }
 793
 794 #define TCP_PAGE(sk)    (sk->sk_sndmsg_page)
 795 #define TCP_OFF(sk)     (sk->sk_sndmsg_off)
 796
 797 static inline int select_size(struct sock *sk)
 798 {
 799         struct tcp_sock *tp = tcp_sk(sk);
 800         int tmp = tp->mss_cache;
 801
 802         if (sk->sk_route_caps & NETIF_F_SG) {
 803                 if (sk_can_gso(sk))
 804                         tmp = 0;
 805                 else {
 806                         int pgbreak = SKB_MAX_HEAD(MAX_TCP_HEADER);
 807
 808                         if (tmp >= pgbreak &&
 809                             tmp <= pgbreak + (MAX_SKB_FRAGS - 1) * PAGE_SIZE)
 810                                 tmp = pgbreak;
 811                 }
 812         }
 813
 814         return tmp;
 815 }
 816
 817 int tcp_sendmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg,
 818                 size_t size)
 819 {
 820         struct sock *sk = sock->sk;
 821         struct iovec *iov;
 822         struct tcp_sock *tp = tcp_sk(sk);
 823         struct sk_buff *skb;
 824         int iovlen, flags;
 825         int mss_now, size_goal;
 826         int err, copied;
 827         long timeo;
 828
 829         lock_sock(sk);
 830         TCP_CHECK_TIMER(sk);
 831
 832         flags = msg->msg_flags;
 833         timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
 834
 835         /* Wait for a connection to finish. */
 836         if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT))
 837                 if ((err = sk_stream_wait_connect(sk, &timeo)) != 0)
 838                         goto out_err;
 839
 840         /* This should be in poll */
 841         clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
 842
 843         mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
 844         size_goal = tp->xmit_size_goal;
 845
 846         /* Ok commence sending. */
 847         iovlen = msg->msg_iovlen;
 848         iov = msg->msg_iov;
 849         copied = 0;
 850
 851         err = -EPIPE;
 852         if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
 853                 goto do_error;
 854
 855         while (--iovlen >= 0) {
 856                 int seglen = iov->iov_len;
 857                 unsigned char __user *from = iov->iov_base;
 858
 859                 iov++;
 860
 861                 while (seglen > 0) {
 862                         int copy;
 863
 864                         skb = tcp_write_queue_tail(sk);
 865
 866                         if (!tcp_send_head(sk) ||
 867                             (copy = size_goal - skb->len) <= 0) {
 868
 869 new_segment:
 870                                 /* Allocate new segment. If the interface is SG,
 871                                  * allocate skb fitting to single page.
 872                                  */
 873                                 if (!sk_stream_memory_free(sk))
 874                                         goto wait_for_sndbuf;
 875
 876                                 skb = sk_stream_alloc_skb(sk, select_size(sk),
 877                                                 sk->sk_allocation);
 878                                 if (!skb)
 879                                         goto wait_for_memory;
 880
 881                                 /*
 882                                  * Check whether we can use HW checksum.
 883                                  */
 884                                 if (sk->sk_route_caps & NETIF_F_ALL_CSUM)
 885                                         skb->ip_summed = CHECKSUM_PARTIAL;
 886
 887                                 skb_entail(sk, skb);
 888                                 copy = size_goal;
 889                         }
 890
 891                         /* Try to append data to the end of skb. */
 892                         if (copy > seglen)
 893                                 copy = seglen;
 894
 895                         /* Where to copy to? */
 896                         if (skb_tailroom(skb) > 0) {
 897                                 /* We have some space in skb head. Superb! */
 898                                 if (copy > skb_tailroom(skb))
 899                                         copy = skb_tailroom(skb);
 900                                 if ((err = skb_add_data(skb, from, copy)) != 0)
 901                                         goto do_fault;
 902                         } else {
 903                                 int merge = 0;
 904                                 int i = skb_shinfo(skb)->nr_frags;
 905                                 struct page *page = TCP_PAGE(sk);
 906                                 int off = TCP_OFF(sk);
 907
 908                                 if (skb_can_coalesce(skb, i, page, off) &&
 909                                     off != PAGE_SIZE) {
 910                                         /* We can extend the last page
 911                                          * fragment. */
 912                                         merge = 1;
 913                                 } else if (i == MAX_SKB_FRAGS ||
 914                                            (!i &&
 915                                            !(sk->sk_route_caps & NETIF_F_SG))) {
 916                                         /* Need to add new fragment and cannot
 917                                          * do this because interface is non-SG,
 918                                          * or because all the page slots are
 919                                          * busy. */
 920                                         tcp_mark_push(tp, skb);
 921                                         goto new_segment;
 922                                 } else if (page) {
 923                                         if (off == PAGE_SIZE) {
 924                                                 put_page(page);
 925                                                 TCP_PAGE(sk) = page = NULL;
 926                                                 off = 0;
 927                                         }
 928                                 } else
 929                                         off = 0;
 930
 931                                 if (copy > PAGE_SIZE - off)
 932                                         copy = PAGE_SIZE - off;
 933
 934                                 if (!sk_wmem_schedule(sk, copy))
 935                                         goto wait_for_memory;
 936
 937                                 if (!page) {
 938                                         /* Allocate new cache page. */
 939                                         if (!(page = sk_stream_alloc_page(sk)))
 940                                                 goto wait_for_memory;
 941                                 }
 942
 943                                 /* Time to copy data. We are close to
 944                                  * the end! */
 945                                 err = skb_copy_to_page(sk, from, skb, page,
 946                                                        off, copy);
 947                                 if (err) {
 948                                         /* If this page was new, give it to the
 949                                          * socket so it does not get leaked.
 950                                          */
 951                                         if (!TCP_PAGE(sk)) {
 952                                                 TCP_PAGE(sk) = page;
 953                                                 TCP_OFF(sk) = 0;
 954                                         }
 955                                         goto do_error;
 956                                 }
 957
 958                                 /* Update the skb. */
 959                                 if (merge) {
 960                                         skb_shinfo(skb)->frags[i - 1].size +=
 961                                                                         copy;
 962                                 } else {
 963                                         skb_fill_page_desc(skb, i, page, off, copy);
 964                                         if (TCP_PAGE(sk)) {
 965                                                 get_page(page);
 966                                         } else if (off + copy < PAGE_SIZE) {
 967                                                 get_page(page);
 968                                                 TCP_PAGE(sk) = page;
 969                                         }
 970                                 }
 971
 972                                 TCP_OFF(sk) = off + copy;
 973                         }
 974
 975                         if (!copied)
 976                                 TCP_SKB_CB(skb)->flags &= ~TCPCB_FLAG_PSH;
 977
 978                         tp->write_seq += copy;
 979                         TCP_SKB_CB(skb)->end_seq += copy;
 980                         skb_shinfo(skb)->gso_segs = 0;
 981
 982                         from += copy;
 983                         copied += copy;
 984                         if ((seglen -= copy) == 0 && iovlen == 0)
 985                                 goto out;
 986
 987                         if (skb->len < size_goal || (flags & MSG_OOB))
 988                                 continue;
 989
 990                         if (forced_push(tp)) {
 991                                 tcp_mark_push(tp, skb);
 992                                 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
 993                         } else if (skb == tcp_send_head(sk))
 994                                 tcp_push_one(sk, mss_now);
 995                         continue;
 996
 997 wait_for_sndbuf:
 998                         set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 999 wait_for_memory:
1000                         if (copied)
1001                                 tcp_push(sk, flags & ~MSG_MORE, mss_now, TCP_NAGLE_PUSH);
1002
1003                         if ((err = sk_stream_wait_memory(sk, &timeo)) != 0)
1004                                 goto do_error;
1005
1006                         mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
1007                         size_goal = tp->xmit_size_goal;
1008                 }
1009         }
1010
1011 out:
1012         if (copied)
1013                 tcp_push(sk, flags, mss_now, tp->nonagle);
1014         TCP_CHECK_TIMER(sk);
1015         release_sock(sk);
1016         return copied;
1017
1018 do_fault:
1019         if (!skb->len) {
1020                 tcp_unlink_write_queue(skb, sk);
1021                 /* It is the one place in all of TCP, except connection
1022                  * reset, where we can be unlinking the send_head.
1023                  */
1024                 tcp_check_send_head(sk, skb);
1025                 sk_wmem_free_skb(sk, skb);
1026         }
1027
1028 do_error:
1029         if (copied)
1030                 goto out;
1031 out_err:
1032         err = sk_stream_error(sk, flags, err);
1033         TCP_CHECK_TIMER(sk);
1034         release_sock(sk);
1035         return err;
1036 }
1037
1038 /*
1039  *      Handle reading urgent data. BSD has very simple semantics for
1040  *      this, no blocking and very strange errors 8)
1041  */
1042
1043 static int tcp_recv_urg(struct sock *sk, long timeo,
1044                         struct msghdr *msg, int len, int flags,
1045                         int *addr_len)
1046 {
1047         struct tcp_sock *tp = tcp_sk(sk);
1048
1049         /* No URG data to read. */
1050         if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data ||
1051             tp->urg_data == TCP_URG_READ)
1052                 return -EINVAL; /* Yes this is right ! */
1053
1054         if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE))
1055                 return -ENOTCONN;
1056
1057         if (tp->urg_data & TCP_URG_VALID) {
1058                 int err = 0;
1059                 char c = tp->urg_data;
1060
1061                 if (!(flags & MSG_PEEK))
1062                         tp->urg_data = TCP_URG_READ;
1063
1064                 /* Read urgent data. */
1065                 msg->msg_flags |= MSG_OOB;
1066
1067                 if (len > 0) {
1068                         if (!(flags & MSG_TRUNC))
1069                                 err = memcpy_toiovec(msg->msg_iov, &c, 1);
1070                         len = 1;
1071                 } else
1072                         msg->msg_flags |= MSG_TRUNC;
1073
1074                 return err ? -EFAULT : len;
1075         }
1076
1077         if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN))
1078                 return 0;
1079
1080         /* Fixed the recv(..., MSG_OOB) behaviour.  BSD docs and
1081          * the available implementations agree in this case:
1082          * this call should never block, independent of the
1083          * blocking state of the socket.
1084          * Mike <pall@rz.uni-karlsruhe.de>
1085          */
1086         return -EAGAIN;
1087 }
1088
1089 /* Clean up the receive buffer for full frames taken by the user,
1090  * then send an ACK if necessary.  COPIED is the number of bytes
1091  * tcp_recvmsg has given to the user so far, it speeds up the
1092  * calculation of whether or not we must ACK for the sake of
1093  * a window update.
1094  */
1095 void tcp_cleanup_rbuf(struct sock *sk, int copied)
1096 {
1097         struct tcp_sock *tp = tcp_sk(sk);
1098         int time_to_ack = 0;
1099
1100 #if TCP_DEBUG
1101         struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
1102
1103         BUG_TRAP(!skb || before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq));
1104 #endif
1105
1106         if (inet_csk_ack_scheduled(sk)) {
1107                 const struct inet_connection_sock *icsk = inet_csk(sk);
1108                    /* Delayed ACKs frequently hit locked sockets during bulk
1109                     * receive. */
1110                 if (icsk->icsk_ack.blocked ||
1111                     /* Once-per-two-segments ACK was not sent by tcp_input.c */
1112                     tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss ||
1113                     /*
1114                      * If this read emptied read buffer, we send ACK, if
1115                      * connection is not bidirectional, user drained
1116                      * receive buffer and there was a small segment
1117                      * in queue.
1118                      */
1119                     (copied > 0 &&
1120                      ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) ||
1121                       ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) &&
1122                        !icsk->icsk_ack.pingpong)) &&
1123                       !atomic_read(&sk->sk_rmem_alloc)))
1124                         time_to_ack = 1;
1125         }
1126
1127         /* We send an ACK if we can now advertise a non-zero window
1128          * which has been raised "significantly".
1129          *
1130          * Even if window raised up to infinity, do not send window open ACK
1131          * in states, where we will not receive more. It is useless.
1132          */
1133         if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) {
1134                 __u32 rcv_window_now = tcp_receive_window(tp);
1135
1136                 /* Optimize, __tcp_select_window() is not cheap. */
1137                 if (2*rcv_window_now <= tp->window_clamp) {
1138                         __u32 new_window = __tcp_select_window(sk);
1139
1140                         /* Send ACK now, if this read freed lots of space
1141                          * in our buffer. Certainly, new_window is new window.
1142                          * We can advertise it now, if it is not less than current one.
1143                          * "Lots" means "at least twice" here.
1144                          */
1145                         if (new_window && new_window >= 2 * rcv_window_now)
1146                                 time_to_ack = 1;
1147                 }
1148         }
1149         if (time_to_ack)
1150                 tcp_send_ack(sk);
1151 }
1152
1153 static void tcp_prequeue_process(struct sock *sk)
1154 {
1155         struct sk_buff *skb;
1156         struct tcp_sock *tp = tcp_sk(sk);
1157
1158         NET_INC_STATS_USER(LINUX_MIB_TCPPREQUEUED);
1159
1160         /* RX process wants to run with disabled BHs, though it is not
1161          * necessary */
1162         local_bh_disable();
1163         while ((skb = __skb_dequeue(&tp->ucopy.prequeue)) != NULL)
1164                 sk->sk_backlog_rcv(sk, skb);
1165         local_bh_enable();
1166
1167         /* Clear memory counter. */
1168         tp->ucopy.memory = 0;
1169 }
1170
1171 static inline struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
1172 {
1173         struct sk_buff *skb;
1174         u32 offset;
1175
1176         skb_queue_walk(&sk->sk_receive_queue, skb) {
1177                 offset = seq - TCP_SKB_CB(skb)->seq;
1178                 if (tcp_hdr(skb)->syn)
1179                         offset--;
1180                 if (offset < skb->len || tcp_hdr(skb)->fin) {
1181                         *off = offset;
1182                         return skb;
1183                 }
1184         }
1185         return NULL;
1186 }
1187
1188 /*
1189  * This routine provides an alternative to tcp_recvmsg() for routines
1190  * that would like to handle copying from skbuffs directly in 'sendfile'
1191  * fashion.
1192  * Note:
1193  *      - It is assumed that the socket was locked by the caller.
1194  *      - The routine does not block.
1195  *      - At present, there is no support for reading OOB data
1196  *        or for 'peeking' the socket using this routine
1197  *        (although both would be easy to implement).
1198  */
1199 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
1200                   sk_read_actor_t recv_actor)
1201 {
1202         struct sk_buff *skb;
1203         struct tcp_sock *tp = tcp_sk(sk);
1204         u32 seq = tp->copied_seq;
1205         u32 offset;
1206         int copied = 0;
1207
1208         if (sk->sk_state == TCP_LISTEN)
1209                 return -ENOTCONN;
1210         while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1211                 if (offset < skb->len) {
1212                         int used;
1213                         size_t len;
1214
1215                         len = skb->len - offset;
1216                         /* Stop reading if we hit a patch of urgent data */
1217                         if (tp->urg_data) {
1218                                 u32 urg_offset = tp->urg_seq - seq;
1219                                 if (urg_offset < len)
1220                                         len = urg_offset;
1221                                 if (!len)
1222                                         break;
1223                         }
1224                         used = recv_actor(desc, skb, offset, len);
1225                         if (used < 0) {
1226                                 if (!copied)
1227                                         copied = used;
1228                                 break;
1229                         } else if (used <= len) {
1230                                 seq += used;
1231                                 copied += used;
1232                                 offset += used;
1233                         }
1234                         /*
1235                          * If recv_actor drops the lock (e.g. TCP splice
1236                          * receive) the skb pointer might be invalid when
1237                          * getting here: tcp_collapse might have deleted it
1238                          * while aggregating skbs from the socket queue.
1239                          */
1240                         skb = tcp_recv_skb(sk, seq-1, &offset);
1241                         if (!skb || (offset+1 != skb->len))
1242                                 break;
1243                 }
1244                 if (tcp_hdr(skb)->fin) {
1245                         sk_eat_skb(sk, skb, 0);
1246                         ++seq;
1247                         break;
1248                 }
1249                 sk_eat_skb(sk, skb, 0);
1250                 if (!desc->count)
1251                         break;
1252         }
1253         tp->copied_seq = seq;
1254
1255         tcp_rcv_space_adjust(sk);
1256
1257         /* Clean up data we have read: This will do ACK frames. */
1258         if (copied > 0)
1259                 tcp_cleanup_rbuf(sk, copied);
1260         return copied;
1261 }
1262
1263 /*
1264  *      This routine copies from a sock struct into the user buffer.
1265  *
1266  *      Technical note: in 2.3 we work on _locked_ socket, so that
1267  *      tricks with *seq access order and skb->users are not required.
1268  *      Probably, code can be easily improved even more.
1269  */
1270
1271 int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
1272                 size_t len, int nonblock, int flags, int *addr_len)
1273 {
1274         struct tcp_sock *tp = tcp_sk(sk);
1275         int copied = 0;
1276         u32 peek_seq;
1277         u32 *seq;
1278         unsigned long used;
1279         int err;
1280         int target;             /* Read at least this many bytes */
1281         long timeo;
1282         struct task_struct *user_recv = NULL;
1283         int copied_early = 0;
1284         struct sk_buff *skb;
1285
1286         lock_sock(sk);
1287
1288         TCP_CHECK_TIMER(sk);
1289
1290         err = -ENOTCONN;
1291         if (sk->sk_state == TCP_LISTEN)
1292                 goto out;
1293
1294         timeo = sock_rcvtimeo(sk, nonblock);
1295
1296         /* Urgent data needs to be handled specially. */
1297         if (flags & MSG_OOB)
1298                 goto recv_urg;
1299
1300         seq = &tp->copied_seq;
1301         if (flags & MSG_PEEK) {
1302                 peek_seq = tp->copied_seq;
1303                 seq = &peek_seq;
1304         }
1305
1306         target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
1307
1308 #ifdef CONFIG_NET_DMA
1309         tp->ucopy.dma_chan = NULL;
1310         preempt_disable();
1311         skb = skb_peek_tail(&sk->sk_receive_queue);
1312         {
1313                 int available = 0;
1314
1315                 if (skb)
1316                         available = TCP_SKB_CB(skb)->seq + skb->len - (*seq);
1317                 if ((available < target) &&
1318                     (len > sysctl_tcp_dma_copybreak) && !(flags & MSG_PEEK) &&
1319                     !sysctl_tcp_low_latency &&
1320                     __get_cpu_var(softnet_data).net_dma) {
1321                         preempt_enable_no_resched();
1322                         tp->ucopy.pinned_list =
1323                                         dma_pin_iovec_pages(msg->msg_iov, len);
1324                 } else {
1325                         preempt_enable_no_resched();
1326                 }
1327         }
1328 #endif
1329
1330         do {
1331                 u32 offset;
1332
1333                 /* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */
1334                 if (tp->urg_data && tp->urg_seq == *seq) {
1335                         if (copied)
1336                                 break;
1337                         if (signal_pending(current)) {
1338                                 copied = timeo ? sock_intr_errno(timeo) : -EAGAIN;
1339                                 break;
1340                         }
1341                 }
1342
1343                 /* Next get a buffer. */
1344
1345                 skb = skb_peek(&sk->sk_receive_queue);
1346                 do {
1347                         if (!skb)
1348                                 break;
1349
1350                         /* Now that we have two receive queues this
1351                          * shouldn't happen.
1352                          */
1353                         if (before(*seq, TCP_SKB_CB(skb)->seq)) {
1354                                 printk(KERN_INFO "recvmsg bug: copied %X "
1355                                        "seq %X\n", *seq, TCP_SKB_CB(skb)->seq);
1356                                 break;
1357                         }
1358                         offset = *seq - TCP_SKB_CB(skb)->seq;
1359                         if (tcp_hdr(skb)->syn)
1360                                 offset--;
1361                         if (offset < skb->len)
1362                                 goto found_ok_skb;
1363                         if (tcp_hdr(skb)->fin)
1364                                 goto found_fin_ok;
1365                         BUG_TRAP(flags & MSG_PEEK);
1366                         skb = skb->next;
1367                 } while (skb != (struct sk_buff *)&sk->sk_receive_queue);
1368
1369                 /* Well, if we have backlog, try to process it now yet. */
1370
1371                 if (copied >= target && !sk->sk_backlog.tail)
1372                         break;
1373
1374                 if (copied) {
1375                         if (sk->sk_err ||
1376                             sk->sk_state == TCP_CLOSE ||
1377                             (sk->sk_shutdown & RCV_SHUTDOWN) ||
1378                             !timeo ||
1379                             signal_pending(current) ||
1380                             (flags & MSG_PEEK))
1381                                 break;
1382                 } else {
1383                         if (sock_flag(sk, SOCK_DONE))
1384                                 break;
1385
1386                         if (sk->sk_err) {
1387                                 copied = sock_error(sk);
1388                                 break;
1389                         }
1390
1391                         if (sk->sk_shutdown & RCV_SHUTDOWN)
1392                                 break;
1393
1394                         if (sk->sk_state == TCP_CLOSE) {
1395                                 if (!sock_flag(sk, SOCK_DONE)) {
1396                                         /* This occurs when user tries to read
1397                                          * from never connected socket.
1398                                          */
1399                                         copied = -ENOTCONN;
1400                                         break;
1401                                 }
1402                                 break;
1403                         }
1404
1405                         if (!timeo) {
1406                                 copied = -EAGAIN;
1407                                 break;
1408                         }
1409
1410                         if (signal_pending(current)) {
1411                                 copied = sock_intr_errno(timeo);
1412                                 break;
1413                         }
1414                 }
1415
1416                 tcp_cleanup_rbuf(sk, copied);
1417
1418                 if (!sysctl_tcp_low_latency && tp->ucopy.task == user_recv) {
1419                         /* Install new reader */
1420                         if (!user_recv && !(flags & (MSG_TRUNC | MSG_PEEK))) {
1421                                 user_recv = current;
1422                                 tp->ucopy.task = user_recv;
1423                                 tp->ucopy.iov = msg->msg_iov;
1424                         }
1425
1426                         tp->ucopy.len = len;
1427
1428                         BUG_TRAP(tp->copied_seq == tp->rcv_nxt ||
1429                                  (flags & (MSG_PEEK | MSG_TRUNC)));
1430
1431                         /* Ugly... If prequeue is not empty, we have to
1432                          * process it before releasing socket, otherwise
1433                          * order will be broken at second iteration.
1434                          * More elegant solution is required!!!
1435                          *
1436                          * Look: we have the following (pseudo)queues:
1437                          *
1438                          * 1. packets in flight
1439                          * 2. backlog
1440                          * 3. prequeue
1441                          * 4. receive_queue
1442                          *
1443                          * Each queue can be processed only if the next ones
1444                          * are empty. At this point we have empty receive_queue.
1445                          * But prequeue _can_ be not empty after 2nd iteration,
1446                          * when we jumped to start of loop because backlog
1447                          * processing added something to receive_queue.
1448                          * We cannot release_sock(), because backlog contains
1449                          * packets arrived _after_ prequeued ones.
1450                          *
1451                          * Shortly, algorithm is clear --- to process all
1452                          * the queues in order. We could make it more directly,
1453                          * requeueing packets from backlog to prequeue, if
1454                          * is not empty. It is more elegant, but eats cycles,
1455                          * unfortunately.
1456                          */
1457                         if (!skb_queue_empty(&tp->ucopy.prequeue))
1458                                 goto do_prequeue;
1459
1460                         /* __ Set realtime policy in scheduler __ */
1461                 }
1462
1463                 if (copied >= target) {
1464                         /* Do not sleep, just process backlog. */
1465                         release_sock(sk);
1466                         lock_sock(sk);
1467                 } else
1468                         sk_wait_data(sk, &timeo);
1469
1470 #ifdef CONFIG_NET_DMA
1471                 tp->ucopy.wakeup = 0;
1472 #endif
1473
1474                 if (user_recv) {
1475                         int chunk;
1476
1477                         /* __ Restore normal policy in scheduler __ */
1478
1479                         if ((chunk = len - tp->ucopy.len) != 0) {
1480                                 NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMBACKLOG, chunk);
1481                                 len -= chunk;
1482                                 copied += chunk;
1483                         }
1484
1485                         if (tp->rcv_nxt == tp->copied_seq &&
1486                             !skb_queue_empty(&tp->ucopy.prequeue)) {
1487 do_prequeue:
1488                                 tcp_prequeue_process(sk);
1489
1490                                 if ((chunk = len - tp->ucopy.len) != 0) {
1491                                         NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk);
1492                                         len -= chunk;
1493                                         copied += chunk;
1494                                 }
1495                         }
1496                 }
1497                 if ((flags & MSG_PEEK) && peek_seq != tp->copied_seq) {
1498                         if (net_ratelimit())
1499                                 printk(KERN_DEBUG "TCP(%s:%d): Application bug, race in MSG_PEEK.\n",
1500                                        current->comm, task_pid_nr(current));
1501                         peek_seq = tp->copied_seq;
1502                 }
1503                 continue;
1504
1505         found_ok_skb:
1506                 /* Ok so how much can we use? */
1507                 used = skb->len - offset;
1508                 if (len < used)
1509                         used = len;
1510
1511                 /* Do we have urgent data here? */
1512                 if (tp->urg_data) {
1513                         u32 urg_offset = tp->urg_seq - *seq;
1514                         if (urg_offset < used) {
1515                                 if (!urg_offset) {
1516                                         if (!sock_flag(sk, SOCK_URGINLINE)) {
1517                                                 ++*seq;
1518                                                 offset++;
1519                                                 used--;
1520                                                 if (!used)
1521                                                         goto skip_copy;
1522                                         }
1523                                 } else
1524                                         used = urg_offset;
1525                         }
1526                 }
1527
1528                 if (!(flags & MSG_TRUNC)) {
1529 #ifdef CONFIG_NET_DMA
1530                         if (!tp->ucopy.dma_chan && tp->ucopy.pinned_list)
1531                                 tp->ucopy.dma_chan = get_softnet_dma();
1532
1533                         if (tp->ucopy.dma_chan) {
1534                                 tp->ucopy.dma_cookie = dma_skb_copy_datagram_iovec(
1535                                         tp->ucopy.dma_chan, skb, offset,
1536                                         msg->msg_iov, used,
1537                                         tp->ucopy.pinned_list);
1538
1539                                 if (tp->ucopy.dma_cookie < 0) {
1540
1541                                         printk(KERN_ALERT "dma_cookie < 0\n");
1542
1543                                         /* Exception. Bailout! */
1544                                         if (!copied)
1545                                                 copied = -EFAULT;
1546                                         break;
1547                                 }
1548                                 if ((offset + used) == skb->len)
1549                                         copied_early = 1;
1550
1551                         } else
1552 #endif
1553                         {
1554                                 err = skb_copy_datagram_iovec(skb, offset,
1555                                                 msg->msg_iov, used);
1556                                 if (err) {
1557                                         /* Exception. Bailout! */
1558                                         if (!copied)
1559                                                 copied = -EFAULT;
1560                                         break;
1561                                 }
1562                         }
1563                 }
1564
1565                 *seq += used;
1566                 copied += used;
1567                 len -= used;
1568
1569                 tcp_rcv_space_adjust(sk);
1570
1571 skip_copy:
1572                 if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) {
1573                         tp->urg_data = 0;
1574                         tcp_fast_path_check(sk);
1575                 }
1576                 if (used + offset < skb->len)
1577                         continue;
1578
1579                 if (tcp_hdr(skb)->fin)
1580                         goto found_fin_ok;
1581                 if (!(flags & MSG_PEEK)) {
1582                         sk_eat_skb(sk, skb, copied_early);
1583                         copied_early = 0;
1584                 }
1585                 continue;
1586
1587         found_fin_ok:
1588                 /* Process the FIN. */
1589                 ++*seq;
1590                 if (!(flags & MSG_PEEK)) {
1591                         sk_eat_skb(sk, skb, copied_early);
1592                         copied_early = 0;
1593                 }
1594                 break;
1595         } while (len > 0);
1596
1597         if (user_recv) {
1598                 if (!skb_queue_empty(&tp->ucopy.prequeue)) {
1599                         int chunk;
1600
1601                         tp->ucopy.len = copied > 0 ? len : 0;
1602
1603                         tcp_prequeue_process(sk);
1604
1605                         if (copied > 0 && (chunk = len - tp->ucopy.len) != 0) {
1606                                 NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk);
1607                                 len -= chunk;
1608                                 copied += chunk;
1609                         }
1610                 }
1611
1612                 tp->ucopy.task = NULL;
1613                 tp->ucopy.len = 0;
1614         }
1615
1616 #ifdef CONFIG_NET_DMA
1617         if (tp->ucopy.dma_chan) {
1618                 dma_cookie_t done, used;
1619
1620                 dma_async_memcpy_issue_pending(tp->ucopy.dma_chan);
1621
1622                 while (dma_async_memcpy_complete(tp->ucopy.dma_chan,
1623                                                  tp->ucopy.dma_cookie, &done,
1624                                                  &used) == DMA_IN_PROGRESS) {
1625                         /* do partial cleanup of sk_async_wait_queue */
1626                         while ((skb = skb_peek(&sk->sk_async_wait_queue)) &&
1627                                (dma_async_is_complete(skb->dma_cookie, done,
1628                                                       used) == DMA_SUCCESS)) {
1629                                 __skb_dequeue(&sk->sk_async_wait_queue);
1630                                 kfree_skb(skb);
1631                         }
1632                 }
1633
1634                 /* Safe to free early-copied skbs now */
1635                 __skb_queue_purge(&sk->sk_async_wait_queue);
1636                 dma_chan_put(tp->ucopy.dma_chan);
1637                 tp->ucopy.dma_chan = NULL;
1638         }
1639         if (tp->ucopy.pinned_list) {
1640                 dma_unpin_iovec_pages(tp->ucopy.pinned_list);
1641                 tp->ucopy.pinned_list = NULL;
1642         }
1643 #endif
1644
1645         /* According to UNIX98, msg_name/msg_namelen are ignored
1646          * on connected socket. I was just happy when found this 8) --ANK
1647          */
1648
1649         /* Clean up data we have read: This will do ACK frames. */
1650         tcp_cleanup_rbuf(sk, copied);
1651
1652         TCP_CHECK_TIMER(sk);
1653         release_sock(sk);
1654         return copied;
1655
1656 out:
1657         TCP_CHECK_TIMER(sk);
1658         release_sock(sk);
1659         return err;
1660
1661 recv_urg:
1662         err = tcp_recv_urg(sk, timeo, msg, len, flags, addr_len);
1663         goto out;
1664 }
1665
1666 void tcp_set_state(struct sock *sk, int state)
1667 {
1668         int oldstate = sk->sk_state;
1669
1670         switch (state) {
1671         case TCP_ESTABLISHED:
1672                 if (oldstate != TCP_ESTABLISHED)
1673                         TCP_INC_STATS(TCP_MIB_CURRESTAB);
1674                 break;
1675
1676         case TCP_CLOSE:
1677                 if (oldstate == TCP_CLOSE_WAIT || oldstate == TCP_ESTABLISHED)
1678                         TCP_INC_STATS(TCP_MIB_ESTABRESETS);
1679
1680                 sk->sk_prot->unhash(sk);
1681                 if (inet_csk(sk)->icsk_bind_hash &&
1682                     !(sk->sk_userlocks & SOCK_BINDPORT_LOCK))
1683                         inet_put_port(sk);
1684                 /* fall through */
1685         default:
1686                 if (oldstate==TCP_ESTABLISHED)
1687                         TCP_DEC_STATS(TCP_MIB_CURRESTAB);
1688         }
1689
1690         /* Change state AFTER socket is unhashed to avoid closed
1691          * socket sitting in hash tables.
1692          */
1693         sk->sk_state = state;
1694
1695 #ifdef STATE_TRACE
1696         SOCK_DEBUG(sk, "TCP sk=%p, State %s -> %s\n",sk, statename[oldstate],statename[state]);
1697 #endif
1698 }
1699 EXPORT_SYMBOL_GPL(tcp_set_state);
1700
1701 /*
1702  *      State processing on a close. This implements the state shift for
1703  *      sending our FIN frame. Note that we only send a FIN for some
1704  *      states. A shutdown() may have already sent the FIN, or we may be
1705  *      closed.
1706  */
1707
1708 static const unsigned char new_state[16] = {
1709   /* current state:        new state:      action:      */
1710   /* (Invalid)          */ TCP_CLOSE,
1711   /* TCP_ESTABLISHED    */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
1712   /* TCP_SYN_SENT       */ TCP_CLOSE,
1713   /* TCP_SYN_RECV       */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
1714   /* TCP_FIN_WAIT1      */ TCP_FIN_WAIT1,
1715   /* TCP_FIN_WAIT2      */ TCP_FIN_WAIT2,
1716   /* TCP_TIME_WAIT      */ TCP_CLOSE,
1717   /* TCP_CLOSE          */ TCP_CLOSE,
1718   /* TCP_CLOSE_WAIT     */ TCP_LAST_ACK  | TCP_ACTION_FIN,
1719   /* TCP_LAST_ACK       */ TCP_LAST_ACK,
1720   /* TCP_LISTEN         */ TCP_CLOSE,
1721   /* TCP_CLOSING        */ TCP_CLOSING,
1722 };
1723
1724 static int tcp_close_state(struct sock *sk)
1725 {
1726         int next = (int)new_state[sk->sk_state];
1727         int ns = next & TCP_STATE_MASK;
1728
1729         tcp_set_state(sk, ns);
1730
1731         return next & TCP_ACTION_FIN;
1732 }
1733
1734 /*
1735  *      Shutdown the sending side of a connection. Much like close except
1736  *      that we don't receive shut down or sock_set_flag(sk, SOCK_DEAD).
1737  */
1738
1739 void tcp_shutdown(struct sock *sk, int how)
1740 {
1741         /*      We need to grab some memory, and put together a FIN,
1742          *      and then put it into the queue to be sent.
1743          *              Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92.
1744          */
1745         if (!(how & SEND_SHUTDOWN))
1746                 return;
1747
1748         /* If we've already sent a FIN, or it's a closed state, skip this. */
1749         if ((1 << sk->sk_state) &
1750             (TCPF_ESTABLISHED | TCPF_SYN_SENT |
1751              TCPF_SYN_RECV | TCPF_CLOSE_WAIT)) {
1752                 /* Clear out any half completed packets.  FIN if needed. */
1753                 if (tcp_close_state(sk))
1754                         tcp_send_fin(sk);
1755         }
1756 }
1757
1758 void tcp_close(struct sock *sk, long timeout)
1759 {
1760         struct sk_buff *skb;
1761         int data_was_unread = 0;
1762         int state;
1763
1764         lock_sock(sk);
1765         sk->sk_shutdown = SHUTDOWN_MASK;
1766
1767         if (sk->sk_state == TCP_LISTEN) {
1768                 tcp_set_state(sk, TCP_CLOSE);
1769
1770                 /* Special case. */
1771                 inet_csk_listen_stop(sk);
1772
1773                 goto adjudge_to_death;
1774         }
1775
1776         /*  We need to flush the recv. buffs.  We do this only on the
1777          *  descriptor close, not protocol-sourced closes, because the
1778          *  reader process may not have drained the data yet!
1779          */
1780         while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
1781                 u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq -
1782                           tcp_hdr(skb)->fin;
1783                 data_was_unread += len;
1784                 __kfree_skb(skb);
1785         }
1786
1787         sk_mem_reclaim(sk);
1788
1789         /* As outlined in RFC 2525, section 2.17, we send a RST here because
1790          * data was lost. To witness the awful effects of the old behavior of
1791          * always doing a FIN, run an older 2.1.x kernel or 2.0.x, start a bulk
1792          * GET in an FTP client, suspend the process, wait for the client to
1793          * advertise a zero window, then kill -9 the FTP client, wheee...
1794          * Note: timeout is always zero in such a case.
1795          */
1796         if (data_was_unread) {
1797                 /* Unread data was tossed, zap the connection. */
1798                 NET_INC_STATS_USER(LINUX_MIB_TCPABORTONCLOSE);
1799                 tcp_set_state(sk, TCP_CLOSE);
1800                 tcp_send_active_reset(sk, GFP_KERNEL);
1801         } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
1802                 /* Check zero linger _after_ checking for unread data. */
1803                 sk->sk_prot->disconnect(sk, 0);
1804                 NET_INC_STATS_USER(LINUX_MIB_TCPABORTONDATA);
1805         } else if (tcp_close_state(sk)) {
1806                 /* We FIN if the application ate all the data before
1807                  * zapping the connection.
1808                  */
1809
1810                 /* RED-PEN. Formally speaking, we have broken TCP state
1811                  * machine. State transitions:
1812                  *
1813                  * TCP_ESTABLISHED -> TCP_FIN_WAIT1
1814                  * TCP_SYN_RECV -> TCP_FIN_WAIT1 (forget it, it's impossible)
1815                  * TCP_CLOSE_WAIT -> TCP_LAST_ACK
1816                  *
1817                  * are legal only when FIN has been sent (i.e. in window),
1818                  * rather than queued out of window. Purists blame.
1819                  *
1820                  * F.e. "RFC state" is ESTABLISHED,
1821                  * if Linux state is FIN-WAIT-1, but FIN is still not sent.
1822                  *
1823                  * The visible declinations are that sometimes
1824                  * we enter time-wait state, when it is not required really
1825                  * (harmless), do not send active resets, when they are
1826                  * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when
1827                  * they look as CLOSING or LAST_ACK for Linux)
1828                  * Probably, I missed some more holelets.
1829                  *                                              --ANK
1830                  */
1831                 tcp_send_fin(sk);
1832         }
1833
1834         sk_stream_wait_close(sk, timeout);
1835
1836 adjudge_to_death:
1837         state = sk->sk_state;
1838         sock_hold(sk);
1839         sock_orphan(sk);
1840         atomic_inc(sk->sk_prot->orphan_count);
1841
1842         /* It is the last release_sock in its life. It will remove backlog. */
1843         release_sock(sk);
1844
1845
1846         /* Now socket is owned by kernel and we acquire BH lock
1847            to finish close. No need to check for user refs.
1848          */
1849         local_bh_disable();
1850         bh_lock_sock(sk);
1851         BUG_TRAP(!sock_owned_by_user(sk));
1852
1853         /* Have we already been destroyed by a softirq or backlog? */
1854         if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
1855                 goto out;
1856
1857         /*      This is a (useful) BSD violating of the RFC. There is a
1858          *      problem with TCP as specified in that the other end could
1859          *      keep a socket open forever with no application left this end.
1860          *      We use a 3 minute timeout (about the same as BSD) then kill
1861          *      our end. If they send after that then tough - BUT: long enough
1862          *      that we won't make the old 4*rto = almost no time - whoops
1863          *      reset mistake.
1864          *
1865          *      Nope, it was not mistake. It is really desired behaviour
1866          *      f.e. on http servers, when such sockets are useless, but
1867          *      consume significant resources. Let's do it with special
1868          *      linger2 option.                                 --ANK
1869          */
1870
1871         if (sk->sk_state == TCP_FIN_WAIT2) {
1872                 struct tcp_sock *tp = tcp_sk(sk);
1873                 if (tp->linger2 < 0) {
1874                         tcp_set_state(sk, TCP_CLOSE);
1875                         tcp_send_active_reset(sk, GFP_ATOMIC);
1876                         NET_INC_STATS_BH(LINUX_MIB_TCPABORTONLINGER);
1877                 } else {
1878                         const int tmo = tcp_fin_time(sk);
1879
1880                         if (tmo > TCP_TIMEWAIT_LEN) {
1881                                 inet_csk_reset_keepalive_timer(sk,
1882                                                 tmo - TCP_TIMEWAIT_LEN);
1883                         } else {
1884                                 tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
1885                                 goto out;
1886                         }
1887                 }
1888         }
1889         if (sk->sk_state != TCP_CLOSE) {
1890                 sk_mem_reclaim(sk);
1891                 if (tcp_too_many_orphans(sk,
1892                                 atomic_read(sk->sk_prot->orphan_count))) {
1893                         if (net_ratelimit())
1894                                 printk(KERN_INFO "TCP: too many of orphaned "
1895                                        "sockets\n");
1896                         tcp_set_state(sk, TCP_CLOSE);
1897                         tcp_send_active_reset(sk, GFP_ATOMIC);
1898                         NET_INC_STATS_BH(LINUX_MIB_TCPABORTONMEMORY);
1899                 }
1900         }
1901
1902         if (sk->sk_state == TCP_CLOSE)
1903                 inet_csk_destroy_sock(sk);
1904         /* Otherwise, socket is reprieved until protocol close. */
1905
1906 out:
1907         bh_unlock_sock(sk);
1908         local_bh_enable();
1909         sock_put(sk);
1910 }
1911
1912 /* These states need RST on ABORT according to RFC793 */
1913
1914 static inline int tcp_need_reset(int state)
1915 {
1916         return (1 << state) &
1917                (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 |
1918                 TCPF_FIN_WAIT2 | TCPF_SYN_RECV);
1919 }
1920
1921 int tcp_disconnect(struct sock *sk, int flags)
1922 {
1923         struct inet_sock *inet = inet_sk(sk);
1924         struct inet_connection_sock *icsk = inet_csk(sk);
1925         struct tcp_sock *tp = tcp_sk(sk);
1926         int err = 0;
1927         int old_state = sk->sk_state;
1928
1929         if (old_state != TCP_CLOSE)
1930                 tcp_set_state(sk, TCP_CLOSE);
1931
1932         /* ABORT function of RFC793 */
1933         if (old_state == TCP_LISTEN) {
1934                 inet_csk_listen_stop(sk);
1935         } else if (tcp_need_reset(old_state) ||
1936                    (tp->snd_nxt != tp->write_seq &&
1937                     (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK))) {
1938                 /* The last check adjusts for discrepancy of Linux wrt. RFC
1939                  * states
1940                  */
1941                 tcp_send_active_reset(sk, gfp_any());
1942                 sk->sk_err = ECONNRESET;
1943         } else if (old_state == TCP_SYN_SENT)
1944                 sk->sk_err = ECONNRESET;
1945
1946         tcp_clear_xmit_timers(sk);
1947         __skb_queue_purge(&sk->sk_receive_queue);
1948         tcp_write_queue_purge(sk);
1949         __skb_queue_purge(&tp->out_of_order_queue);
1950 #ifdef CONFIG_NET_DMA
1951         __skb_queue_purge(&sk->sk_async_wait_queue);
1952 #endif
1953
1954         inet->dport = 0;
1955
1956         if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK))
1957                 inet_reset_saddr(sk);
1958
1959         sk->sk_shutdown = 0;
1960         sock_reset_flag(sk, SOCK_DONE);
1961         tp->srtt = 0;
1962         if ((tp->write_seq += tp->max_window + 2) == 0)
1963                 tp->write_seq = 1;
1964         icsk->icsk_backoff = 0;
1965         tp->snd_cwnd = 2;
1966         icsk->icsk_probes_out = 0;
1967         tp->packets_out = 0;
1968         tp->snd_ssthresh = 0x7fffffff;
1969         tp->snd_cwnd_cnt = 0;
1970         tp->bytes_acked = 0;
1971         tcp_set_ca_state(sk, TCP_CA_Open);
1972         tcp_clear_retrans(tp);
1973         inet_csk_delack_init(sk);
1974         tcp_init_send_head(sk);
1975         memset(&tp->rx_opt, 0, sizeof(tp->rx_opt));
1976         __sk_dst_reset(sk);
1977
1978         BUG_TRAP(!inet->num || icsk->icsk_bind_hash);
1979
1980         sk->sk_error_report(sk);
1981         return err;
1982 }
1983
1984 /*
1985  *      Socket option code for TCP.
1986  */
1987 static int do_tcp_setsockopt(struct sock *sk, int level,
1988                 int optname, char __user *optval, int optlen)
1989 {
1990         struct tcp_sock *tp = tcp_sk(sk);
1991         struct inet_connection_sock *icsk = inet_csk(sk);
1992         int val;
1993         int err = 0;
1994
1995         /* This is a string value all the others are int's */
1996         if (optname == TCP_CONGESTION) {
1997                 char name[TCP_CA_NAME_MAX];
1998
1999                 if (optlen < 1)
2000                         return -EINVAL;
2001
2002                 val = strncpy_from_user(name, optval,
2003                                         min(TCP_CA_NAME_MAX-1, optlen));
2004                 if (val < 0)
2005                         return -EFAULT;
2006                 name[val] = 0;
2007
2008                 lock_sock(sk);
2009                 err = tcp_set_congestion_control(sk, name);
2010                 release_sock(sk);
2011                 return err;
2012         }
2013
2014         if (optlen < sizeof(int))
2015                 return -EINVAL;
2016
2017         if (get_user(val, (int __user *)optval))
2018                 return -EFAULT;
2019
2020         lock_sock(sk);
2021
2022         switch (optname) {
2023         case TCP_MAXSEG:
2024                 /* Values greater than interface MTU won't take effect. However
2025                  * at the point when this call is done we typically don't yet
2026                  * know which interface is going to be used */
2027                 if (val < 8 || val > MAX_TCP_WINDOW) {
2028                         err = -EINVAL;
2029                         break;
2030                 }
2031                 tp->rx_opt.user_mss = val;
2032                 break;
2033
2034         case TCP_NODELAY:
2035                 if (val) {
2036                         /* TCP_NODELAY is weaker than TCP_CORK, so that
2037                          * this option on corked socket is remembered, but
2038                          * it is not activated until cork is cleared.
2039                          *
2040                          * However, when TCP_NODELAY is set we make
2041                          * an explicit push, which overrides even TCP_CORK
2042                          * for currently queued segments.
2043                          */
2044                         tp->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH;
2045                         tcp_push_pending_frames(sk);
2046                 } else {
2047                         tp->nonagle &= ~TCP_NAGLE_OFF;
2048                 }
2049                 break;
2050
2051         case TCP_CORK:
2052                 /* When set indicates to always queue non-full frames.
2053                  * Later the user clears this option and we transmit
2054                  * any pending partial frames in the queue.  This is
2055                  * meant to be used alongside sendfile() to get properly
2056                  * filled frames when the user (for example) must write
2057                  * out headers with a write() call first and then use
2058                  * sendfile to send out the data parts.
2059                  *
2060                  * TCP_CORK can be set together with TCP_NODELAY and it is
2061                  * stronger than TCP_NODELAY.
2062                  */
2063                 if (val) {
2064                         tp->nonagle |= TCP_NAGLE_CORK;
2065                 } else {
2066                         tp->nonagle &= ~TCP_NAGLE_CORK;
2067                         if (tp->nonagle&TCP_NAGLE_OFF)
2068                                 tp->nonagle |= TCP_NAGLE_PUSH;
2069                         tcp_push_pending_frames(sk);
2070                 }
2071                 break;
2072
2073         case TCP_KEEPIDLE:
2074                 if (val < 1 || val > MAX_TCP_KEEPIDLE)
2075                         err = -EINVAL;
2076                 else {
2077                         tp->keepalive_time = val * HZ;
2078                         if (sock_flag(sk, SOCK_KEEPOPEN) &&
2079                             !((1 << sk->sk_state) &
2080                               (TCPF_CLOSE | TCPF_LISTEN))) {
2081                                 __u32 elapsed = tcp_time_stamp - tp->rcv_tstamp;
2082                                 if (tp->keepalive_time > elapsed)
2083                                         elapsed = tp->keepalive_time - elapsed;
2084                                 else
2085                                         elapsed = 0;
2086                                 inet_csk_reset_keepalive_timer(sk, elapsed);
2087                         }
2088                 }
2089                 break;
2090         case TCP_KEEPINTVL:
2091                 if (val < 1 || val > MAX_TCP_KEEPINTVL)
2092                         err = -EINVAL;
2093                 else
2094                         tp->keepalive_intvl = val * HZ;
2095                 break;
2096         case TCP_KEEPCNT:
2097                 if (val < 1 || val > MAX_TCP_KEEPCNT)
2098                         err = -EINVAL;
2099                 else
2100                         tp->keepalive_probes = val;
2101                 break;
2102         case TCP_SYNCNT:
2103                 if (val < 1 || val > MAX_TCP_SYNCNT)
2104                         err = -EINVAL;
2105                 else
2106                         icsk->icsk_syn_retries = val;
2107                 break;
2108
2109         case TCP_LINGER2:
2110                 if (val < 0)
2111                         tp->linger2 = -1;
2112                 else if (val > sysctl_tcp_fin_timeout / HZ)
2113                         tp->linger2 = 0;
2114                 else
2115                         tp->linger2 = val * HZ;
2116                 break;
2117
2118         case TCP_DEFER_ACCEPT:
2119                 icsk->icsk_accept_queue.rskq_defer_accept = 0;
2120                 if (val > 0) {
2121                         /* Translate value in seconds to number of
2122                          * retransmits */
2123                         while (icsk->icsk_accept_queue.rskq_defer_accept < 32 &&
2124                                val > ((TCP_TIMEOUT_INIT / HZ) <<
2125                                        icsk->icsk_accept_queue.rskq_defer_accept))
2126                                 icsk->icsk_accept_queue.rskq_defer_accept++;
2127                         icsk->icsk_accept_queue.rskq_defer_accept++;
2128                 }
2129                 break;
2130
2131         case TCP_WINDOW_CLAMP:
2132                 if (!val) {
2133                         if (sk->sk_state != TCP_CLOSE) {
2134                                 err = -EINVAL;
2135                                 break;
2136                         }
2137                         tp->window_clamp = 0;
2138                 } else
2139                         tp->window_clamp = val < SOCK_MIN_RCVBUF / 2 ?
2140                                                 SOCK_MIN_RCVBUF / 2 : val;
2141                 break;
2142
2143         case TCP_QUICKACK:
2144                 if (!val) {
2145                         icsk->icsk_ack.pingpong = 1;
2146                 } else {
2147                         icsk->icsk_ack.pingpong = 0;
2148                         if ((1 << sk->sk_state) &
2149                             (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) &&
2150                             inet_csk_ack_scheduled(sk)) {
2151                                 icsk->icsk_ack.pending |= ICSK_ACK_PUSHED;
2152                                 tcp_cleanup_rbuf(sk, 1);
2153                                 if (!(val & 1))
2154                                         icsk->icsk_ack.pingpong = 1;
2155                         }
2156                 }
2157                 break;
2158
2159 #ifdef CONFIG_TCP_MD5SIG
2160         case TCP_MD5SIG:
2161                 /* Read the IP->Key mappings from userspace */
2162                 err = tp->af_specific->md5_parse(sk, optval, optlen);
2163                 break;
2164 #endif
2165
2166         default:
2167                 err = -ENOPROTOOPT;
2168                 break;
2169         }
2170
2171         release_sock(sk);
2172         return err;
2173 }
2174
2175 int tcp_setsockopt(struct sock *sk, int level, int optname, char __user *optval,
2176                    int optlen)
2177 {
2178         struct inet_connection_sock *icsk = inet_csk(sk);
2179
2180         if (level != SOL_TCP)
2181                 return icsk->icsk_af_ops->setsockopt(sk, level, optname,
2182                                                      optval, optlen);
2183         return do_tcp_setsockopt(sk, level, optname, optval, optlen);
2184 }
2185
2186 #ifdef CONFIG_COMPAT
2187 int compat_tcp_setsockopt(struct sock *sk, int level, int optname,
2188                           char __user *optval, int optlen)
2189 {
2190         if (level != SOL_TCP)
2191                 return inet_csk_compat_setsockopt(sk, level, optname,
2192                                                   optval, optlen);
2193         return do_tcp_setsockopt(sk, level, optname, optval, optlen);
2194 }
2195
2196 EXPORT_SYMBOL(compat_tcp_setsockopt);
2197 #endif
2198
2199 /* Return information about state of tcp endpoint in API format. */
2200 void tcp_get_info(struct sock *sk, struct tcp_info *info)
2201 {
2202         struct tcp_sock *tp = tcp_sk(sk);
2203         const struct inet_connection_sock *icsk = inet_csk(sk);
2204         u32 now = tcp_time_stamp;
2205
2206         memset(info, 0, sizeof(*info));
2207
2208         info->tcpi_state = sk->sk_state;
2209         info->tcpi_ca_state = icsk->icsk_ca_state;
2210         info->tcpi_retransmits = icsk->icsk_retransmits;
2211         info->tcpi_probes = icsk->icsk_probes_out;
2212         info->tcpi_backoff = icsk->icsk_backoff;
2213
2214         if (tp->rx_opt.tstamp_ok)
2215                 info->tcpi_options |= TCPI_OPT_TIMESTAMPS;
2216         if (tcp_is_sack(tp))
2217                 info->tcpi_options |= TCPI_OPT_SACK;
2218         if (tp->rx_opt.wscale_ok) {
2219                 info->tcpi_options |= TCPI_OPT_WSCALE;
2220                 info->tcpi_snd_wscale = tp->rx_opt.snd_wscale;
2221                 info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale;
2222         }
2223
2224         if (tp->ecn_flags&TCP_ECN_OK)
2225                 info->tcpi_options |= TCPI_OPT_ECN;
2226
2227         info->tcpi_rto = jiffies_to_usecs(icsk->icsk_rto);
2228         info->tcpi_ato = jiffies_to_usecs(icsk->icsk_ack.ato);
2229         info->tcpi_snd_mss = tp->mss_cache;
2230         info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss;
2231
2232         if (sk->sk_state == TCP_LISTEN) {
2233                 info->tcpi_unacked = sk->sk_ack_backlog;
2234                 info->tcpi_sacked = sk->sk_max_ack_backlog;
2235         } else {
2236                 info->tcpi_unacked = tp->packets_out;
2237                 info->tcpi_sacked = tp->sacked_out;
2238         }
2239         info->tcpi_lost = tp->lost_out;
2240         info->tcpi_retrans = tp->retrans_out;
2241         info->tcpi_fackets = tp->fackets_out;
2242
2243         info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime);
2244         info->tcpi_last_data_recv = jiffies_to_msecs(now - icsk->icsk_ack.lrcvtime);
2245         info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp);
2246
2247         info->tcpi_pmtu = icsk->icsk_pmtu_cookie;
2248         info->tcpi_rcv_ssthresh = tp->rcv_ssthresh;
2249         info->tcpi_rtt = jiffies_to_usecs(tp->srtt)>>3;
2250         info->tcpi_rttvar = jiffies_to_usecs(tp->mdev)>>2;
2251         info->tcpi_snd_ssthresh = tp->snd_ssthresh;
2252         info->tcpi_snd_cwnd = tp->snd_cwnd;
2253         info->tcpi_advmss = tp->advmss;
2254         info->tcpi_reordering = tp->reordering;
2255
2256         info->tcpi_rcv_rtt = jiffies_to_usecs(tp->rcv_rtt_est.rtt)>>3;
2257         info->tcpi_rcv_space = tp->rcvq_space.space;
2258
2259         info->tcpi_total_retrans = tp->total_retrans;
2260 }
2261
2262 EXPORT_SYMBOL_GPL(tcp_get_info);
2263
2264 static int do_tcp_getsockopt(struct sock *sk, int level,
2265                 int optname, char __user *optval, int __user *optlen)
2266 {
2267         struct inet_connection_sock *icsk = inet_csk(sk);
2268         struct tcp_sock *tp = tcp_sk(sk);
2269         int val, len;
2270
2271         if (get_user(len, optlen))
2272                 return -EFAULT;
2273
2274         len = min_t(unsigned int, len, sizeof(int));
2275
2276         if (len < 0)
2277                 return -EINVAL;
2278
2279         switch (optname) {
2280         case TCP_MAXSEG:
2281                 val = tp->mss_cache;
2282                 if (!val && ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
2283                         val = tp->rx_opt.user_mss;
2284                 break;
2285         case TCP_NODELAY:
2286                 val = !!(tp->nonagle&TCP_NAGLE_OFF);
2287                 break;
2288         case TCP_CORK:
2289                 val = !!(tp->nonagle&TCP_NAGLE_CORK);
2290                 break;
2291         case TCP_KEEPIDLE:
2292                 val = (tp->keepalive_time ? : sysctl_tcp_keepalive_time) / HZ;
2293                 break;
2294         case TCP_KEEPINTVL:
2295                 val = (tp->keepalive_intvl ? : sysctl_tcp_keepalive_intvl) / HZ;
2296                 break;
2297         case TCP_KEEPCNT:
2298                 val = tp->keepalive_probes ? : sysctl_tcp_keepalive_probes;
2299                 break;
2300         case TCP_SYNCNT:
2301                 val = icsk->icsk_syn_retries ? : sysctl_tcp_syn_retries;
2302                 break;
2303         case TCP_LINGER2:
2304                 val = tp->linger2;
2305                 if (val >= 0)
2306                         val = (val ? : sysctl_tcp_fin_timeout) / HZ;
2307                 break;
2308         case TCP_DEFER_ACCEPT:
2309                 val = !icsk->icsk_accept_queue.rskq_defer_accept ? 0 :
2310                         ((TCP_TIMEOUT_INIT / HZ) << (icsk->icsk_accept_queue.rskq_defer_accept - 1));
2311                 break;
2312         case TCP_WINDOW_CLAMP:
2313                 val = tp->window_clamp;
2314                 break;
2315         case TCP_INFO: {
2316                 struct tcp_info info;
2317
2318                 if (get_user(len, optlen))
2319                         return -EFAULT;
2320
2321                 tcp_get_info(sk, &info);
2322
2323                 len = min_t(unsigned int, len, sizeof(info));
2324                 if (put_user(len, optlen))
2325                         return -EFAULT;
2326                 if (copy_to_user(optval, &info, len))
2327                         return -EFAULT;
2328                 return 0;
2329         }
2330         case TCP_QUICKACK:
2331                 val = !icsk->icsk_ack.pingpong;
2332                 break;
2333
2334         case TCP_CONGESTION:
2335                 if (get_user(len, optlen))
2336                         return -EFAULT;
2337                 len = min_t(unsigned int, len, TCP_CA_NAME_MAX);
2338                 if (put_user(len, optlen))
2339                         return -EFAULT;
2340                 if (copy_to_user(optval, icsk->icsk_ca_ops->name, len))
2341                         return -EFAULT;
2342                 return 0;
2343         default:
2344                 return -ENOPROTOOPT;
2345         }
2346
2347         if (put_user(len, optlen))
2348                 return -EFAULT;
2349         if (copy_to_user(optval, &val, len))
2350                 return -EFAULT;
2351         return 0;
2352 }
2353
2354 int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
2355                    int __user *optlen)
2356 {
2357         struct inet_connection_sock *icsk = inet_csk(sk);
2358
2359         if (level != SOL_TCP)
2360                 return icsk->icsk_af_ops->getsockopt(sk, level, optname,
2361                                                      optval, optlen);
2362         return do_tcp_getsockopt(sk, level, optname, optval, optlen);
2363 }
2364
2365 #ifdef CONFIG_COMPAT
2366 int compat_tcp_getsockopt(struct sock *sk, int level, int optname,
2367                           char __user *optval, int __user *optlen)
2368 {
2369         if (level != SOL_TCP)
2370                 return inet_csk_compat_getsockopt(sk, level, optname,
2371                                                   optval, optlen);
2372         return do_tcp_getsockopt(sk, level, optname, optval, optlen);
2373 }
2374
2375 EXPORT_SYMBOL(compat_tcp_getsockopt);
2376 #endif
2377
2378 struct sk_buff *tcp_tso_segment(struct sk_buff *skb, int features)
2379 {
2380         struct sk_buff *segs = ERR_PTR(-EINVAL);
2381         struct tcphdr *th;
2382         unsigned thlen;
2383         unsigned int seq;
2384         __be32 delta;
2385         unsigned int oldlen;
2386         unsigned int len;
2387
2388         if (!pskb_may_pull(skb, sizeof(*th)))
2389                 goto out;
2390
2391         th = tcp_hdr(skb);
2392         thlen = th->doff * 4;
2393         if (thlen < sizeof(*th))
2394                 goto out;
2395
2396         if (!pskb_may_pull(skb, thlen))
2397                 goto out;
2398
2399         oldlen = (u16)~skb->len;
2400         __skb_pull(skb, thlen);
2401
2402         if (skb_gso_ok(skb, features | NETIF_F_GSO_ROBUST)) {
2403                 /* Packet is from an untrusted source, reset gso_segs. */
2404                 int type = skb_shinfo(skb)->gso_type;
2405                 int mss;
2406
2407                 if (unlikely(type &
2408                              ~(SKB_GSO_TCPV4 |
2409                                SKB_GSO_DODGY |
2410                                SKB_GSO_TCP_ECN |
2411                                SKB_GSO_TCPV6 |
2412                                0) ||
2413                              !(type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))))
2414                         goto out;
2415
2416                 mss = skb_shinfo(skb)->gso_size;
2417                 skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(skb->len, mss);
2418
2419                 segs = NULL;
2420                 goto out;
2421         }
2422
2423         segs = skb_segment(skb, features);
2424         if (IS_ERR(segs))
2425                 goto out;
2426
2427         len = skb_shinfo(skb)->gso_size;
2428         delta = htonl(oldlen + (thlen + len));
2429
2430         skb = segs;
2431         th = tcp_hdr(skb);
2432         seq = ntohl(th->seq);
2433
2434         do {
2435                 th->fin = th->psh = 0;
2436
2437                 th->check = ~csum_fold((__force __wsum)((__force u32)th->check +
2438                                        (__force u32)delta));
2439                 if (skb->ip_summed != CHECKSUM_PARTIAL)
2440                         th->check =
2441                              csum_fold(csum_partial(skb_transport_header(skb),
2442                                                     thlen, skb->csum));
2443
2444                 seq += len;
2445                 skb = skb->next;
2446                 th = tcp_hdr(skb);
2447
2448                 th->seq = htonl(seq);
2449                 th->cwr = 0;
2450         } while (skb->next);
2451
2452         delta = htonl(oldlen + (skb->tail - skb->transport_header) +
2453                       skb->data_len);
2454         th->check = ~csum_fold((__force __wsum)((__force u32)th->check +
2455                                 (__force u32)delta));
2456         if (skb->ip_summed != CHECKSUM_PARTIAL)
2457                 th->check = csum_fold(csum_partial(skb_transport_header(skb),
2458                                                    thlen, skb->csum));
2459
2460 out:
2461         return segs;
2462 }
2463 EXPORT_SYMBOL(tcp_tso_segment);
2464
2465 #ifdef CONFIG_TCP_MD5SIG
2466 static unsigned long tcp_md5sig_users;
2467 static struct tcp_md5sig_pool **tcp_md5sig_pool;
2468 static DEFINE_SPINLOCK(tcp_md5sig_pool_lock);
2469
2470 static void __tcp_free_md5sig_pool(struct tcp_md5sig_pool **pool)
2471 {
2472         int cpu;
2473         for_each_possible_cpu(cpu) {
2474                 struct tcp_md5sig_pool *p = *per_cpu_ptr(pool, cpu);
2475                 if (p) {
2476                         if (p->md5_desc.tfm)
2477                                 crypto_free_hash(p->md5_desc.tfm);
2478                         kfree(p);
2479                         p = NULL;
2480                 }
2481         }
2482         free_percpu(pool);
2483 }
2484
2485 void tcp_free_md5sig_pool(void)
2486 {
2487         struct tcp_md5sig_pool **pool = NULL;
2488
2489         spin_lock_bh(&tcp_md5sig_pool_lock);
2490         if (--tcp_md5sig_users == 0) {
2491                 pool = tcp_md5sig_pool;
2492                 tcp_md5sig_pool = NULL;
2493         }
2494         spin_unlock_bh(&tcp_md5sig_pool_lock);
2495         if (pool)
2496                 __tcp_free_md5sig_pool(pool);
2497 }
2498
2499 EXPORT_SYMBOL(tcp_free_md5sig_pool);
2500
2501 static struct tcp_md5sig_pool **__tcp_alloc_md5sig_pool(void)
2502 {
2503         int cpu;
2504         struct tcp_md5sig_pool **pool;
2505
2506         pool = alloc_percpu(struct tcp_md5sig_pool *);
2507         if (!pool)
2508                 return NULL;
2509
2510         for_each_possible_cpu(cpu) {
2511                 struct tcp_md5sig_pool *p;
2512                 struct crypto_hash *hash;
2513
2514                 p = kzalloc(sizeof(*p), GFP_KERNEL);
2515                 if (!p)
2516                         goto out_free;
2517                 *per_cpu_ptr(pool, cpu) = p;
2518
2519                 hash = crypto_alloc_hash("md5", 0, CRYPTO_ALG_ASYNC);
2520                 if (!hash || IS_ERR(hash))
2521                         goto out_free;
2522
2523                 p->md5_desc.tfm = hash;
2524         }
2525         return pool;
2526 out_free:
2527         __tcp_free_md5sig_pool(pool);
2528         return NULL;
2529 }
2530
2531 struct tcp_md5sig_pool **tcp_alloc_md5sig_pool(void)
2532 {
2533         struct tcp_md5sig_pool **pool;
2534         int alloc = 0;
2535
2536 retry:
2537         spin_lock_bh(&tcp_md5sig_pool_lock);
2538         pool = tcp_md5sig_pool;
2539         if (tcp_md5sig_users++ == 0) {
2540                 alloc = 1;
2541                 spin_unlock_bh(&tcp_md5sig_pool_lock);
2542         } else if (!pool) {
2543                 tcp_md5sig_users--;
2544                 spin_unlock_bh(&tcp_md5sig_pool_lock);
2545                 cpu_relax();
2546                 goto retry;
2547         } else
2548                 spin_unlock_bh(&tcp_md5sig_pool_lock);
2549
2550         if (alloc) {
2551                 /* we cannot hold spinlock here because this may sleep. */
2552                 struct tcp_md5sig_pool **p = __tcp_alloc_md5sig_pool();
2553                 spin_lock_bh(&tcp_md5sig_pool_lock);
2554                 if (!p) {
2555                         tcp_md5sig_users--;
2556                         spin_unlock_bh(&tcp_md5sig_pool_lock);
2557                         return NULL;
2558                 }
2559                 pool = tcp_md5sig_pool;
2560                 if (pool) {
2561                         /* oops, it has already been assigned. */
2562                         spin_unlock_bh(&tcp_md5sig_pool_lock);
2563                         __tcp_free_md5sig_pool(p);
2564                 } else {
2565                         tcp_md5sig_pool = pool = p;
2566                         spin_unlock_bh(&tcp_md5sig_pool_lock);
2567                 }
2568         }
2569         return pool;
2570 }
2571
2572 EXPORT_SYMBOL(tcp_alloc_md5sig_pool);
2573
2574 struct tcp_md5sig_pool *__tcp_get_md5sig_pool(int cpu)
2575 {
2576         struct tcp_md5sig_pool **p;
2577         spin_lock_bh(&tcp_md5sig_pool_lock);
2578         p = tcp_md5sig_pool;
2579         if (p)
2580                 tcp_md5sig_users++;
2581         spin_unlock_bh(&tcp_md5sig_pool_lock);
2582         return (p ? *per_cpu_ptr(p, cpu) : NULL);
2583 }
2584
2585 EXPORT_SYMBOL(__tcp_get_md5sig_pool);
2586
2587 void __tcp_put_md5sig_pool(void)
2588 {
2589         tcp_free_md5sig_pool();
2590 }
2591
2592 EXPORT_SYMBOL(__tcp_put_md5sig_pool);
2593 #endif
2594
2595 void tcp_done(struct sock *sk)
2596 {
2597         if(sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV)
2598                 TCP_INC_STATS_BH(TCP_MIB_ATTEMPTFAILS);
2599
2600         tcp_set_state(sk, TCP_CLOSE);
2601         tcp_clear_xmit_timers(sk);
2602
2603         sk->sk_shutdown = SHUTDOWN_MASK;
2604
2605         if (!sock_flag(sk, SOCK_DEAD))
2606                 sk->sk_state_change(sk);
2607         else
2608                 inet_csk_destroy_sock(sk);
2609 }
2610 EXPORT_SYMBOL_GPL(tcp_done);
2611
2612 extern struct tcp_congestion_ops tcp_reno;
2613
2614 static __initdata unsigned long thash_entries;
2615 static int __init set_thash_entries(char *str)
2616 {
2617         if (!str)
2618                 return 0;
2619         thash_entries = simple_strtoul(str, &str, 0);
2620         return 1;
2621 }
2622 __setup("thash_entries=", set_thash_entries);
2623
2624 void __init tcp_init(void)
2625 {
2626         struct sk_buff *skb = NULL;
2627         unsigned long nr_pages, limit;
2628         int order, i, max_share;
2629
2630         BUILD_BUG_ON(sizeof(struct tcp_skb_cb) > sizeof(skb->cb));
2631
2632         tcp_hashinfo.bind_bucket_cachep =
2633                 kmem_cache_create("tcp_bind_bucket",
2634                                   sizeof(struct inet_bind_bucket), 0,
2635                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
2636
2637         /* Size and allocate the main established and bind bucket
2638          * hash tables.
2639          *
2640          * The methodology is similar to that of the buffer cache.
2641          */
2642         tcp_hashinfo.ehash =
2643                 alloc_large_system_hash("TCP established",
2644                                         sizeof(struct inet_ehash_bucket),
2645                                         thash_entries,
2646                                         (num_physpages >= 128 * 1024) ?
2647                                         13 : 15,
2648                                         0,
2649                                         &tcp_hashinfo.ehash_size,
2650                                         NULL,
2651                                         thash_entries ? 0 : 512 * 1024);
2652         tcp_hashinfo.ehash_size = 1 << tcp_hashinfo.ehash_size;
2653         for (i = 0; i < tcp_hashinfo.ehash_size; i++) {
2654                 INIT_HLIST_HEAD(&tcp_hashinfo.ehash[i].chain);
2655                 INIT_HLIST_HEAD(&tcp_hashinfo.ehash[i].twchain);
2656         }
2657         if (inet_ehash_locks_alloc(&tcp_hashinfo))
2658                 panic("TCP: failed to alloc ehash_locks");
2659         tcp_hashinfo.bhash =
2660                 alloc_large_system_hash("TCP bind",
2661                                         sizeof(struct inet_bind_hashbucket),
2662                                         tcp_hashinfo.ehash_size,
2663                                         (num_physpages >= 128 * 1024) ?
2664                                         13 : 15,
2665                                         0,
2666                                         &tcp_hashinfo.bhash_size,
2667                                         NULL,
2668                                         64 * 1024);
2669         tcp_hashinfo.bhash_size = 1 << tcp_hashinfo.bhash_size;
2670         for (i = 0; i < tcp_hashinfo.bhash_size; i++) {
2671                 spin_lock_init(&tcp_hashinfo.bhash[i].lock);
2672                 INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain);
2673         }
2674
2675         /* Try to be a bit smarter and adjust defaults depending
2676          * on available memory.
2677          */
2678         for (order = 0; ((1 << order) << PAGE_SHIFT) <
2679                         (tcp_hashinfo.bhash_size * sizeof(struct inet_bind_hashbucket));
2680                         order++)
2681                 ;
2682         if (order >= 4) {
2683                 tcp_death_row.sysctl_max_tw_buckets = 180000;
2684                 sysctl_tcp_max_orphans = 4096 << (order - 4);
2685                 sysctl_max_syn_backlog = 1024;
2686         } else if (order < 3) {
2687                 tcp_death_row.sysctl_max_tw_buckets >>= (3 - order);
2688                 sysctl_tcp_max_orphans >>= (3 - order);
2689                 sysctl_max_syn_backlog = 128;
2690         }
2691
2692         /* Set the pressure threshold to be a fraction of global memory that
2693          * is up to 1/2 at 256 MB, decreasing toward zero with the amount of
2694          * memory, with a floor of 128 pages.
2695          */
2696         nr_pages = totalram_pages - totalhigh_pages;
2697         limit = min(nr_pages, 1UL<<(28-PAGE_SHIFT)) >> (20-PAGE_SHIFT);
2698         limit = (limit * (nr_pages >> (20-PAGE_SHIFT))) >> (PAGE_SHIFT-11);
2699         limit = max(limit, 128UL);
2700         sysctl_tcp_mem[0] = limit / 4 * 3;
2701         sysctl_tcp_mem[1] = limit;
2702         sysctl_tcp_mem[2] = sysctl_tcp_mem[0] * 2;
2703
2704         /* Set per-socket limits to no more than 1/128 the pressure threshold */
2705         limit = ((unsigned long)sysctl_tcp_mem[1]) << (PAGE_SHIFT - 7);
2706         max_share = min(4UL*1024*1024, limit);
2707
2708         sysctl_tcp_wmem[0] = SK_MEM_QUANTUM;
2709         sysctl_tcp_wmem[1] = 16*1024;
2710         sysctl_tcp_wmem[2] = max(64*1024, max_share);
2711
2712         sysctl_tcp_rmem[0] = SK_MEM_QUANTUM;
2713         sysctl_tcp_rmem[1] = 87380;
2714         sysctl_tcp_rmem[2] = max(87380, max_share);
2715
2716         printk(KERN_INFO "TCP: Hash tables configured "
2717                "(established %d bind %d)\n",
2718                tcp_hashinfo.ehash_size, tcp_hashinfo.bhash_size);
2719
2720         tcp_register_congestion_control(&tcp_reno);
2721 }
2722
2723 EXPORT_SYMBOL(tcp_close);
2724 EXPORT_SYMBOL(tcp_disconnect);
2725 EXPORT_SYMBOL(tcp_getsockopt);
2726 EXPORT_SYMBOL(tcp_ioctl);
2727 EXPORT_SYMBOL(tcp_poll);
2728 EXPORT_SYMBOL(tcp_read_sock);
2729 EXPORT_SYMBOL(tcp_recvmsg);
2730 EXPORT_SYMBOL(tcp_sendmsg);
2731 EXPORT_SYMBOL(tcp_splice_read);
2732 EXPORT_SYMBOL(tcp_sendpage);
2733 EXPORT_SYMBOL(tcp_setsockopt);
2734 EXPORT_SYMBOL(tcp_shutdown);
2735 EXPORT_SYMBOL(tcp_statistics);