net/ipv4/tcp.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              Implementation of the Transmission Control Protocol(TCP).
   7  *
   8  * Version:     $Id: tcp.c,v 1.216 2002/02/01 22:01:04 davem Exp $
   9  *
  10  * Authors:     Ross Biro
  11  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  12  *              Mark Evans, <evansmp@uhura.aston.ac.uk>
  13  *              Corey Minyard <wf-rch!minyard@relay.EU.net>
  14  *              Florian La Roche, <flla@stud.uni-sb.de>
  15  *              Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
  16  *              Linus Torvalds, <torvalds@cs.helsinki.fi>
  17  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  18  *              Matthew Dillon, <dillon@apollo.west.oic.com>
  19  *              Arnt Gulbrandsen, <agulbra@nvg.unit.no>
  20  *              Jorge Cwik, <jorge@laser.satlink.net>
  21  *
  22  * Fixes:
  23  *              Alan Cox        :       Numerous verify_area() calls
  24  *              Alan Cox        :       Set the ACK bit on a reset
  25  *              Alan Cox        :       Stopped it crashing if it closed while
  26  *                                      sk->inuse=1 and was trying to connect
  27  *                                      (tcp_err()).
  28  *              Alan Cox        :       All icmp error handling was broken
  29  *                                      pointers passed where wrong and the
  30  *                                      socket was looked up backwards. Nobody
  31  *                                      tested any icmp error code obviously.
  32  *              Alan Cox        :       tcp_err() now handled properly. It
  33  *                                      wakes people on errors. poll
  34  *                                      behaves and the icmp error race
  35  *                                      has gone by moving it into sock.c
  36  *              Alan Cox        :       tcp_send_reset() fixed to work for
  37  *                                      everything not just packets for
  38  *                                      unknown sockets.
  39  *              Alan Cox        :       tcp option processing.
  40  *              Alan Cox        :       Reset tweaked (still not 100%) [Had
  41  *                                      syn rule wrong]
  42  *              Herp Rosmanith  :       More reset fixes
  43  *              Alan Cox        :       No longer acks invalid rst frames.
  44  *                                      Acking any kind of RST is right out.
  45  *              Alan Cox        :       Sets an ignore me flag on an rst
  46  *                                      receive otherwise odd bits of prattle
  47  *                                      escape still
  48  *              Alan Cox        :       Fixed another acking RST frame bug.
  49  *                                      Should stop LAN workplace lockups.
  50  *              Alan Cox        :       Some tidyups using the new skb list
  51  *                                      facilities
  52  *              Alan Cox        :       sk->keepopen now seems to work
  53  *              Alan Cox        :       Pulls options out correctly on accepts
  54  *              Alan Cox        :       Fixed assorted sk->rqueue->next errors
  55  *              Alan Cox        :       PSH doesn't end a TCP read. Switched a
  56  *                                      bit to skb ops.
  57  *              Alan Cox        :       Tidied tcp_data to avoid a potential
  58  *                                      nasty.
  59  *              Alan Cox        :       Added some better commenting, as the
  60  *                                      tcp is hard to follow
  61  *              Alan Cox        :       Removed incorrect check for 20 * psh
  62  *      Michael O'Reilly        :       ack < copied bug fix.
  63  *      Johannes Stille         :       Misc tcp fixes (not all in yet).
  64  *              Alan Cox        :       FIN with no memory -> CRASH
  65  *              Alan Cox        :       Added socket option proto entries.
  66  *                                      Also added awareness of them to accept.
  67  *              Alan Cox        :       Added TCP options (SOL_TCP)
  68  *              Alan Cox        :       Switched wakeup calls to callbacks,
  69  *                                      so the kernel can layer network
  70  *                                      sockets.
  71  *              Alan Cox        :       Use ip_tos/ip_ttl settings.
  72  *              Alan Cox        :       Handle FIN (more) properly (we hope).
  73  *              Alan Cox        :       RST frames sent on unsynchronised
  74  *                                      state ack error.
  75  *              Alan Cox        :       Put in missing check for SYN bit.
  76  *              Alan Cox        :       Added tcp_select_window() aka NET2E
  77  *                                      window non shrink trick.
  78  *              Alan Cox        :       Added a couple of small NET2E timer
  79  *                                      fixes
  80  *              Charles Hedrick :       TCP fixes
  81  *              Toomas Tamm     :       TCP window fixes
  82  *              Alan Cox        :       Small URG fix to rlogin ^C ack fight
  83  *              Charles Hedrick :       Rewrote most of it to actually work
  84  *              Linus           :       Rewrote tcp_read() and URG handling
  85  *                                      completely
  86  *              Gerhard Koerting:       Fixed some missing timer handling
  87  *              Matthew Dillon  :       Reworked TCP machine states as per RFC
  88  *              Gerhard Koerting:       PC/TCP workarounds
  89  *              Adam Caldwell   :       Assorted timer/timing errors
  90  *              Matthew Dillon  :       Fixed another RST bug
  91  *              Alan Cox        :       Move to kernel side addressing changes.
  92  *              Alan Cox        :       Beginning work on TCP fastpathing
  93  *                                      (not yet usable)
  94  *              Arnt Gulbrandsen:       Turbocharged tcp_check() routine.
  95  *              Alan Cox        :       TCP fast path debugging
  96  *              Alan Cox        :       Window clamping
  97  *              Michael Riepe   :       Bug in tcp_check()
  98  *              Matt Dillon     :       More TCP improvements and RST bug fixes
  99  *              Matt Dillon     :       Yet more small nasties remove from the
 100  *                                      TCP code (Be very nice to this man if
 101  *                                      tcp finally works 100%) 8)
 102  *              Alan Cox        :       BSD accept semantics.
 103  *              Alan Cox        :       Reset on closedown bug.
 104  *      Peter De Schrijver      :       ENOTCONN check missing in tcp_sendto().
 105  *              Michael Pall    :       Handle poll() after URG properly in
 106  *                                      all cases.
 107  *              Michael Pall    :       Undo the last fix in tcp_read_urg()
 108  *                                      (multi URG PUSH broke rlogin).
 109  *              Michael Pall    :       Fix the multi URG PUSH problem in
 110  *                                      tcp_readable(), poll() after URG
 111  *                                      works now.
 112  *              Michael Pall    :       recv(...,MSG_OOB) never blocks in the
 113  *                                      BSD api.
 114  *              Alan Cox        :       Changed the semantics of sk->socket to
 115  *                                      fix a race and a signal problem with
 116  *                                      accept() and async I/O.
 117  *              Alan Cox        :       Relaxed the rules on tcp_sendto().
 118  *              Yury Shevchuk   :       Really fixed accept() blocking problem.
 119  *              Craig I. Hagan  :       Allow for BSD compatible TIME_WAIT for
 120  *                                      clients/servers which listen in on
 121  *                                      fixed ports.
 122  *              Alan Cox        :       Cleaned the above up and shrank it to
 123  *                                      a sensible code size.
 124  *              Alan Cox        :       Self connect lockup fix.
 125  *              Alan Cox        :       No connect to multicast.
 126  *              Ross Biro       :       Close unaccepted children on master
 127  *                                      socket close.
 128  *              Alan Cox        :       Reset tracing code.
 129  *              Alan Cox        :       Spurious resets on shutdown.
 130  *              Alan Cox        :       Giant 15 minute/60 second timer error
 131  *              Alan Cox        :       Small whoops in polling before an
 132  *                                      accept.
 133  *              Alan Cox        :       Kept the state trace facility since
 134  *                                      it's handy for debugging.
 135  *              Alan Cox        :       More reset handler fixes.
 136  *              Alan Cox        :       Started rewriting the code based on
 137  *                                      the RFC's for other useful protocol
 138  *                                      references see: Comer, KA9Q NOS, and
 139  *                                      for a reference on the difference
 140  *                                      between specifications and how BSD
 141  *                                      works see the 4.4lite source.
 142  *              A.N.Kuznetsov   :       Don't time wait on completion of tidy
 143  *                                      close.
 144  *              Linus Torvalds  :       Fin/Shutdown & copied_seq changes.
 145  *              Linus Torvalds  :       Fixed BSD port reuse to work first syn
 146  *              Alan Cox        :       Reimplemented timers as per the RFC
 147  *                                      and using multiple timers for sanity.
 148  *              Alan Cox        :       Small bug fixes, and a lot of new
 149  *                                      comments.
 150  *              Alan Cox        :       Fixed dual reader crash by locking
 151  *                                      the buffers (much like datagram.c)
 152  *              Alan Cox        :       Fixed stuck sockets in probe. A probe
 153  *                                      now gets fed up of retrying without
 154  *                                      (even a no space) answer.
 155  *              Alan Cox        :       Extracted closing code better
 156  *              Alan Cox        :       Fixed the closing state machine to
 157  *                                      resemble the RFC.
 158  *              Alan Cox        :       More 'per spec' fixes.
 159  *              Jorge Cwik      :       Even faster checksumming.
 160  *              Alan Cox        :       tcp_data() doesn't ack illegal PSH
 161  *                                      only frames. At least one pc tcp stack
 162  *                                      generates them.
 163  *              Alan Cox        :       Cache last socket.
 164  *              Alan Cox        :       Per route irtt.
 165  *              Matt Day        :       poll()->select() match BSD precisely on error
 166  *              Alan Cox        :       New buffers
 167  *              Marc Tamsky     :       Various sk->prot->retransmits and
 168  *                                      sk->retransmits misupdating fixed.
 169  *                                      Fixed tcp_write_timeout: stuck close,
 170  *                                      and TCP syn retries gets used now.
 171  *              Mark Yarvis     :       In tcp_read_wakeup(), don't send an
 172  *                                      ack if state is TCP_CLOSED.
 173  *              Alan Cox        :       Look up device on a retransmit - routes may
 174  *                                      change. Doesn't yet cope with MSS shrink right
 175  *                                      but it's a start!
 176  *              Marc Tamsky     :       Closing in closing fixes.
 177  *              Mike Shaver     :       RFC1122 verifications.
 178  *              Alan Cox        :       rcv_saddr errors.
 179  *              Alan Cox        :       Block double connect().
 180  *              Alan Cox        :       Small hooks for enSKIP.
 181  *              Alexey Kuznetsov:       Path MTU discovery.
 182  *              Alan Cox        :       Support soft errors.
 183  *              Alan Cox        :       Fix MTU discovery pathological case
 184  *                                      when the remote claims no mtu!
 185  *              Marc Tamsky     :       TCP_CLOSE fix.
 186  *              Colin (G3TNE)   :       Send a reset on syn ack replies in
 187  *                                      window but wrong (fixes NT lpd problems)
 188  *              Pedro Roque     :       Better TCP window handling, delayed ack.
 189  *              Joerg Reuter    :       No modification of locked buffers in
 190  *                                      tcp_do_retransmit()
 191  *              Eric Schenk     :       Changed receiver side silly window
 192  *                                      avoidance algorithm to BSD style
 193  *                                      algorithm. This doubles throughput
 194  *                                      against machines running Solaris,
 195  *                                      and seems to result in general
 196  *                                      improvement.
 197  *      Stefan Magdalinski      :       adjusted tcp_readable() to fix FIONREAD
 198  *      Willy Konynenberg       :       Transparent proxying support.
 199  *      Mike McLagan            :       Routing by source
 200  *              Keith Owens     :       Do proper merging with partial SKB's in
 201  *                                      tcp_do_sendmsg to avoid burstiness.
 202  *              Eric Schenk     :       Fix fast close down bug with
 203  *                                      shutdown() followed by close().
 204  *              Andi Kleen      :       Make poll agree with SIGIO
 205  *      Salvatore Sanfilippo    :       Support SO_LINGER with linger == 1 and
 206  *                                      lingertime == 0 (RFC 793 ABORT Call)
 207  *      Hirokazu Takahashi      :       Use copy_from_user() instead of
 208  *                                      csum_and_copy_from_user() if possible.
 209  *
 210  *              This program is free software; you can redistribute it and/or
 211  *              modify it under the terms of the GNU General Public License
 212  *              as published by the Free Software Foundation; either version
 213  *              2 of the License, or(at your option) any later version.
 214  *
 215  * Description of States:
 216  *
 217  *      TCP_SYN_SENT            sent a connection request, waiting for ack
 218  *
 219  *      TCP_SYN_RECV            received a connection request, sent ack,
 220  *                              waiting for final ack in three-way handshake.
 221  *
 222  *      TCP_ESTABLISHED         connection established
 223  *
 224  *      TCP_FIN_WAIT1           our side has shutdown, waiting to complete
 225  *                              transmission of remaining buffered data
 226  *
 227  *      TCP_FIN_WAIT2           all buffered data sent, waiting for remote
 228  *                              to shutdown
 229  *
 230  *      TCP_CLOSING             both sides have shutdown but we still have
 231  *                              data we have to finish sending
 232  *
 233  *      TCP_TIME_WAIT           timeout to catch resent junk before entering
 234  *                              closed, can only be entered from FIN_WAIT2
 235  *                              or CLOSING.  Required because the other end
 236  *                              may not have gotten our last ACK causing it
 237  *                              to retransmit the data packet (which we ignore)
 238  *
 239  *      TCP_CLOSE_WAIT          remote side has shutdown and is waiting for
 240  *                              us to finish writing our data and to shutdown
 241  *                              (we have to close() to move on to LAST_ACK)
 242  *
 243  *      TCP_LAST_ACK            out side has shutdown after remote has
 244  *                              shutdown.  There may still be data in our
 245  *                              buffer that we have to finish sending
 246  *
 247  *      TCP_CLOSE               socket is finished
 248  */
 249
 250 #include <linux/config.h>
 251 #include <linux/module.h>
 252 #include <linux/types.h>
 253 #include <linux/fcntl.h>
 254 #include <linux/poll.h>
 255 #include <linux/init.h>
 256 #include <linux/smp_lock.h>
 257 #include <linux/fs.h>
 258 #include <linux/random.h>
 259 #include <linux/bootmem.h>
 260
 261 #include <net/icmp.h>
 262 #include <net/tcp.h>
 263 #include <net/xfrm.h>
 264 #include <net/ip.h>
 265
 266
 267 #include <asm/uaccess.h>
 268 #include <asm/ioctls.h>
 269
 270 int sysctl_tcp_fin_timeout = TCP_FIN_TIMEOUT;
 271
 272 DEFINE_SNMP_STAT(struct tcp_mib, tcp_statistics);
 273
 274 kmem_cache_t *tcp_bucket_cachep;
 275 kmem_cache_t *tcp_timewait_cachep;
 276
 277 atomic_t tcp_orphan_count = ATOMIC_INIT(0);
 278
 279 int sysctl_tcp_mem[3];
 280 int sysctl_tcp_wmem[3] = { 4 * 1024, 16 * 1024, 128 * 1024 };
 281 int sysctl_tcp_rmem[3] = { 4 * 1024, 87380, 87380 * 2 };
 282
 283 EXPORT_SYMBOL(sysctl_tcp_mem);
 284 EXPORT_SYMBOL(sysctl_tcp_rmem);
 285 EXPORT_SYMBOL(sysctl_tcp_wmem);
 286
 287 atomic_t tcp_memory_allocated;  /* Current allocated memory. */
 288 atomic_t tcp_sockets_allocated; /* Current number of TCP sockets. */
 289
 290 EXPORT_SYMBOL(tcp_memory_allocated);
 291 EXPORT_SYMBOL(tcp_sockets_allocated);
 292
 293 /*
 294  * Pressure flag: try to collapse.
 295  * Technical note: it is used by multiple contexts non atomically.
 296  * All the sk_stream_mem_schedule() is of this nature: accounting
 297  * is strict, actions are advisory and have some latency.
 298  */
 299 int tcp_memory_pressure;
 300
 301 EXPORT_SYMBOL(tcp_memory_pressure);
 302
 303 void tcp_enter_memory_pressure(void)
 304 {
 305         if (!tcp_memory_pressure) {
 306                 NET_INC_STATS(LINUX_MIB_TCPMEMORYPRESSURES);
 307                 tcp_memory_pressure = 1;
 308         }
 309 }
 310
 311 EXPORT_SYMBOL(tcp_enter_memory_pressure);
 312
 313 /*
 314  * LISTEN is a special case for poll..
 315  */
 316 static __inline__ unsigned int tcp_listen_poll(struct sock *sk,
 317                                                poll_table *wait)
 318 {
 319         return !reqsk_queue_empty(&tcp_sk(sk)->accept_queue) ? (POLLIN | POLLRDNORM) : 0;
 320 }
 321
 322 /*
 323  *      Wait for a TCP event.
 324  *
 325  *      Note that we don't need to lock the socket, as the upper poll layers
 326  *      take care of normal races (between the test and the event) and we don't
 327  *      go look at any of the socket buffers directly.
 328  */
 329 unsigned int tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
 330 {
 331         unsigned int mask;
 332         struct sock *sk = sock->sk;
 333         struct tcp_sock *tp = tcp_sk(sk);
 334
 335         poll_wait(file, sk->sk_sleep, wait);
 336         if (sk->sk_state == TCP_LISTEN)
 337                 return tcp_listen_poll(sk, wait);
 338
 339         /* Socket is not locked. We are protected from async events
 340            by poll logic and correct handling of state changes
 341            made by another threads is impossible in any case.
 342          */
 343
 344         mask = 0;
 345         if (sk->sk_err)
 346                 mask = POLLERR;
 347
 348         /*
 349          * POLLHUP is certainly not done right. But poll() doesn't
 350          * have a notion of HUP in just one direction, and for a
 351          * socket the read side is more interesting.
 352          *
 353          * Some poll() documentation says that POLLHUP is incompatible
 354          * with the POLLOUT/POLLWR flags, so somebody should check this
 355          * all. But careful, it tends to be safer to return too many
 356          * bits than too few, and you can easily break real applications
 357          * if you don't tell them that something has hung up!
 358          *
 359          * Check-me.
 360          *
 361          * Check number 1. POLLHUP is _UNMASKABLE_ event (see UNIX98 and
 362          * our fs/select.c). It means that after we received EOF,
 363          * poll always returns immediately, making impossible poll() on write()
 364          * in state CLOSE_WAIT. One solution is evident --- to set POLLHUP
 365          * if and only if shutdown has been made in both directions.
 366          * Actually, it is interesting to look how Solaris and DUX
 367          * solve this dilemma. I would prefer, if PULLHUP were maskable,
 368          * then we could set it on SND_SHUTDOWN. BTW examples given
 369          * in Stevens' books assume exactly this behaviour, it explains
 370          * why PULLHUP is incompatible with POLLOUT.    --ANK
 371          *
 372          * NOTE. Check for TCP_CLOSE is added. The goal is to prevent
 373          * blocking on fresh not-connected or disconnected socket. --ANK
 374          */
 375         if (sk->sk_shutdown == SHUTDOWN_MASK || sk->sk_state == TCP_CLOSE)
 376                 mask |= POLLHUP;
 377         if (sk->sk_shutdown & RCV_SHUTDOWN)
 378                 mask |= POLLIN | POLLRDNORM;
 379
 380         /* Connected? */
 381         if ((1 << sk->sk_state) & ~(TCPF_SYN_SENT | TCPF_SYN_RECV)) {
 382                 /* Potential race condition. If read of tp below will
 383                  * escape above sk->sk_state, we can be illegally awaken
 384                  * in SYN_* states. */
 385                 if ((tp->rcv_nxt != tp->copied_seq) &&
 386                     (tp->urg_seq != tp->copied_seq ||
 387                      tp->rcv_nxt != tp->copied_seq + 1 ||
 388                      sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data))
 389                         mask |= POLLIN | POLLRDNORM;
 390
 391                 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
 392                         if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk)) {
 393                                 mask |= POLLOUT | POLLWRNORM;
 394                         } else {  /* send SIGIO later */
 395                                 set_bit(SOCK_ASYNC_NOSPACE,
 396                                         &sk->sk_socket->flags);
 397                                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 398
 399                                 /* Race breaker. If space is freed after
 400                                  * wspace test but before the flags are set,
 401                                  * IO signal will be lost.
 402                                  */
 403                                 if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk))
 404                                         mask |= POLLOUT | POLLWRNORM;
 405                         }
 406                 }
 407
 408                 if (tp->urg_data & TCP_URG_VALID)
 409                         mask |= POLLPRI;
 410         }
 411         return mask;
 412 }
 413
 414 int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg)
 415 {
 416         struct tcp_sock *tp = tcp_sk(sk);
 417         int answ;
 418
 419         switch (cmd) {
 420         case SIOCINQ:
 421                 if (sk->sk_state == TCP_LISTEN)
 422                         return -EINVAL;
 423
 424                 lock_sock(sk);
 425                 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
 426                         answ = 0;
 427                 else if (sock_flag(sk, SOCK_URGINLINE) ||
 428                          !tp->urg_data ||
 429                          before(tp->urg_seq, tp->copied_seq) ||
 430                          !before(tp->urg_seq, tp->rcv_nxt)) {
 431                         answ = tp->rcv_nxt - tp->copied_seq;
 432
 433                         /* Subtract 1, if FIN is in queue. */
 434                         if (answ && !skb_queue_empty(&sk->sk_receive_queue))
 435                                 answ -=
 436                        ((struct sk_buff *)sk->sk_receive_queue.prev)->h.th->fin;
 437                 } else
 438                         answ = tp->urg_seq - tp->copied_seq;
 439                 release_sock(sk);
 440                 break;
 441         case SIOCATMARK:
 442                 answ = tp->urg_data && tp->urg_seq == tp->copied_seq;
 443                 break;
 444         case SIOCOUTQ:
 445                 if (sk->sk_state == TCP_LISTEN)
 446                         return -EINVAL;
 447
 448                 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
 449                         answ = 0;
 450                 else
 451                         answ = tp->write_seq - tp->snd_una;
 452                 break;
 453         default:
 454                 return -ENOIOCTLCMD;
 455         };
 456
 457         return put_user(answ, (int __user *)arg);
 458 }
 459
 460
 461 int tcp_listen_start(struct sock *sk)
 462 {
 463         struct inet_sock *inet = inet_sk(sk);
 464         struct tcp_sock *tp = tcp_sk(sk);
 465         int rc = reqsk_queue_alloc(&tp->accept_queue, TCP_SYNQ_HSIZE);
 466
 467         if (rc != 0)
 468                 return rc;
 469
 470         sk->sk_max_ack_backlog = 0;
 471         sk->sk_ack_backlog = 0;
 472         tcp_delack_init(tp);
 473
 474         /* There is race window here: we announce ourselves listening,
 475          * but this transition is still not validated by get_port().
 476          * It is OK, because this socket enters to hash table only
 477          * after validation is complete.
 478          */
 479         sk->sk_state = TCP_LISTEN;
 480         if (!sk->sk_prot->get_port(sk, inet->num)) {
 481                 inet->sport = htons(inet->num);
 482
 483                 sk_dst_reset(sk);
 484                 sk->sk_prot->hash(sk);
 485
 486                 return 0;
 487         }
 488
 489         sk->sk_state = TCP_CLOSE;
 490         reqsk_queue_destroy(&tp->accept_queue);
 491         return -EADDRINUSE;
 492 }
 493
 494 /*
 495  *      This routine closes sockets which have been at least partially
 496  *      opened, but not yet accepted.
 497  */
 498
 499 static void tcp_listen_stop (struct sock *sk)
 500 {
 501         struct tcp_sock *tp = tcp_sk(sk);
 502         struct tcp_listen_opt *lopt;
 503         struct request_sock *acc_req;
 504         struct request_sock *req;
 505         int i;
 506
 507         tcp_delete_keepalive_timer(sk);
 508
 509         /* make all the listen_opt local to us */
 510         lopt = reqsk_queue_yank_listen_sk(&tp->accept_queue);
 511         acc_req = reqsk_queue_yank_acceptq(&tp->accept_queue);
 512
 513         if (lopt->qlen) {
 514                 for (i = 0; i < TCP_SYNQ_HSIZE; i++) {
 515                         while ((req = lopt->syn_table[i]) != NULL) {
 516                                 lopt->syn_table[i] = req->dl_next;
 517                                 lopt->qlen--;
 518                                 reqsk_free(req);
 519
 520                 /* Following specs, it would be better either to send FIN
 521                  * (and enter FIN-WAIT-1, it is normal close)
 522                  * or to send active reset (abort).
 523                  * Certainly, it is pretty dangerous while synflood, but it is
 524                  * bad justification for our negligence 8)
 525                  * To be honest, we are not able to make either
 526                  * of the variants now.                 --ANK
 527                  */
 528                         }
 529                 }
 530         }
 531         BUG_TRAP(!lopt->qlen);
 532
 533         kfree(lopt);
 534
 535         while ((req = acc_req) != NULL) {
 536                 struct sock *child = req->sk;
 537
 538                 acc_req = req->dl_next;
 539
 540                 local_bh_disable();
 541                 bh_lock_sock(child);
 542                 BUG_TRAP(!sock_owned_by_user(child));
 543                 sock_hold(child);
 544
 545                 tcp_disconnect(child, O_NONBLOCK);
 546
 547                 sock_orphan(child);
 548
 549                 atomic_inc(&tcp_orphan_count);
 550
 551                 tcp_destroy_sock(child);
 552
 553                 bh_unlock_sock(child);
 554                 local_bh_enable();
 555                 sock_put(child);
 556
 557                 sk_acceptq_removed(sk);
 558                 __reqsk_free(req);
 559         }
 560         BUG_TRAP(!sk->sk_ack_backlog);
 561 }
 562
 563 static inline void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb)
 564 {
 565         TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_PSH;
 566         tp->pushed_seq = tp->write_seq;
 567 }
 568
 569 static inline int forced_push(struct tcp_sock *tp)
 570 {
 571         return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1));
 572 }
 573
 574 static inline void skb_entail(struct sock *sk, struct tcp_sock *tp,
 575                               struct sk_buff *skb)
 576 {
 577         skb->csum = 0;
 578         TCP_SKB_CB(skb)->seq = tp->write_seq;
 579         TCP_SKB_CB(skb)->end_seq = tp->write_seq;
 580         TCP_SKB_CB(skb)->flags = TCPCB_FLAG_ACK;
 581         TCP_SKB_CB(skb)->sacked = 0;
 582         skb_header_release(skb);
 583         __skb_queue_tail(&sk->sk_write_queue, skb);
 584         sk_charge_skb(sk, skb);
 585         if (!sk->sk_send_head)
 586                 sk->sk_send_head = skb;
 587         else if (tp->nonagle&TCP_NAGLE_PUSH)
 588                 tp->nonagle &= ~TCP_NAGLE_PUSH;
 589 }
 590
 591 static inline void tcp_mark_urg(struct tcp_sock *tp, int flags,
 592                                 struct sk_buff *skb)
 593 {
 594         if (flags & MSG_OOB) {
 595                 tp->urg_mode = 1;
 596                 tp->snd_up = tp->write_seq;
 597                 TCP_SKB_CB(skb)->sacked |= TCPCB_URG;
 598         }
 599 }
 600
 601 static inline void tcp_push(struct sock *sk, struct tcp_sock *tp, int flags,
 602                             int mss_now, int nonagle)
 603 {
 604         if (sk->sk_send_head) {
 605                 struct sk_buff *skb = sk->sk_write_queue.prev;
 606                 if (!(flags & MSG_MORE) || forced_push(tp))
 607                         tcp_mark_push(tp, skb);
 608                 tcp_mark_urg(tp, flags, skb);
 609                 __tcp_push_pending_frames(sk, tp, mss_now,
 610                                           (flags & MSG_MORE) ? TCP_NAGLE_CORK : nonagle);
 611         }
 612 }
 613
 614 static ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffset,
 615                          size_t psize, int flags)
 616 {
 617         struct tcp_sock *tp = tcp_sk(sk);
 618         int mss_now;
 619         int err;
 620         ssize_t copied;
 621         long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
 622
 623         /* Wait for a connection to finish. */
 624         if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT))
 625                 if ((err = sk_stream_wait_connect(sk, &timeo)) != 0)
 626                         goto out_err;
 627
 628         clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
 629
 630         mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
 631         copied = 0;
 632
 633         err = -EPIPE;
 634         if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
 635                 goto do_error;
 636
 637         while (psize > 0) {
 638                 struct sk_buff *skb = sk->sk_write_queue.prev;
 639                 struct page *page = pages[poffset / PAGE_SIZE];
 640                 int copy, i, can_coalesce;
 641                 int offset = poffset % PAGE_SIZE;
 642                 int size = min_t(size_t, psize, PAGE_SIZE - offset);
 643
 644                 if (!sk->sk_send_head || (copy = mss_now - skb->len) <= 0) {
 645 new_segment:
 646                         if (!sk_stream_memory_free(sk))
 647                                 goto wait_for_sndbuf;
 648
 649                         skb = sk_stream_alloc_pskb(sk, 0, 0,
 650                                                    sk->sk_allocation);
 651                         if (!skb)
 652                                 goto wait_for_memory;
 653
 654                         skb_entail(sk, tp, skb);
 655                         copy = mss_now;
 656                 }
 657
 658                 if (copy > size)
 659                         copy = size;
 660
 661                 i = skb_shinfo(skb)->nr_frags;
 662                 can_coalesce = skb_can_coalesce(skb, i, page, offset);
 663                 if (!can_coalesce && i >= MAX_SKB_FRAGS) {
 664                         tcp_mark_push(tp, skb);
 665                         goto new_segment;
 666                 }
 667                 if (sk->sk_forward_alloc < copy &&
 668                     !sk_stream_mem_schedule(sk, copy, 0))
 669                         goto wait_for_memory;
 670
 671                 if (can_coalesce) {
 672                         skb_shinfo(skb)->frags[i - 1].size += copy;
 673                 } else {
 674                         get_page(page);
 675                         skb_fill_page_desc(skb, i, page, offset, copy);
 676                 }
 677
 678                 skb->len += copy;
 679                 skb->data_len += copy;
 680                 skb->truesize += copy;
 681                 sk->sk_wmem_queued += copy;
 682                 sk->sk_forward_alloc -= copy;
 683                 skb->ip_summed = CHECKSUM_HW;
 684                 tp->write_seq += copy;
 685                 TCP_SKB_CB(skb)->end_seq += copy;
 686                 skb_shinfo(skb)->tso_segs = 0;
 687
 688                 if (!copied)
 689                         TCP_SKB_CB(skb)->flags &= ~TCPCB_FLAG_PSH;
 690
 691                 copied += copy;
 692                 poffset += copy;
 693                 if (!(psize -= copy))
 694                         goto out;
 695
 696                 if (skb->len != mss_now || (flags & MSG_OOB))
 697                         continue;
 698
 699                 if (forced_push(tp)) {
 700                         tcp_mark_push(tp, skb);
 701                         __tcp_push_pending_frames(sk, tp, mss_now, TCP_NAGLE_PUSH);
 702                 } else if (skb == sk->sk_send_head)
 703                         tcp_push_one(sk, mss_now);
 704                 continue;
 705
 706 wait_for_sndbuf:
 707                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 708 wait_for_memory:
 709                 if (copied)
 710                         tcp_push(sk, tp, flags & ~MSG_MORE, mss_now, TCP_NAGLE_PUSH);
 711
 712                 if ((err = sk_stream_wait_memory(sk, &timeo)) != 0)
 713                         goto do_error;
 714
 715                 mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
 716         }
 717
 718 out:
 719         if (copied)
 720                 tcp_push(sk, tp, flags, mss_now, tp->nonagle);
 721         return copied;
 722
 723 do_error:
 724         if (copied)
 725                 goto out;
 726 out_err:
 727         return sk_stream_error(sk, flags, err);
 728 }
 729
 730 ssize_t tcp_sendpage(struct socket *sock, struct page *page, int offset,
 731                      size_t size, int flags)
 732 {
 733         ssize_t res;
 734         struct sock *sk = sock->sk;
 735
 736 #define TCP_ZC_CSUM_FLAGS (NETIF_F_IP_CSUM | NETIF_F_NO_CSUM | NETIF_F_HW_CSUM)
 737
 738         if (!(sk->sk_route_caps & NETIF_F_SG) ||
 739             !(sk->sk_route_caps & TCP_ZC_CSUM_FLAGS))
 740                 return sock_no_sendpage(sock, page, offset, size, flags);
 741
 742 #undef TCP_ZC_CSUM_FLAGS
 743
 744         lock_sock(sk);
 745         TCP_CHECK_TIMER(sk);
 746         res = do_tcp_sendpages(sk, &page, offset, size, flags);
 747         TCP_CHECK_TIMER(sk);
 748         release_sock(sk);
 749         return res;
 750 }
 751
 752 #define TCP_PAGE(sk)    (sk->sk_sndmsg_page)
 753 #define TCP_OFF(sk)     (sk->sk_sndmsg_off)
 754
 755 static inline int select_size(struct sock *sk, struct tcp_sock *tp)
 756 {
 757         int tmp = tp->mss_cache_std;
 758
 759         if (sk->sk_route_caps & NETIF_F_SG) {
 760                 int pgbreak = SKB_MAX_HEAD(MAX_TCP_HEADER);
 761
 762                 if (tmp >= pgbreak &&
 763                     tmp <= pgbreak + (MAX_SKB_FRAGS - 1) * PAGE_SIZE)
 764                         tmp = pgbreak;
 765         }
 766         return tmp;
 767 }
 768
 769 int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
 770                 size_t size)
 771 {
 772         struct iovec *iov;
 773         struct tcp_sock *tp = tcp_sk(sk);
 774         struct sk_buff *skb;
 775         int iovlen, flags;
 776         int mss_now;
 777         int err, copied;
 778         long timeo;
 779
 780         lock_sock(sk);
 781         TCP_CHECK_TIMER(sk);
 782
 783         flags = msg->msg_flags;
 784         timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
 785
 786         /* Wait for a connection to finish. */
 787         if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT))
 788                 if ((err = sk_stream_wait_connect(sk, &timeo)) != 0)
 789                         goto out_err;
 790
 791         /* This should be in poll */
 792         clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
 793
 794         mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
 795
 796         /* Ok commence sending. */
 797         iovlen = msg->msg_iovlen;
 798         iov = msg->msg_iov;
 799         copied = 0;
 800
 801         err = -EPIPE;
 802         if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
 803                 goto do_error;
 804
 805         while (--iovlen >= 0) {
 806                 int seglen = iov->iov_len;
 807                 unsigned char __user *from = iov->iov_base;
 808
 809                 iov++;
 810
 811                 while (seglen > 0) {
 812                         int copy;
 813
 814                         skb = sk->sk_write_queue.prev;
 815
 816                         if (!sk->sk_send_head ||
 817                             (copy = mss_now - skb->len) <= 0) {
 818
 819 new_segment:
 820                                 /* Allocate new segment. If the interface is SG,
 821                                  * allocate skb fitting to single page.
 822                                  */
 823                                 if (!sk_stream_memory_free(sk))
 824                                         goto wait_for_sndbuf;
 825
 826                                 skb = sk_stream_alloc_pskb(sk, select_size(sk, tp),
 827                                                            0, sk->sk_allocation);
 828                                 if (!skb)
 829                                         goto wait_for_memory;
 830
 831                                 /*
 832                                  * Check whether we can use HW checksum.
 833                                  */
 834                                 if (sk->sk_route_caps &
 835                                     (NETIF_F_IP_CSUM | NETIF_F_NO_CSUM |
 836                                      NETIF_F_HW_CSUM))
 837                                         skb->ip_summed = CHECKSUM_HW;
 838
 839                                 skb_entail(sk, tp, skb);
 840                                 copy = mss_now;
 841                         }
 842
 843                         /* Try to append data to the end of skb. */
 844                         if (copy > seglen)
 845                                 copy = seglen;
 846
 847                         /* Where to copy to? */
 848                         if (skb_tailroom(skb) > 0) {
 849                                 /* We have some space in skb head. Superb! */
 850                                 if (copy > skb_tailroom(skb))
 851                                         copy = skb_tailroom(skb);
 852                                 if ((err = skb_add_data(skb, from, copy)) != 0)
 853                                         goto do_fault;
 854                         } else {
 855                                 int merge = 0;
 856                                 int i = skb_shinfo(skb)->nr_frags;
 857                                 struct page *page = TCP_PAGE(sk);
 858                                 int off = TCP_OFF(sk);
 859
 860                                 if (skb_can_coalesce(skb, i, page, off) &&
 861                                     off != PAGE_SIZE) {
 862                                         /* We can extend the last page
 863                                          * fragment. */
 864                                         merge = 1;
 865                                 } else if (i == MAX_SKB_FRAGS ||
 866                                            (!i &&
 867                                            !(sk->sk_route_caps & NETIF_F_SG))) {
 868                                         /* Need to add new fragment and cannot
 869                                          * do this because interface is non-SG,
 870                                          * or because all the page slots are
 871                                          * busy. */
 872                                         tcp_mark_push(tp, skb);
 873                                         goto new_segment;
 874                                 } else if (page) {
 875                                         /* If page is cached, align
 876                                          * offset to L1 cache boundary
 877                                          */
 878                                         off = (off + L1_CACHE_BYTES - 1) &
 879                                               ~(L1_CACHE_BYTES - 1);
 880                                         if (off == PAGE_SIZE) {
 881                                                 put_page(page);
 882                                                 TCP_PAGE(sk) = page = NULL;
 883                                         }
 884                                 }
 885
 886                                 if (!page) {
 887                                         /* Allocate new cache page. */
 888                                         if (!(page = sk_stream_alloc_page(sk)))
 889                                                 goto wait_for_memory;
 890                                         off = 0;
 891                                 }
 892
 893                                 if (copy > PAGE_SIZE - off)
 894                                         copy = PAGE_SIZE - off;
 895
 896                                 /* Time to copy data. We are close to
 897                                  * the end! */
 898                                 err = skb_copy_to_page(sk, from, skb, page,
 899                                                        off, copy);
 900                                 if (err) {
 901                                         /* If this page was new, give it to the
 902                                          * socket so it does not get leaked.
 903                                          */
 904                                         if (!TCP_PAGE(sk)) {
 905                                                 TCP_PAGE(sk) = page;
 906                                                 TCP_OFF(sk) = 0;
 907                                         }
 908                                         goto do_error;
 909                                 }
 910
 911                                 /* Update the skb. */
 912                                 if (merge) {
 913                                         skb_shinfo(skb)->frags[i - 1].size +=
 914                                                                         copy;
 915                                 } else {
 916                                         skb_fill_page_desc(skb, i, page, off, copy);
 917                                         if (TCP_PAGE(sk)) {
 918                                                 get_page(page);
 919                                         } else if (off + copy < PAGE_SIZE) {
 920                                                 get_page(page);
 921                                                 TCP_PAGE(sk) = page;
 922                                         }
 923                                 }
 924
 925                                 TCP_OFF(sk) = off + copy;
 926                         }
 927
 928                         if (!copied)
 929                                 TCP_SKB_CB(skb)->flags &= ~TCPCB_FLAG_PSH;
 930
 931                         tp->write_seq += copy;
 932                         TCP_SKB_CB(skb)->end_seq += copy;
 933                         skb_shinfo(skb)->tso_segs = 0;
 934
 935                         from += copy;
 936                         copied += copy;
 937                         if ((seglen -= copy) == 0 && iovlen == 0)
 938                                 goto out;
 939
 940                         if (skb->len != mss_now || (flags & MSG_OOB))
 941                                 continue;
 942
 943                         if (forced_push(tp)) {
 944                                 tcp_mark_push(tp, skb);
 945                                 __tcp_push_pending_frames(sk, tp, mss_now, TCP_NAGLE_PUSH);
 946                         } else if (skb == sk->sk_send_head)
 947                                 tcp_push_one(sk, mss_now);
 948                         continue;
 949
 950 wait_for_sndbuf:
 951                         set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 952 wait_for_memory:
 953                         if (copied)
 954                                 tcp_push(sk, tp, flags & ~MSG_MORE, mss_now, TCP_NAGLE_PUSH);
 955
 956                         if ((err = sk_stream_wait_memory(sk, &timeo)) != 0)
 957                                 goto do_error;
 958
 959                         mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
 960                 }
 961         }
 962
 963 out:
 964         if (copied)
 965                 tcp_push(sk, tp, flags, mss_now, tp->nonagle);
 966         TCP_CHECK_TIMER(sk);
 967         release_sock(sk);
 968         return copied;
 969
 970 do_fault:
 971         if (!skb->len) {
 972                 if (sk->sk_send_head == skb)
 973                         sk->sk_send_head = NULL;
 974                 __skb_unlink(skb, skb->list);
 975                 sk_stream_free_skb(sk, skb);
 976         }
 977
 978 do_error:
 979         if (copied)
 980                 goto out;
 981 out_err:
 982         err = sk_stream_error(sk, flags, err);
 983         TCP_CHECK_TIMER(sk);
 984         release_sock(sk);
 985         return err;
 986 }
 987
 988 /*
 989  *      Handle reading urgent data. BSD has very simple semantics for
 990  *      this, no blocking and very strange errors 8)
 991  */
 992
 993 static int tcp_recv_urg(struct sock *sk, long timeo,
 994                         struct msghdr *msg, int len, int flags,
 995                         int *addr_len)
 996 {
 997         struct tcp_sock *tp = tcp_sk(sk);
 998
 999         /* No URG data to read. */
1000         if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data ||
1001             tp->urg_data == TCP_URG_READ)
1002                 return -EINVAL; /* Yes this is right ! */
1003
1004         if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE))
1005                 return -ENOTCONN;
1006
1007         if (tp->urg_data & TCP_URG_VALID) {
1008                 int err = 0;
1009                 char c = tp->urg_data;
1010
1011                 if (!(flags & MSG_PEEK))
1012                         tp->urg_data = TCP_URG_READ;
1013
1014                 /* Read urgent data. */
1015                 msg->msg_flags |= MSG_OOB;
1016
1017                 if (len > 0) {
1018                         if (!(flags & MSG_TRUNC))
1019                                 err = memcpy_toiovec(msg->msg_iov, &c, 1);
1020                         len = 1;
1021                 } else
1022                         msg->msg_flags |= MSG_TRUNC;
1023
1024                 return err ? -EFAULT : len;
1025         }
1026
1027         if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN))
1028                 return 0;
1029
1030         /* Fixed the recv(..., MSG_OOB) behaviour.  BSD docs and
1031          * the available implementations agree in this case:
1032          * this call should never block, independent of the
1033          * blocking state of the socket.
1034          * Mike <pall@rz.uni-karlsruhe.de>
1035          */
1036         return -EAGAIN;
1037 }
1038
1039 /* Clean up the receive buffer for full frames taken by the user,
1040  * then send an ACK if necessary.  COPIED is the number of bytes
1041  * tcp_recvmsg has given to the user so far, it speeds up the
1042  * calculation of whether or not we must ACK for the sake of
1043  * a window update.
1044  */
1045 static void cleanup_rbuf(struct sock *sk, int copied)
1046 {
1047         struct tcp_sock *tp = tcp_sk(sk);
1048         int time_to_ack = 0;
1049
1050 #if TCP_DEBUG
1051         struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
1052
1053         BUG_TRAP(!skb || before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq));
1054 #endif
1055
1056         if (tcp_ack_scheduled(tp)) {
1057                    /* Delayed ACKs frequently hit locked sockets during bulk
1058                     * receive. */
1059                 if (tp->ack.blocked ||
1060                     /* Once-per-two-segments ACK was not sent by tcp_input.c */
1061                     tp->rcv_nxt - tp->rcv_wup > tp->ack.rcv_mss ||
1062                     /*
1063                      * If this read emptied read buffer, we send ACK, if
1064                      * connection is not bidirectional, user drained
1065                      * receive buffer and there was a small segment
1066                      * in queue.
1067                      */
1068                     (copied > 0 && (tp->ack.pending & TCP_ACK_PUSHED) &&
1069                      !tp->ack.pingpong && !atomic_read(&sk->sk_rmem_alloc)))
1070                         time_to_ack = 1;
1071         }
1072
1073         /* We send an ACK if we can now advertise a non-zero window
1074          * which has been raised "significantly".
1075          *
1076          * Even if window raised up to infinity, do not send window open ACK
1077          * in states, where we will not receive more. It is useless.
1078          */
1079         if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) {
1080                 __u32 rcv_window_now = tcp_receive_window(tp);
1081
1082                 /* Optimize, __tcp_select_window() is not cheap. */
1083                 if (2*rcv_window_now <= tp->window_clamp) {
1084                         __u32 new_window = __tcp_select_window(sk);
1085
1086                         /* Send ACK now, if this read freed lots of space
1087                          * in our buffer. Certainly, new_window is new window.
1088                          * We can advertise it now, if it is not less than current one.
1089                          * "Lots" means "at least twice" here.
1090                          */
1091                         if (new_window && new_window >= 2 * rcv_window_now)
1092                                 time_to_ack = 1;
1093                 }
1094         }
1095         if (time_to_ack)
1096                 tcp_send_ack(sk);
1097 }
1098
1099 static void tcp_prequeue_process(struct sock *sk)
1100 {
1101         struct sk_buff *skb;
1102         struct tcp_sock *tp = tcp_sk(sk);
1103
1104         NET_ADD_STATS_USER(LINUX_MIB_TCPPREQUEUED, skb_queue_len(&tp->ucopy.prequeue));
1105
1106         /* RX process wants to run with disabled BHs, though it is not
1107          * necessary */
1108         local_bh_disable();
1109         while ((skb = __skb_dequeue(&tp->ucopy.prequeue)) != NULL)
1110                 sk->sk_backlog_rcv(sk, skb);
1111         local_bh_enable();
1112
1113         /* Clear memory counter. */
1114         tp->ucopy.memory = 0;
1115 }
1116
1117 static inline struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
1118 {
1119         struct sk_buff *skb;
1120         u32 offset;
1121
1122         skb_queue_walk(&sk->sk_receive_queue, skb) {
1123                 offset = seq - TCP_SKB_CB(skb)->seq;
1124                 if (skb->h.th->syn)
1125                         offset--;
1126                 if (offset < skb->len || skb->h.th->fin) {
1127                         *off = offset;
1128                         return skb;
1129                 }
1130         }
1131         return NULL;
1132 }
1133
1134 /*
1135  * This routine provides an alternative to tcp_recvmsg() for routines
1136  * that would like to handle copying from skbuffs directly in 'sendfile'
1137  * fashion.
1138  * Note:
1139  *      - It is assumed that the socket was locked by the caller.
1140  *      - The routine does not block.
1141  *      - At present, there is no support for reading OOB data
1142  *        or for 'peeking' the socket using this routine
1143  *        (although both would be easy to implement).
1144  */
1145 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
1146                   sk_read_actor_t recv_actor)
1147 {
1148         struct sk_buff *skb;
1149         struct tcp_sock *tp = tcp_sk(sk);
1150         u32 seq = tp->copied_seq;
1151         u32 offset;
1152         int copied = 0;
1153
1154         if (sk->sk_state == TCP_LISTEN)
1155                 return -ENOTCONN;
1156         while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1157                 if (offset < skb->len) {
1158                         size_t used, len;
1159
1160                         len = skb->len - offset;
1161                         /* Stop reading if we hit a patch of urgent data */
1162                         if (tp->urg_data) {
1163                                 u32 urg_offset = tp->urg_seq - seq;
1164                                 if (urg_offset < len)
1165                                         len = urg_offset;
1166                                 if (!len)
1167                                         break;
1168                         }
1169                         used = recv_actor(desc, skb, offset, len);
1170                         if (used <= len) {
1171                                 seq += used;
1172                                 copied += used;
1173                                 offset += used;
1174                         }
1175                         if (offset != skb->len)
1176                                 break;
1177                 }
1178                 if (skb->h.th->fin) {
1179                         sk_eat_skb(sk, skb);
1180                         ++seq;
1181                         break;
1182                 }
1183                 sk_eat_skb(sk, skb);
1184                 if (!desc->count)
1185                         break;
1186         }
1187         tp->copied_seq = seq;
1188
1189         tcp_rcv_space_adjust(sk);
1190
1191         /* Clean up data we have read: This will do ACK frames. */
1192         if (copied)
1193                 cleanup_rbuf(sk, copied);
1194         return copied;
1195 }
1196
1197 /*
1198  *      This routine copies from a sock struct into the user buffer.
1199  *
1200  *      Technical note: in 2.3 we work on _locked_ socket, so that
1201  *      tricks with *seq access order and skb->users are not required.
1202  *      Probably, code can be easily improved even more.
1203  */
1204
1205 int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
1206                 size_t len, int nonblock, int flags, int *addr_len)
1207 {
1208         struct tcp_sock *tp = tcp_sk(sk);
1209         int copied = 0;
1210         u32 peek_seq;
1211         u32 *seq;
1212         unsigned long used;
1213         int err;
1214         int target;             /* Read at least this many bytes */
1215         long timeo;
1216         struct task_struct *user_recv = NULL;
1217
1218         lock_sock(sk);
1219
1220         TCP_CHECK_TIMER(sk);
1221
1222         err = -ENOTCONN;
1223         if (sk->sk_state == TCP_LISTEN)
1224                 goto out;
1225
1226         timeo = sock_rcvtimeo(sk, nonblock);
1227
1228         /* Urgent data needs to be handled specially. */
1229         if (flags & MSG_OOB)
1230                 goto recv_urg;
1231
1232         seq = &tp->copied_seq;
1233         if (flags & MSG_PEEK) {
1234                 peek_seq = tp->copied_seq;
1235                 seq = &peek_seq;
1236         }
1237
1238         target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
1239
1240         do {
1241                 struct sk_buff *skb;
1242                 u32 offset;
1243
1244                 /* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */
1245                 if (tp->urg_data && tp->urg_seq == *seq) {
1246                         if (copied)
1247                                 break;
1248                         if (signal_pending(current)) {
1249                                 copied = timeo ? sock_intr_errno(timeo) : -EAGAIN;
1250                                 break;
1251                         }
1252                 }
1253
1254                 /* Next get a buffer. */
1255
1256                 skb = skb_peek(&sk->sk_receive_queue);
1257                 do {
1258                         if (!skb)
1259                                 break;
1260
1261                         /* Now that we have two receive queues this
1262                          * shouldn't happen.
1263                          */
1264                         if (before(*seq, TCP_SKB_CB(skb)->seq)) {
1265                                 printk(KERN_INFO "recvmsg bug: copied %X "
1266                                        "seq %X\n", *seq, TCP_SKB_CB(skb)->seq);
1267                                 break;
1268                         }
1269                         offset = *seq - TCP_SKB_CB(skb)->seq;
1270                         if (skb->h.th->syn)
1271                                 offset--;
1272                         if (offset < skb->len)
1273                                 goto found_ok_skb;
1274                         if (skb->h.th->fin)
1275                                 goto found_fin_ok;
1276                         BUG_TRAP(flags & MSG_PEEK);
1277                         skb = skb->next;
1278                 } while (skb != (struct sk_buff *)&sk->sk_receive_queue);
1279
1280                 /* Well, if we have backlog, try to process it now yet. */
1281
1282                 if (copied >= target && !sk->sk_backlog.tail)
1283                         break;
1284
1285                 if (copied) {
1286                         if (sk->sk_err ||
1287                             sk->sk_state == TCP_CLOSE ||
1288                             (sk->sk_shutdown & RCV_SHUTDOWN) ||
1289                             !timeo ||
1290                             signal_pending(current) ||
1291                             (flags & MSG_PEEK))
1292                                 break;
1293                 } else {
1294                         if (sock_flag(sk, SOCK_DONE))
1295                                 break;
1296
1297                         if (sk->sk_err) {
1298                                 copied = sock_error(sk);
1299                                 break;
1300                         }
1301
1302                         if (sk->sk_shutdown & RCV_SHUTDOWN)
1303                                 break;
1304
1305                         if (sk->sk_state == TCP_CLOSE) {
1306                                 if (!sock_flag(sk, SOCK_DONE)) {
1307                                         /* This occurs when user tries to read
1308                                          * from never connected socket.
1309                                          */
1310                                         copied = -ENOTCONN;
1311                                         break;
1312                                 }
1313                                 break;
1314                         }
1315
1316                         if (!timeo) {
1317                                 copied = -EAGAIN;
1318                                 break;
1319                         }
1320
1321                         if (signal_pending(current)) {
1322                                 copied = sock_intr_errno(timeo);
1323                                 break;
1324                         }
1325                 }
1326
1327                 cleanup_rbuf(sk, copied);
1328
1329                 if (tp->ucopy.task == user_recv) {
1330                         /* Install new reader */
1331                         if (!user_recv && !(flags & (MSG_TRUNC | MSG_PEEK))) {
1332                                 user_recv = current;
1333                                 tp->ucopy.task = user_recv;
1334                                 tp->ucopy.iov = msg->msg_iov;
1335                         }
1336
1337                         tp->ucopy.len = len;
1338
1339                         BUG_TRAP(tp->copied_seq == tp->rcv_nxt ||
1340                                  (flags & (MSG_PEEK | MSG_TRUNC)));
1341
1342                         /* Ugly... If prequeue is not empty, we have to
1343                          * process it before releasing socket, otherwise
1344                          * order will be broken at second iteration.
1345                          * More elegant solution is required!!!
1346                          *
1347                          * Look: we have the following (pseudo)queues:
1348                          *
1349                          * 1. packets in flight
1350                          * 2. backlog
1351                          * 3. prequeue
1352                          * 4. receive_queue
1353                          *
1354                          * Each queue can be processed only if the next ones
1355                          * are empty. At this point we have empty receive_queue.
1356                          * But prequeue _can_ be not empty after 2nd iteration,
1357                          * when we jumped to start of loop because backlog
1358                          * processing added something to receive_queue.
1359                          * We cannot release_sock(), because backlog contains
1360                          * packets arrived _after_ prequeued ones.
1361                          *
1362                          * Shortly, algorithm is clear --- to process all
1363                          * the queues in order. We could make it more directly,
1364                          * requeueing packets from backlog to prequeue, if
1365                          * is not empty. It is more elegant, but eats cycles,
1366                          * unfortunately.
1367                          */
1368                         if (skb_queue_len(&tp->ucopy.prequeue))
1369                                 goto do_prequeue;
1370
1371                         /* __ Set realtime policy in scheduler __ */
1372                 }
1373
1374                 if (copied >= target) {
1375                         /* Do not sleep, just process backlog. */
1376                         release_sock(sk);
1377                         lock_sock(sk);
1378                 } else
1379                         sk_wait_data(sk, &timeo);
1380
1381                 if (user_recv) {
1382                         int chunk;
1383
1384                         /* __ Restore normal policy in scheduler __ */
1385
1386                         if ((chunk = len - tp->ucopy.len) != 0) {
1387                                 NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMBACKLOG, chunk);
1388                                 len -= chunk;
1389                                 copied += chunk;
1390                         }
1391
1392                         if (tp->rcv_nxt == tp->copied_seq &&
1393                             skb_queue_len(&tp->ucopy.prequeue)) {
1394 do_prequeue:
1395                                 tcp_prequeue_process(sk);
1396
1397                                 if ((chunk = len - tp->ucopy.len) != 0) {
1398                                         NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk);
1399                                         len -= chunk;
1400                                         copied += chunk;
1401                                 }
1402                         }
1403                 }
1404                 if ((flags & MSG_PEEK) && peek_seq != tp->copied_seq) {
1405                         if (net_ratelimit())
1406                                 printk(KERN_DEBUG "TCP(%s:%d): Application bug, race in MSG_PEEK.\n",
1407                                        current->comm, current->pid);
1408                         peek_seq = tp->copied_seq;
1409                 }
1410                 continue;
1411
1412         found_ok_skb:
1413                 /* Ok so how much can we use? */
1414                 used = skb->len - offset;
1415                 if (len < used)
1416                         used = len;
1417
1418                 /* Do we have urgent data here? */
1419                 if (tp->urg_data) {
1420                         u32 urg_offset = tp->urg_seq - *seq;
1421                         if (urg_offset < used) {
1422                                 if (!urg_offset) {
1423                                         if (!sock_flag(sk, SOCK_URGINLINE)) {
1424                                                 ++*seq;
1425                                                 offset++;
1426                                                 used--;
1427                                                 if (!used)
1428                                                         goto skip_copy;
1429                                         }
1430                                 } else
1431                                         used = urg_offset;
1432                         }
1433                 }
1434
1435                 if (!(flags & MSG_TRUNC)) {
1436                         err = skb_copy_datagram_iovec(skb, offset,
1437                                                       msg->msg_iov, used);
1438                         if (err) {
1439                                 /* Exception. Bailout! */
1440                                 if (!copied)
1441                                         copied = -EFAULT;
1442                                 break;
1443                         }
1444                 }
1445
1446                 *seq += used;
1447                 copied += used;
1448                 len -= used;
1449
1450                 tcp_rcv_space_adjust(sk);
1451
1452 skip_copy:
1453                 if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) {
1454                         tp->urg_data = 0;
1455                         tcp_fast_path_check(sk, tp);
1456                 }
1457                 if (used + offset < skb->len)
1458                         continue;
1459
1460                 if (skb->h.th->fin)
1461                         goto found_fin_ok;
1462                 if (!(flags & MSG_PEEK))
1463                         sk_eat_skb(sk, skb);
1464                 continue;
1465
1466         found_fin_ok:
1467                 /* Process the FIN. */
1468                 ++*seq;
1469                 if (!(flags & MSG_PEEK))
1470                         sk_eat_skb(sk, skb);
1471                 break;
1472         } while (len > 0);
1473
1474         if (user_recv) {
1475                 if (skb_queue_len(&tp->ucopy.prequeue)) {
1476                         int chunk;
1477
1478                         tp->ucopy.len = copied > 0 ? len : 0;
1479
1480                         tcp_prequeue_process(sk);
1481
1482                         if (copied > 0 && (chunk = len - tp->ucopy.len) != 0) {
1483                                 NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk);
1484                                 len -= chunk;
1485                                 copied += chunk;
1486                         }
1487                 }
1488
1489                 tp->ucopy.task = NULL;
1490                 tp->ucopy.len = 0;
1491         }
1492
1493         /* According to UNIX98, msg_name/msg_namelen are ignored
1494          * on connected socket. I was just happy when found this 8) --ANK
1495          */
1496
1497         /* Clean up data we have read: This will do ACK frames. */
1498         cleanup_rbuf(sk, copied);
1499
1500         TCP_CHECK_TIMER(sk);
1501         release_sock(sk);
1502         return copied;
1503
1504 out:
1505         TCP_CHECK_TIMER(sk);
1506         release_sock(sk);
1507         return err;
1508
1509 recv_urg:
1510         err = tcp_recv_urg(sk, timeo, msg, len, flags, addr_len);
1511         goto out;
1512 }
1513
1514 /*
1515  *      State processing on a close. This implements the state shift for
1516  *      sending our FIN frame. Note that we only send a FIN for some
1517  *      states. A shutdown() may have already sent the FIN, or we may be
1518  *      closed.
1519  */
1520
1521 static unsigned char new_state[16] = {
1522   /* current state:        new state:      action:      */
1523   /* (Invalid)          */ TCP_CLOSE,
1524   /* TCP_ESTABLISHED    */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
1525   /* TCP_SYN_SENT       */ TCP_CLOSE,
1526   /* TCP_SYN_RECV       */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
1527   /* TCP_FIN_WAIT1      */ TCP_FIN_WAIT1,
1528   /* TCP_FIN_WAIT2      */ TCP_FIN_WAIT2,
1529   /* TCP_TIME_WAIT      */ TCP_CLOSE,
1530   /* TCP_CLOSE          */ TCP_CLOSE,
1531   /* TCP_CLOSE_WAIT     */ TCP_LAST_ACK  | TCP_ACTION_FIN,
1532   /* TCP_LAST_ACK       */ TCP_LAST_ACK,
1533   /* TCP_LISTEN         */ TCP_CLOSE,
1534   /* TCP_CLOSING        */ TCP_CLOSING,
1535 };
1536
1537 static int tcp_close_state(struct sock *sk)
1538 {
1539         int next = (int)new_state[sk->sk_state];
1540         int ns = next & TCP_STATE_MASK;
1541
1542         tcp_set_state(sk, ns);
1543
1544         return next & TCP_ACTION_FIN;
1545 }
1546
1547 /*
1548  *      Shutdown the sending side of a connection. Much like close except
1549  *      that we don't receive shut down or set_sock_flag(sk, SOCK_DEAD).
1550  */
1551
1552 void tcp_shutdown(struct sock *sk, int how)
1553 {
1554         /*      We need to grab some memory, and put together a FIN,
1555          *      and then put it into the queue to be sent.
1556          *              Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92.
1557          */
1558         if (!(how & SEND_SHUTDOWN))
1559                 return;
1560
1561         /* If we've already sent a FIN, or it's a closed state, skip this. */
1562         if ((1 << sk->sk_state) &
1563             (TCPF_ESTABLISHED | TCPF_SYN_SENT |
1564              TCPF_SYN_RECV | TCPF_CLOSE_WAIT)) {
1565                 /* Clear out any half completed packets.  FIN if needed. */
1566                 if (tcp_close_state(sk))
1567                         tcp_send_fin(sk);
1568         }
1569 }
1570
1571 /*
1572  * At this point, there should be no process reference to this
1573  * socket, and thus no user references at all.  Therefore we
1574  * can assume the socket waitqueue is inactive and nobody will
1575  * try to jump onto it.
1576  */
1577 void tcp_destroy_sock(struct sock *sk)
1578 {
1579         BUG_TRAP(sk->sk_state == TCP_CLOSE);
1580         BUG_TRAP(sock_flag(sk, SOCK_DEAD));
1581
1582         /* It cannot be in hash table! */
1583         BUG_TRAP(sk_unhashed(sk));
1584
1585         /* If it has not 0 inet_sk(sk)->num, it must be bound */
1586         BUG_TRAP(!inet_sk(sk)->num || tcp_sk(sk)->bind_hash);
1587
1588         sk->sk_prot->destroy(sk);
1589
1590         sk_stream_kill_queues(sk);
1591
1592         xfrm_sk_free_policy(sk);
1593
1594 #ifdef INET_REFCNT_DEBUG
1595         if (atomic_read(&sk->sk_refcnt) != 1) {
1596                 printk(KERN_DEBUG "Destruction TCP %p delayed, c=%d\n",
1597                        sk, atomic_read(&sk->sk_refcnt));
1598         }
1599 #endif
1600
1601         atomic_dec(&tcp_orphan_count);
1602         sock_put(sk);
1603 }
1604
1605 void tcp_close(struct sock *sk, long timeout)
1606 {
1607         struct sk_buff *skb;
1608         int data_was_unread = 0;
1609
1610         lock_sock(sk);
1611         sk->sk_shutdown = SHUTDOWN_MASK;
1612
1613         if (sk->sk_state == TCP_LISTEN) {
1614                 tcp_set_state(sk, TCP_CLOSE);
1615
1616                 /* Special case. */
1617                 tcp_listen_stop(sk);
1618
1619                 goto adjudge_to_death;
1620         }
1621
1622         /*  We need to flush the recv. buffs.  We do this only on the
1623          *  descriptor close, not protocol-sourced closes, because the
1624          *  reader process may not have drained the data yet!
1625          */
1626         while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
1627                 u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq -
1628                           skb->h.th->fin;
1629                 data_was_unread += len;
1630                 __kfree_skb(skb);
1631         }
1632
1633         sk_stream_mem_reclaim(sk);
1634
1635         /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section
1636          * 3.10, we send a RST here because data was lost.  To
1637          * witness the awful effects of the old behavior of always
1638          * doing a FIN, run an older 2.1.x kernel or 2.0.x, start
1639          * a bulk GET in an FTP client, suspend the process, wait
1640          * for the client to advertise a zero window, then kill -9
1641          * the FTP client, wheee...  Note: timeout is always zero
1642          * in such a case.
1643          */
1644         if (data_was_unread) {
1645                 /* Unread data was tossed, zap the connection. */
1646                 NET_INC_STATS_USER(LINUX_MIB_TCPABORTONCLOSE);
1647                 tcp_set_state(sk, TCP_CLOSE);
1648                 tcp_send_active_reset(sk, GFP_KERNEL);
1649         } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
1650                 /* Check zero linger _after_ checking for unread data. */
1651                 sk->sk_prot->disconnect(sk, 0);
1652                 NET_INC_STATS_USER(LINUX_MIB_TCPABORTONDATA);
1653         } else if (tcp_close_state(sk)) {
1654                 /* We FIN if the application ate all the data before
1655                  * zapping the connection.
1656                  */
1657
1658                 /* RED-PEN. Formally speaking, we have broken TCP state
1659                  * machine. State transitions:
1660                  *
1661                  * TCP_ESTABLISHED -> TCP_FIN_WAIT1
1662                  * TCP_SYN_RECV -> TCP_FIN_WAIT1 (forget it, it's impossible)
1663                  * TCP_CLOSE_WAIT -> TCP_LAST_ACK
1664                  *
1665                  * are legal only when FIN has been sent (i.e. in window),
1666                  * rather than queued out of window. Purists blame.
1667                  *
1668                  * F.e. "RFC state" is ESTABLISHED,
1669                  * if Linux state is FIN-WAIT-1, but FIN is still not sent.
1670                  *
1671                  * The visible declinations are that sometimes
1672                  * we enter time-wait state, when it is not required really
1673                  * (harmless), do not send active resets, when they are
1674                  * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when
1675                  * they look as CLOSING or LAST_ACK for Linux)
1676                  * Probably, I missed some more holelets.
1677                  *                                              --ANK
1678                  */
1679                 tcp_send_fin(sk);
1680         }
1681
1682         sk_stream_wait_close(sk, timeout);
1683
1684 adjudge_to_death:
1685         /* It is the last release_sock in its life. It will remove backlog. */
1686         release_sock(sk);
1687
1688
1689         /* Now socket is owned by kernel and we acquire BH lock
1690            to finish close. No need to check for user refs.
1691          */
1692         local_bh_disable();
1693         bh_lock_sock(sk);
1694         BUG_TRAP(!sock_owned_by_user(sk));
1695
1696         sock_hold(sk);
1697         sock_orphan(sk);
1698
1699         /*      This is a (useful) BSD violating of the RFC. There is a
1700          *      problem with TCP as specified in that the other end could
1701          *      keep a socket open forever with no application left this end.
1702          *      We use a 3 minute timeout (about the same as BSD) then kill
1703          *      our end. If they send after that then tough - BUT: long enough
1704          *      that we won't make the old 4*rto = almost no time - whoops
1705          *      reset mistake.
1706          *
1707          *      Nope, it was not mistake. It is really desired behaviour
1708          *      f.e. on http servers, when such sockets are useless, but
1709          *      consume significant resources. Let's do it with special
1710          *      linger2 option.                                 --ANK
1711          */
1712
1713         if (sk->sk_state == TCP_FIN_WAIT2) {
1714                 struct tcp_sock *tp = tcp_sk(sk);
1715                 if (tp->linger2 < 0) {
1716                         tcp_set_state(sk, TCP_CLOSE);
1717                         tcp_send_active_reset(sk, GFP_ATOMIC);
1718                         NET_INC_STATS_BH(LINUX_MIB_TCPABORTONLINGER);
1719                 } else {
1720                         int tmo = tcp_fin_time(tp);
1721
1722                         if (tmo > TCP_TIMEWAIT_LEN) {
1723                                 tcp_reset_keepalive_timer(sk, tcp_fin_time(tp));
1724                         } else {
1725                                 atomic_inc(&tcp_orphan_count);
1726                                 tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
1727                                 goto out;
1728                         }
1729                 }
1730         }
1731         if (sk->sk_state != TCP_CLOSE) {
1732                 sk_stream_mem_reclaim(sk);
1733                 if (atomic_read(&tcp_orphan_count) > sysctl_tcp_max_orphans ||
1734                     (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&
1735                      atomic_read(&tcp_memory_allocated) > sysctl_tcp_mem[2])) {
1736                         if (net_ratelimit())
1737                                 printk(KERN_INFO "TCP: too many of orphaned "
1738                                        "sockets\n");
1739                         tcp_set_state(sk, TCP_CLOSE);
1740                         tcp_send_active_reset(sk, GFP_ATOMIC);
1741                         NET_INC_STATS_BH(LINUX_MIB_TCPABORTONMEMORY);
1742                 }
1743         }
1744         atomic_inc(&tcp_orphan_count);
1745
1746         if (sk->sk_state == TCP_CLOSE)
1747                 tcp_destroy_sock(sk);
1748         /* Otherwise, socket is reprieved until protocol close. */
1749
1750 out:
1751         bh_unlock_sock(sk);
1752         local_bh_enable();
1753         sock_put(sk);
1754 }
1755
1756 /* These states need RST on ABORT according to RFC793 */
1757
1758 static inline int tcp_need_reset(int state)
1759 {
1760         return (1 << state) &
1761                (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 |
1762                 TCPF_FIN_WAIT2 | TCPF_SYN_RECV);
1763 }
1764
1765 int tcp_disconnect(struct sock *sk, int flags)
1766 {
1767         struct inet_sock *inet = inet_sk(sk);
1768         struct tcp_sock *tp = tcp_sk(sk);
1769         int err = 0;
1770         int old_state = sk->sk_state;
1771
1772         if (old_state != TCP_CLOSE)
1773                 tcp_set_state(sk, TCP_CLOSE);
1774
1775         /* ABORT function of RFC793 */
1776         if (old_state == TCP_LISTEN) {
1777                 tcp_listen_stop(sk);
1778         } else if (tcp_need_reset(old_state) ||
1779                    (tp->snd_nxt != tp->write_seq &&
1780                     (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK))) {
1781                 /* The last check adjusts for discrepance of Linux wrt. RFC
1782                  * states
1783                  */
1784                 tcp_send_active_reset(sk, gfp_any());
1785                 sk->sk_err = ECONNRESET;
1786         } else if (old_state == TCP_SYN_SENT)
1787                 sk->sk_err = ECONNRESET;
1788
1789         tcp_clear_xmit_timers(sk);
1790         __skb_queue_purge(&sk->sk_receive_queue);
1791         sk_stream_writequeue_purge(sk);
1792         __skb_queue_purge(&tp->out_of_order_queue);
1793
1794         inet->dport = 0;
1795
1796         if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK))
1797                 inet_reset_saddr(sk);
1798
1799         sk->sk_shutdown = 0;
1800         sock_reset_flag(sk, SOCK_DONE);
1801         tp->srtt = 0;
1802         if ((tp->write_seq += tp->max_window + 2) == 0)
1803                 tp->write_seq = 1;
1804         tp->backoff = 0;
1805         tp->snd_cwnd = 2;
1806         tp->probes_out = 0;
1807         tp->packets_out = 0;
1808         tp->snd_ssthresh = 0x7fffffff;
1809         tp->snd_cwnd_cnt = 0;
1810         tcp_set_ca_state(tp, TCP_CA_Open);
1811         tcp_clear_retrans(tp);
1812         tcp_delack_init(tp);
1813         sk->sk_send_head = NULL;
1814         tp->rx_opt.saw_tstamp = 0;
1815         tcp_sack_reset(&tp->rx_opt);
1816         __sk_dst_reset(sk);
1817
1818         BUG_TRAP(!inet->num || tp->bind_hash);
1819
1820         sk->sk_error_report(sk);
1821         return err;
1822 }
1823
1824 /*
1825  *      Wait for an incoming connection, avoid race
1826  *      conditions. This must be called with the socket locked.
1827  */
1828 static int wait_for_connect(struct sock *sk, long timeo)
1829 {
1830         struct tcp_sock *tp = tcp_sk(sk);
1831         DEFINE_WAIT(wait);
1832         int err;
1833
1834         /*
1835          * True wake-one mechanism for incoming connections: only
1836          * one process gets woken up, not the 'whole herd'.
1837          * Since we do not 'race & poll' for established sockets
1838          * anymore, the common case will execute the loop only once.
1839          *
1840          * Subtle issue: "add_wait_queue_exclusive()" will be added
1841          * after any current non-exclusive waiters, and we know that
1842          * it will always _stay_ after any new non-exclusive waiters
1843          * because all non-exclusive waiters are added at the
1844          * beginning of the wait-queue. As such, it's ok to "drop"
1845          * our exclusiveness temporarily when we get woken up without
1846          * having to remove and re-insert us on the wait queue.
1847          */
1848         for (;;) {
1849                 prepare_to_wait_exclusive(sk->sk_sleep, &wait,
1850                                           TASK_INTERRUPTIBLE);
1851                 release_sock(sk);
1852                 if (reqsk_queue_empty(&tp->accept_queue))
1853                         timeo = schedule_timeout(timeo);
1854                 lock_sock(sk);
1855                 err = 0;
1856                 if (!reqsk_queue_empty(&tp->accept_queue))
1857                         break;
1858                 err = -EINVAL;
1859                 if (sk->sk_state != TCP_LISTEN)
1860                         break;
1861                 err = sock_intr_errno(timeo);
1862                 if (signal_pending(current))
1863                         break;
1864                 err = -EAGAIN;
1865                 if (!timeo)
1866                         break;
1867         }
1868         finish_wait(sk->sk_sleep, &wait);
1869         return err;
1870 }
1871
1872 /*
1873  *      This will accept the next outstanding connection.
1874  */
1875
1876 struct sock *tcp_accept(struct sock *sk, int flags, int *err)
1877 {
1878         struct tcp_sock *tp = tcp_sk(sk);
1879         struct sock *newsk;
1880         int error;
1881
1882         lock_sock(sk);
1883
1884         /* We need to make sure that this socket is listening,
1885          * and that it has something pending.
1886          */
1887         error = -EINVAL;
1888         if (sk->sk_state != TCP_LISTEN)
1889                 goto out_err;
1890
1891         /* Find already established connection */
1892         if (reqsk_queue_empty(&tp->accept_queue)) {
1893                 long timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
1894
1895                 /* If this is a non blocking socket don't sleep */
1896                 error = -EAGAIN;
1897                 if (!timeo)
1898                         goto out_err;
1899
1900                 error = wait_for_connect(sk, timeo);
1901                 if (error)
1902                         goto out_err;
1903         }
1904
1905         newsk = reqsk_queue_get_child(&tp->accept_queue, sk);
1906         BUG_TRAP(newsk->sk_state != TCP_SYN_RECV);
1907 out:
1908         release_sock(sk);
1909         return newsk;
1910 out_err:
1911         newsk = NULL;
1912         *err = error;
1913         goto out;
1914 }
1915
1916 /*
1917  *      Socket option code for TCP.
1918  */
1919 int tcp_setsockopt(struct sock *sk, int level, int optname, char __user *optval,
1920                    int optlen)
1921 {
1922         struct tcp_sock *tp = tcp_sk(sk);
1923         int val;
1924         int err = 0;
1925
1926         if (level != SOL_TCP)
1927                 return tp->af_specific->setsockopt(sk, level, optname,
1928                                                    optval, optlen);
1929
1930         if (optlen < sizeof(int))
1931                 return -EINVAL;
1932
1933         if (get_user(val, (int __user *)optval))
1934                 return -EFAULT;
1935
1936         lock_sock(sk);
1937
1938         switch (optname) {
1939         case TCP_MAXSEG:
1940                 /* Values greater than interface MTU won't take effect. However
1941                  * at the point when this call is done we typically don't yet
1942                  * know which interface is going to be used */
1943                 if (val < 8 || val > MAX_TCP_WINDOW) {
1944                         err = -EINVAL;
1945                         break;
1946                 }
1947                 tp->rx_opt.user_mss = val;
1948                 break;
1949
1950         case TCP_NODELAY:
1951                 if (val) {
1952                         /* TCP_NODELAY is weaker than TCP_CORK, so that
1953                          * this option on corked socket is remembered, but
1954                          * it is not activated until cork is cleared.
1955                          *
1956                          * However, when TCP_NODELAY is set we make
1957                          * an explicit push, which overrides even TCP_CORK
1958                          * for currently queued segments.
1959                          */
1960                         tp->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH;
1961                         tcp_push_pending_frames(sk, tp);
1962                 } else {
1963                         tp->nonagle &= ~TCP_NAGLE_OFF;
1964                 }
1965                 break;
1966
1967         case TCP_CORK:
1968                 /* When set indicates to always queue non-full frames.
1969                  * Later the user clears this option and we transmit
1970                  * any pending partial frames in the queue.  This is
1971                  * meant to be used alongside sendfile() to get properly
1972                  * filled frames when the user (for example) must write
1973                  * out headers with a write() call first and then use
1974                  * sendfile to send out the data parts.
1975                  *
1976                  * TCP_CORK can be set together with TCP_NODELAY and it is
1977                  * stronger than TCP_NODELAY.
1978                  */
1979                 if (val) {
1980                         tp->nonagle |= TCP_NAGLE_CORK;
1981                 } else {
1982                         tp->nonagle &= ~TCP_NAGLE_CORK;
1983                         if (tp->nonagle&TCP_NAGLE_OFF)
1984                                 tp->nonagle |= TCP_NAGLE_PUSH;
1985                         tcp_push_pending_frames(sk, tp);
1986                 }
1987                 break;
1988
1989         case TCP_KEEPIDLE:
1990                 if (val < 1 || val > MAX_TCP_KEEPIDLE)
1991                         err = -EINVAL;
1992                 else {
1993                         tp->keepalive_time = val * HZ;
1994                         if (sock_flag(sk, SOCK_KEEPOPEN) &&
1995                             !((1 << sk->sk_state) &
1996                               (TCPF_CLOSE | TCPF_LISTEN))) {
1997                                 __u32 elapsed = tcp_time_stamp - tp->rcv_tstamp;
1998                                 if (tp->keepalive_time > elapsed)
1999                                         elapsed = tp->keepalive_time - elapsed;
2000                                 else
2001                                         elapsed = 0;
2002                                 tcp_reset_keepalive_timer(sk, elapsed);
2003                         }
2004                 }
2005                 break;
2006         case TCP_KEEPINTVL:
2007                 if (val < 1 || val > MAX_TCP_KEEPINTVL)
2008                         err = -EINVAL;
2009                 else
2010                         tp->keepalive_intvl = val * HZ;
2011                 break;
2012         case TCP_KEEPCNT:
2013                 if (val < 1 || val > MAX_TCP_KEEPCNT)
2014                         err = -EINVAL;
2015                 else
2016                         tp->keepalive_probes = val;
2017                 break;
2018         case TCP_SYNCNT:
2019                 if (val < 1 || val > MAX_TCP_SYNCNT)
2020                         err = -EINVAL;
2021                 else
2022                         tp->syn_retries = val;
2023                 break;
2024
2025         case TCP_LINGER2:
2026                 if (val < 0)
2027                         tp->linger2 = -1;
2028                 else if (val > sysctl_tcp_fin_timeout / HZ)
2029                         tp->linger2 = 0;
2030                 else
2031                         tp->linger2 = val * HZ;
2032                 break;
2033
2034         case TCP_DEFER_ACCEPT:
2035                 tp->defer_accept = 0;
2036                 if (val > 0) {
2037                         /* Translate value in seconds to number of
2038                          * retransmits */
2039                         while (tp->defer_accept < 32 &&
2040                                val > ((TCP_TIMEOUT_INIT / HZ) <<
2041                                        tp->defer_accept))
2042                                 tp->defer_accept++;
2043                         tp->defer_accept++;
2044                 }
2045                 break;
2046
2047         case TCP_WINDOW_CLAMP:
2048                 if (!val) {
2049                         if (sk->sk_state != TCP_CLOSE) {
2050                                 err = -EINVAL;
2051                                 break;
2052                         }
2053                         tp->window_clamp = 0;
2054                 } else
2055                         tp->window_clamp = val < SOCK_MIN_RCVBUF / 2 ?
2056                                                 SOCK_MIN_RCVBUF / 2 : val;
2057                 break;
2058
2059         case TCP_QUICKACK:
2060                 if (!val) {
2061                         tp->ack.pingpong = 1;
2062                 } else {
2063                         tp->ack.pingpong = 0;
2064                         if ((1 << sk->sk_state) &
2065                             (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) &&
2066                             tcp_ack_scheduled(tp)) {
2067                                 tp->ack.pending |= TCP_ACK_PUSHED;
2068                                 cleanup_rbuf(sk, 1);
2069                                 if (!(val & 1))
2070                                         tp->ack.pingpong = 1;
2071                         }
2072                 }
2073                 break;
2074
2075         default:
2076                 err = -ENOPROTOOPT;
2077                 break;
2078         };
2079         release_sock(sk);
2080         return err;
2081 }
2082
2083 /* Return information about state of tcp endpoint in API format. */
2084 void tcp_get_info(struct sock *sk, struct tcp_info *info)
2085 {
2086         struct tcp_sock *tp = tcp_sk(sk);
2087         u32 now = tcp_time_stamp;
2088
2089         memset(info, 0, sizeof(*info));
2090
2091         info->tcpi_state = sk->sk_state;
2092         info->tcpi_ca_state = tp->ca_state;
2093         info->tcpi_retransmits = tp->retransmits;
2094         info->tcpi_probes = tp->probes_out;
2095         info->tcpi_backoff = tp->backoff;
2096
2097         if (tp->rx_opt.tstamp_ok)
2098                 info->tcpi_options |= TCPI_OPT_TIMESTAMPS;
2099         if (tp->rx_opt.sack_ok)
2100                 info->tcpi_options |= TCPI_OPT_SACK;
2101         if (tp->rx_opt.wscale_ok) {
2102                 info->tcpi_options |= TCPI_OPT_WSCALE;
2103                 info->tcpi_snd_wscale = tp->rx_opt.snd_wscale;
2104                 info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale;
2105         }
2106
2107         if (tp->ecn_flags&TCP_ECN_OK)
2108                 info->tcpi_options |= TCPI_OPT_ECN;
2109
2110         info->tcpi_rto = jiffies_to_usecs(tp->rto);
2111         info->tcpi_ato = jiffies_to_usecs(tp->ack.ato);
2112         info->tcpi_snd_mss = tp->mss_cache_std;
2113         info->tcpi_rcv_mss = tp->ack.rcv_mss;
2114
2115         info->tcpi_unacked = tp->packets_out;
2116         info->tcpi_sacked = tp->sacked_out;
2117         info->tcpi_lost = tp->lost_out;
2118         info->tcpi_retrans = tp->retrans_out;
2119         info->tcpi_fackets = tp->fackets_out;
2120
2121         info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime);
2122         info->tcpi_last_data_recv = jiffies_to_msecs(now - tp->ack.lrcvtime);
2123         info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp);
2124
2125         info->tcpi_pmtu = tp->pmtu_cookie;
2126         info->tcpi_rcv_ssthresh = tp->rcv_ssthresh;
2127         info->tcpi_rtt = jiffies_to_usecs(tp->srtt)>>3;
2128         info->tcpi_rttvar = jiffies_to_usecs(tp->mdev)>>2;
2129         info->tcpi_snd_ssthresh = tp->snd_ssthresh;
2130         info->tcpi_snd_cwnd = tp->snd_cwnd;
2131         info->tcpi_advmss = tp->advmss;
2132         info->tcpi_reordering = tp->reordering;
2133
2134         info->tcpi_rcv_rtt = jiffies_to_usecs(tp->rcv_rtt_est.rtt)>>3;
2135         info->tcpi_rcv_space = tp->rcvq_space.space;
2136
2137         info->tcpi_total_retrans = tp->total_retrans;
2138 }
2139
2140 EXPORT_SYMBOL_GPL(tcp_get_info);
2141
2142 int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
2143                    int __user *optlen)
2144 {
2145         struct tcp_sock *tp = tcp_sk(sk);
2146         int val, len;
2147
2148         if (level != SOL_TCP)
2149                 return tp->af_specific->getsockopt(sk, level, optname,
2150                                                    optval, optlen);
2151
2152         if (get_user(len, optlen))
2153                 return -EFAULT;
2154
2155         len = min_t(unsigned int, len, sizeof(int));
2156
2157         if (len < 0)
2158                 return -EINVAL;
2159
2160         switch (optname) {
2161         case TCP_MAXSEG:
2162                 val = tp->mss_cache_std;
2163                 if (!val && ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
2164                         val = tp->rx_opt.user_mss;
2165                 break;
2166         case TCP_NODELAY:
2167                 val = !!(tp->nonagle&TCP_NAGLE_OFF);
2168                 break;
2169         case TCP_CORK:
2170                 val = !!(tp->nonagle&TCP_NAGLE_CORK);
2171                 break;
2172         case TCP_KEEPIDLE:
2173                 val = (tp->keepalive_time ? : sysctl_tcp_keepalive_time) / HZ;
2174                 break;
2175         case TCP_KEEPINTVL:
2176                 val = (tp->keepalive_intvl ? : sysctl_tcp_keepalive_intvl) / HZ;
2177                 break;
2178         case TCP_KEEPCNT:
2179                 val = tp->keepalive_probes ? : sysctl_tcp_keepalive_probes;
2180                 break;
2181         case TCP_SYNCNT:
2182                 val = tp->syn_retries ? : sysctl_tcp_syn_retries;
2183                 break;
2184         case TCP_LINGER2:
2185                 val = tp->linger2;
2186                 if (val >= 0)
2187                         val = (val ? : sysctl_tcp_fin_timeout) / HZ;
2188                 break;
2189         case TCP_DEFER_ACCEPT:
2190                 val = !tp->defer_accept ? 0 : ((TCP_TIMEOUT_INIT / HZ) <<
2191                                                (tp->defer_accept - 1));
2192                 break;
2193         case TCP_WINDOW_CLAMP:
2194                 val = tp->window_clamp;
2195                 break;
2196         case TCP_INFO: {
2197                 struct tcp_info info;
2198
2199                 if (get_user(len, optlen))
2200                         return -EFAULT;
2201
2202                 tcp_get_info(sk, &info);
2203
2204                 len = min_t(unsigned int, len, sizeof(info));
2205                 if (put_user(len, optlen))
2206                         return -EFAULT;
2207                 if (copy_to_user(optval, &info, len))
2208                         return -EFAULT;
2209                 return 0;
2210         }
2211         case TCP_QUICKACK:
2212                 val = !tp->ack.pingpong;
2213                 break;
2214         default:
2215                 return -ENOPROTOOPT;
2216         };
2217
2218         if (put_user(len, optlen))
2219                 return -EFAULT;
2220         if (copy_to_user(optval, &val, len))
2221                 return -EFAULT;
2222         return 0;
2223 }
2224
2225
2226 extern void __skb_cb_too_small_for_tcp(int, int);
2227 extern void tcpdiag_init(void);
2228
2229 static __initdata unsigned long thash_entries;
2230 static int __init set_thash_entries(char *str)
2231 {
2232         if (!str)
2233                 return 0;
2234         thash_entries = simple_strtoul(str, &str, 0);
2235         return 1;
2236 }
2237 __setup("thash_entries=", set_thash_entries);
2238
2239 void __init tcp_init(void)
2240 {
2241         struct sk_buff *skb = NULL;
2242         int order, i;
2243
2244         if (sizeof(struct tcp_skb_cb) > sizeof(skb->cb))
2245                 __skb_cb_too_small_for_tcp(sizeof(struct tcp_skb_cb),
2246                                            sizeof(skb->cb));
2247
2248         tcp_bucket_cachep = kmem_cache_create("tcp_bind_bucket",
2249                                               sizeof(struct tcp_bind_bucket),
2250                                               0, SLAB_HWCACHE_ALIGN,
2251                                               NULL, NULL);
2252         if (!tcp_bucket_cachep)
2253                 panic("tcp_init: Cannot alloc tcp_bind_bucket cache.");
2254
2255         tcp_timewait_cachep = kmem_cache_create("tcp_tw_bucket",
2256                                                 sizeof(struct tcp_tw_bucket),
2257                                                 0, SLAB_HWCACHE_ALIGN,
2258                                                 NULL, NULL);
2259         if (!tcp_timewait_cachep)
2260                 panic("tcp_init: Cannot alloc tcp_tw_bucket cache.");
2261
2262         /* Size and allocate the main established and bind bucket
2263          * hash tables.
2264          *
2265          * The methodology is similar to that of the buffer cache.
2266          */
2267         tcp_ehash = (struct tcp_ehash_bucket *)
2268                 alloc_large_system_hash("TCP established",
2269                                         sizeof(struct tcp_ehash_bucket),
2270                                         thash_entries,
2271                                         (num_physpages >= 128 * 1024) ?
2272                                                 (25 - PAGE_SHIFT) :
2273                                                 (27 - PAGE_SHIFT),
2274                                         HASH_HIGHMEM,
2275                                         &tcp_ehash_size,
2276                                         NULL,
2277                                         0);
2278         tcp_ehash_size = (1 << tcp_ehash_size) >> 1;
2279         for (i = 0; i < (tcp_ehash_size << 1); i++) {
2280                 rwlock_init(&tcp_ehash[i].lock);
2281                 INIT_HLIST_HEAD(&tcp_ehash[i].chain);
2282         }
2283
2284         tcp_bhash = (struct tcp_bind_hashbucket *)
2285                 alloc_large_system_hash("TCP bind",
2286                                         sizeof(struct tcp_bind_hashbucket),
2287                                         tcp_ehash_size,
2288                                         (num_physpages >= 128 * 1024) ?
2289                                                 (25 - PAGE_SHIFT) :
2290                                                 (27 - PAGE_SHIFT),
2291                                         HASH_HIGHMEM,
2292                                         &tcp_bhash_size,
2293                                         NULL,
2294                                         64 * 1024);
2295         tcp_bhash_size = 1 << tcp_bhash_size;
2296         for (i = 0; i < tcp_bhash_size; i++) {
2297                 spin_lock_init(&tcp_bhash[i].lock);
2298                 INIT_HLIST_HEAD(&tcp_bhash[i].chain);
2299         }
2300
2301         /* Try to be a bit smarter and adjust defaults depending
2302          * on available memory.
2303          */
2304         for (order = 0; ((1 << order) << PAGE_SHIFT) <
2305                         (tcp_bhash_size * sizeof(struct tcp_bind_hashbucket));
2306                         order++)
2307                 ;
2308         if (order >= 4) {
2309                 sysctl_local_port_range[0] = 32768;
2310                 sysctl_local_port_range[1] = 61000;
2311                 sysctl_tcp_max_tw_buckets = 180000;
2312                 sysctl_tcp_max_orphans = 4096 << (order - 4);
2313                 sysctl_max_syn_backlog = 1024;
2314         } else if (order < 3) {
2315                 sysctl_local_port_range[0] = 1024 * (3 - order);
2316                 sysctl_tcp_max_tw_buckets >>= (3 - order);
2317                 sysctl_tcp_max_orphans >>= (3 - order);
2318                 sysctl_max_syn_backlog = 128;
2319         }
2320         tcp_port_rover = sysctl_local_port_range[0] - 1;
2321
2322         sysctl_tcp_mem[0] =  768 << order;
2323         sysctl_tcp_mem[1] = 1024 << order;
2324         sysctl_tcp_mem[2] = 1536 << order;
2325
2326         if (order < 3) {
2327                 sysctl_tcp_wmem[2] = 64 * 1024;
2328                 sysctl_tcp_rmem[0] = PAGE_SIZE;
2329                 sysctl_tcp_rmem[1] = 43689;
2330                 sysctl_tcp_rmem[2] = 2 * 43689;
2331         }
2332
2333         printk(KERN_INFO "TCP: Hash tables configured "
2334                "(established %d bind %d)\n",
2335                tcp_ehash_size << 1, tcp_bhash_size);
2336 }
2337
2338 EXPORT_SYMBOL(tcp_accept);
2339 EXPORT_SYMBOL(tcp_close);
2340 EXPORT_SYMBOL(tcp_destroy_sock);
2341 EXPORT_SYMBOL(tcp_disconnect);
2342 EXPORT_SYMBOL(tcp_getsockopt);
2343 EXPORT_SYMBOL(tcp_ioctl);
2344 EXPORT_SYMBOL(tcp_poll);
2345 EXPORT_SYMBOL(tcp_read_sock);
2346 EXPORT_SYMBOL(tcp_recvmsg);
2347 EXPORT_SYMBOL(tcp_sendmsg);
2348 EXPORT_SYMBOL(tcp_sendpage);
2349 EXPORT_SYMBOL(tcp_setsockopt);
2350 EXPORT_SYMBOL(tcp_shutdown);
2351 EXPORT_SYMBOL(tcp_statistics);
2352 EXPORT_SYMBOL(tcp_timewait_cachep);