tcp: fix skb_availroom()
[pandora-kernel.git] / net / ipv4 / tcp.c
1 /*
2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
3  *              operating system.  INET is implemented using the  BSD Socket
4  *              interface as the means of communication with the user level.
5  *
6  *              Implementation of the Transmission Control Protocol(TCP).
7  *
8  * Authors:     Ross Biro
9  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10  *              Mark Evans, <evansmp@uhura.aston.ac.uk>
11  *              Corey Minyard <wf-rch!minyard@relay.EU.net>
12  *              Florian La Roche, <flla@stud.uni-sb.de>
13  *              Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
14  *              Linus Torvalds, <torvalds@cs.helsinki.fi>
15  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
16  *              Matthew Dillon, <dillon@apollo.west.oic.com>
17  *              Arnt Gulbrandsen, <agulbra@nvg.unit.no>
18  *              Jorge Cwik, <jorge@laser.satlink.net>
19  *
20  * Fixes:
21  *              Alan Cox        :       Numerous verify_area() calls
22  *              Alan Cox        :       Set the ACK bit on a reset
23  *              Alan Cox        :       Stopped it crashing if it closed while
24  *                                      sk->inuse=1 and was trying to connect
25  *                                      (tcp_err()).
26  *              Alan Cox        :       All icmp error handling was broken
27  *                                      pointers passed where wrong and the
28  *                                      socket was looked up backwards. Nobody
29  *                                      tested any icmp error code obviously.
30  *              Alan Cox        :       tcp_err() now handled properly. It
31  *                                      wakes people on errors. poll
32  *                                      behaves and the icmp error race
33  *                                      has gone by moving it into sock.c
34  *              Alan Cox        :       tcp_send_reset() fixed to work for
35  *                                      everything not just packets for
36  *                                      unknown sockets.
37  *              Alan Cox        :       tcp option processing.
38  *              Alan Cox        :       Reset tweaked (still not 100%) [Had
39  *                                      syn rule wrong]
40  *              Herp Rosmanith  :       More reset fixes
41  *              Alan Cox        :       No longer acks invalid rst frames.
42  *                                      Acking any kind of RST is right out.
43  *              Alan Cox        :       Sets an ignore me flag on an rst
44  *                                      receive otherwise odd bits of prattle
45  *                                      escape still
46  *              Alan Cox        :       Fixed another acking RST frame bug.
47  *                                      Should stop LAN workplace lockups.
48  *              Alan Cox        :       Some tidyups using the new skb list
49  *                                      facilities
50  *              Alan Cox        :       sk->keepopen now seems to work
51  *              Alan Cox        :       Pulls options out correctly on accepts
52  *              Alan Cox        :       Fixed assorted sk->rqueue->next errors
53  *              Alan Cox        :       PSH doesn't end a TCP read. Switched a
54  *                                      bit to skb ops.
55  *              Alan Cox        :       Tidied tcp_data to avoid a potential
56  *                                      nasty.
57  *              Alan Cox        :       Added some better commenting, as the
58  *                                      tcp is hard to follow
59  *              Alan Cox        :       Removed incorrect check for 20 * psh
60  *      Michael O'Reilly        :       ack < copied bug fix.
61  *      Johannes Stille         :       Misc tcp fixes (not all in yet).
62  *              Alan Cox        :       FIN with no memory -> CRASH
63  *              Alan Cox        :       Added socket option proto entries.
64  *                                      Also added awareness of them to accept.
65  *              Alan Cox        :       Added TCP options (SOL_TCP)
66  *              Alan Cox        :       Switched wakeup calls to callbacks,
67  *                                      so the kernel can layer network
68  *                                      sockets.
69  *              Alan Cox        :       Use ip_tos/ip_ttl settings.
70  *              Alan Cox        :       Handle FIN (more) properly (we hope).
71  *              Alan Cox        :       RST frames sent on unsynchronised
72  *                                      state ack error.
73  *              Alan Cox        :       Put in missing check for SYN bit.
74  *              Alan Cox        :       Added tcp_select_window() aka NET2E
75  *                                      window non shrink trick.
76  *              Alan Cox        :       Added a couple of small NET2E timer
77  *                                      fixes
78  *              Charles Hedrick :       TCP fixes
79  *              Toomas Tamm     :       TCP window fixes
80  *              Alan Cox        :       Small URG fix to rlogin ^C ack fight
81  *              Charles Hedrick :       Rewrote most of it to actually work
82  *              Linus           :       Rewrote tcp_read() and URG handling
83  *                                      completely
84  *              Gerhard Koerting:       Fixed some missing timer handling
85  *              Matthew Dillon  :       Reworked TCP machine states as per RFC
86  *              Gerhard Koerting:       PC/TCP workarounds
87  *              Adam Caldwell   :       Assorted timer/timing errors
88  *              Matthew Dillon  :       Fixed another RST bug
89  *              Alan Cox        :       Move to kernel side addressing changes.
90  *              Alan Cox        :       Beginning work on TCP fastpathing
91  *                                      (not yet usable)
92  *              Arnt Gulbrandsen:       Turbocharged tcp_check() routine.
93  *              Alan Cox        :       TCP fast path debugging
94  *              Alan Cox        :       Window clamping
95  *              Michael Riepe   :       Bug in tcp_check()
96  *              Matt Dillon     :       More TCP improvements and RST bug fixes
97  *              Matt Dillon     :       Yet more small nasties remove from the
98  *                                      TCP code (Be very nice to this man if
99  *                                      tcp finally works 100%) 8)
100  *              Alan Cox        :       BSD accept semantics.
101  *              Alan Cox        :       Reset on closedown bug.
102  *      Peter De Schrijver      :       ENOTCONN check missing in tcp_sendto().
103  *              Michael Pall    :       Handle poll() after URG properly in
104  *                                      all cases.
105  *              Michael Pall    :       Undo the last fix in tcp_read_urg()
106  *                                      (multi URG PUSH broke rlogin).
107  *              Michael Pall    :       Fix the multi URG PUSH problem in
108  *                                      tcp_readable(), poll() after URG
109  *                                      works now.
110  *              Michael Pall    :       recv(...,MSG_OOB) never blocks in the
111  *                                      BSD api.
112  *              Alan Cox        :       Changed the semantics of sk->socket to
113  *                                      fix a race and a signal problem with
114  *                                      accept() and async I/O.
115  *              Alan Cox        :       Relaxed the rules on tcp_sendto().
116  *              Yury Shevchuk   :       Really fixed accept() blocking problem.
117  *              Craig I. Hagan  :       Allow for BSD compatible TIME_WAIT for
118  *                                      clients/servers which listen in on
119  *                                      fixed ports.
120  *              Alan Cox        :       Cleaned the above up and shrank it to
121  *                                      a sensible code size.
122  *              Alan Cox        :       Self connect lockup fix.
123  *              Alan Cox        :       No connect to multicast.
124  *              Ross Biro       :       Close unaccepted children on master
125  *                                      socket close.
126  *              Alan Cox        :       Reset tracing code.
127  *              Alan Cox        :       Spurious resets on shutdown.
128  *              Alan Cox        :       Giant 15 minute/60 second timer error
129  *              Alan Cox        :       Small whoops in polling before an
130  *                                      accept.
131  *              Alan Cox        :       Kept the state trace facility since
132  *                                      it's handy for debugging.
133  *              Alan Cox        :       More reset handler fixes.
134  *              Alan Cox        :       Started rewriting the code based on
135  *                                      the RFC's for other useful protocol
136  *                                      references see: Comer, KA9Q NOS, and
137  *                                      for a reference on the difference
138  *                                      between specifications and how BSD
139  *                                      works see the 4.4lite source.
140  *              A.N.Kuznetsov   :       Don't time wait on completion of tidy
141  *                                      close.
142  *              Linus Torvalds  :       Fin/Shutdown & copied_seq changes.
143  *              Linus Torvalds  :       Fixed BSD port reuse to work first syn
144  *              Alan Cox        :       Reimplemented timers as per the RFC
145  *                                      and using multiple timers for sanity.
146  *              Alan Cox        :       Small bug fixes, and a lot of new
147  *                                      comments.
148  *              Alan Cox        :       Fixed dual reader crash by locking
149  *                                      the buffers (much like datagram.c)
150  *              Alan Cox        :       Fixed stuck sockets in probe. A probe
151  *                                      now gets fed up of retrying without
152  *                                      (even a no space) answer.
153  *              Alan Cox        :       Extracted closing code better
154  *              Alan Cox        :       Fixed the closing state machine to
155  *                                      resemble the RFC.
156  *              Alan Cox        :       More 'per spec' fixes.
157  *              Jorge Cwik      :       Even faster checksumming.
158  *              Alan Cox        :       tcp_data() doesn't ack illegal PSH
159  *                                      only frames. At least one pc tcp stack
160  *                                      generates them.
161  *              Alan Cox        :       Cache last socket.
162  *              Alan Cox        :       Per route irtt.
163  *              Matt Day        :       poll()->select() match BSD precisely on error
164  *              Alan Cox        :       New buffers
165  *              Marc Tamsky     :       Various sk->prot->retransmits and
166  *                                      sk->retransmits misupdating fixed.
167  *                                      Fixed tcp_write_timeout: stuck close,
168  *                                      and TCP syn retries gets used now.
169  *              Mark Yarvis     :       In tcp_read_wakeup(), don't send an
170  *                                      ack if state is TCP_CLOSED.
171  *              Alan Cox        :       Look up device on a retransmit - routes may
172  *                                      change. Doesn't yet cope with MSS shrink right
173  *                                      but it's a start!
174  *              Marc Tamsky     :       Closing in closing fixes.
175  *              Mike Shaver     :       RFC1122 verifications.
176  *              Alan Cox        :       rcv_saddr errors.
177  *              Alan Cox        :       Block double connect().
178  *              Alan Cox        :       Small hooks for enSKIP.
179  *              Alexey Kuznetsov:       Path MTU discovery.
180  *              Alan Cox        :       Support soft errors.
181  *              Alan Cox        :       Fix MTU discovery pathological case
182  *                                      when the remote claims no mtu!
183  *              Marc Tamsky     :       TCP_CLOSE fix.
184  *              Colin (G3TNE)   :       Send a reset on syn ack replies in
185  *                                      window but wrong (fixes NT lpd problems)
186  *              Pedro Roque     :       Better TCP window handling, delayed ack.
187  *              Joerg Reuter    :       No modification of locked buffers in
188  *                                      tcp_do_retransmit()
189  *              Eric Schenk     :       Changed receiver side silly window
190  *                                      avoidance algorithm to BSD style
191  *                                      algorithm. This doubles throughput
192  *                                      against machines running Solaris,
193  *                                      and seems to result in general
194  *                                      improvement.
195  *      Stefan Magdalinski      :       adjusted tcp_readable() to fix FIONREAD
196  *      Willy Konynenberg       :       Transparent proxying support.
197  *      Mike McLagan            :       Routing by source
198  *              Keith Owens     :       Do proper merging with partial SKB's in
199  *                                      tcp_do_sendmsg to avoid burstiness.
200  *              Eric Schenk     :       Fix fast close down bug with
201  *                                      shutdown() followed by close().
202  *              Andi Kleen      :       Make poll agree with SIGIO
203  *      Salvatore Sanfilippo    :       Support SO_LINGER with linger == 1 and
204  *                                      lingertime == 0 (RFC 793 ABORT Call)
205  *      Hirokazu Takahashi      :       Use copy_from_user() instead of
206  *                                      csum_and_copy_from_user() if possible.
207  *
208  *              This program is free software; you can redistribute it and/or
209  *              modify it under the terms of the GNU General Public License
210  *              as published by the Free Software Foundation; either version
211  *              2 of the License, or(at your option) any later version.
212  *
213  * Description of States:
214  *
215  *      TCP_SYN_SENT            sent a connection request, waiting for ack
216  *
217  *      TCP_SYN_RECV            received a connection request, sent ack,
218  *                              waiting for final ack in three-way handshake.
219  *
220  *      TCP_ESTABLISHED         connection established
221  *
222  *      TCP_FIN_WAIT1           our side has shutdown, waiting to complete
223  *                              transmission of remaining buffered data
224  *
225  *      TCP_FIN_WAIT2           all buffered data sent, waiting for remote
226  *                              to shutdown
227  *
228  *      TCP_CLOSING             both sides have shutdown but we still have
229  *                              data we have to finish sending
230  *
231  *      TCP_TIME_WAIT           timeout to catch resent junk before entering
232  *                              closed, can only be entered from FIN_WAIT2
233  *                              or CLOSING.  Required because the other end
234  *                              may not have gotten our last ACK causing it
235  *                              to retransmit the data packet (which we ignore)
236  *
237  *      TCP_CLOSE_WAIT          remote side has shutdown and is waiting for
238  *                              us to finish writing our data and to shutdown
239  *                              (we have to close() to move on to LAST_ACK)
240  *
241  *      TCP_LAST_ACK            out side has shutdown after remote has
242  *                              shutdown.  There may still be data in our
243  *                              buffer that we have to finish sending
244  *
245  *      TCP_CLOSE               socket is finished
246  */
247
248 #include <linux/kernel.h>
249 #include <linux/module.h>
250 #include <linux/types.h>
251 #include <linux/fcntl.h>
252 #include <linux/poll.h>
253 #include <linux/init.h>
254 #include <linux/fs.h>
255 #include <linux/skbuff.h>
256 #include <linux/scatterlist.h>
257 #include <linux/splice.h>
258 #include <linux/net.h>
259 #include <linux/socket.h>
260 #include <linux/random.h>
261 #include <linux/bootmem.h>
262 #include <linux/highmem.h>
263 #include <linux/swap.h>
264 #include <linux/cache.h>
265 #include <linux/err.h>
266 #include <linux/crypto.h>
267 #include <linux/time.h>
268 #include <linux/slab.h>
269
270 #include <net/icmp.h>
271 #include <net/tcp.h>
272 #include <net/xfrm.h>
273 #include <net/ip.h>
274 #include <net/netdma.h>
275 #include <net/sock.h>
276
277 #include <asm/uaccess.h>
278 #include <asm/ioctls.h>
279
280 int sysctl_tcp_fin_timeout __read_mostly = TCP_FIN_TIMEOUT;
281
282 struct percpu_counter tcp_orphan_count;
283 EXPORT_SYMBOL_GPL(tcp_orphan_count);
284
285 long sysctl_tcp_mem[3] __read_mostly;
286 int sysctl_tcp_wmem[3] __read_mostly;
287 int sysctl_tcp_rmem[3] __read_mostly;
288
289 EXPORT_SYMBOL(sysctl_tcp_mem);
290 EXPORT_SYMBOL(sysctl_tcp_rmem);
291 EXPORT_SYMBOL(sysctl_tcp_wmem);
292
293 atomic_long_t tcp_memory_allocated;     /* Current allocated memory. */
294 EXPORT_SYMBOL(tcp_memory_allocated);
295
296 /*
297  * Current number of TCP sockets.
298  */
299 struct percpu_counter tcp_sockets_allocated;
300 EXPORT_SYMBOL(tcp_sockets_allocated);
301
302 /*
303  * TCP splice context
304  */
305 struct tcp_splice_state {
306         struct pipe_inode_info *pipe;
307         size_t len;
308         unsigned int flags;
309 };
310
311 /*
312  * Pressure flag: try to collapse.
313  * Technical note: it is used by multiple contexts non atomically.
314  * All the __sk_mem_schedule() is of this nature: accounting
315  * is strict, actions are advisory and have some latency.
316  */
317 int tcp_memory_pressure __read_mostly;
318 EXPORT_SYMBOL(tcp_memory_pressure);
319
320 void tcp_enter_memory_pressure(struct sock *sk)
321 {
322         if (!tcp_memory_pressure) {
323                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURES);
324                 tcp_memory_pressure = 1;
325         }
326 }
327 EXPORT_SYMBOL(tcp_enter_memory_pressure);
328
329 /* Convert seconds to retransmits based on initial and max timeout */
330 static u8 secs_to_retrans(int seconds, int timeout, int rto_max)
331 {
332         u8 res = 0;
333
334         if (seconds > 0) {
335                 int period = timeout;
336
337                 res = 1;
338                 while (seconds > period && res < 255) {
339                         res++;
340                         timeout <<= 1;
341                         if (timeout > rto_max)
342                                 timeout = rto_max;
343                         period += timeout;
344                 }
345         }
346         return res;
347 }
348
349 /* Convert retransmits to seconds based on initial and max timeout */
350 static int retrans_to_secs(u8 retrans, int timeout, int rto_max)
351 {
352         int period = 0;
353
354         if (retrans > 0) {
355                 period = timeout;
356                 while (--retrans) {
357                         timeout <<= 1;
358                         if (timeout > rto_max)
359                                 timeout = rto_max;
360                         period += timeout;
361                 }
362         }
363         return period;
364 }
365
366 /*
367  *      Wait for a TCP event.
368  *
369  *      Note that we don't need to lock the socket, as the upper poll layers
370  *      take care of normal races (between the test and the event) and we don't
371  *      go look at any of the socket buffers directly.
372  */
373 unsigned int tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
374 {
375         unsigned int mask;
376         struct sock *sk = sock->sk;
377         const struct tcp_sock *tp = tcp_sk(sk);
378
379         sock_poll_wait(file, sk_sleep(sk), wait);
380         if (sk->sk_state == TCP_LISTEN)
381                 return inet_csk_listen_poll(sk);
382
383         /* Socket is not locked. We are protected from async events
384          * by poll logic and correct handling of state changes
385          * made by other threads is impossible in any case.
386          */
387
388         mask = 0;
389
390         /*
391          * POLLHUP is certainly not done right. But poll() doesn't
392          * have a notion of HUP in just one direction, and for a
393          * socket the read side is more interesting.
394          *
395          * Some poll() documentation says that POLLHUP is incompatible
396          * with the POLLOUT/POLLWR flags, so somebody should check this
397          * all. But careful, it tends to be safer to return too many
398          * bits than too few, and you can easily break real applications
399          * if you don't tell them that something has hung up!
400          *
401          * Check-me.
402          *
403          * Check number 1. POLLHUP is _UNMASKABLE_ event (see UNIX98 and
404          * our fs/select.c). It means that after we received EOF,
405          * poll always returns immediately, making impossible poll() on write()
406          * in state CLOSE_WAIT. One solution is evident --- to set POLLHUP
407          * if and only if shutdown has been made in both directions.
408          * Actually, it is interesting to look how Solaris and DUX
409          * solve this dilemma. I would prefer, if POLLHUP were maskable,
410          * then we could set it on SND_SHUTDOWN. BTW examples given
411          * in Stevens' books assume exactly this behaviour, it explains
412          * why POLLHUP is incompatible with POLLOUT.    --ANK
413          *
414          * NOTE. Check for TCP_CLOSE is added. The goal is to prevent
415          * blocking on fresh not-connected or disconnected socket. --ANK
416          */
417         if (sk->sk_shutdown == SHUTDOWN_MASK || sk->sk_state == TCP_CLOSE)
418                 mask |= POLLHUP;
419         if (sk->sk_shutdown & RCV_SHUTDOWN)
420                 mask |= POLLIN | POLLRDNORM | POLLRDHUP;
421
422         /* Connected? */
423         if ((1 << sk->sk_state) & ~(TCPF_SYN_SENT | TCPF_SYN_RECV)) {
424                 int target = sock_rcvlowat(sk, 0, INT_MAX);
425
426                 if (tp->urg_seq == tp->copied_seq &&
427                     !sock_flag(sk, SOCK_URGINLINE) &&
428                     tp->urg_data)
429                         target++;
430
431                 /* Potential race condition. If read of tp below will
432                  * escape above sk->sk_state, we can be illegally awaken
433                  * in SYN_* states. */
434                 if (tp->rcv_nxt - tp->copied_seq >= target)
435                         mask |= POLLIN | POLLRDNORM;
436
437                 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
438                         if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk)) {
439                                 mask |= POLLOUT | POLLWRNORM;
440                         } else {  /* send SIGIO later */
441                                 set_bit(SOCK_ASYNC_NOSPACE,
442                                         &sk->sk_socket->flags);
443                                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
444
445                                 /* Race breaker. If space is freed after
446                                  * wspace test but before the flags are set,
447                                  * IO signal will be lost.
448                                  */
449                                 if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk))
450                                         mask |= POLLOUT | POLLWRNORM;
451                         }
452                 } else
453                         mask |= POLLOUT | POLLWRNORM;
454
455                 if (tp->urg_data & TCP_URG_VALID)
456                         mask |= POLLPRI;
457         }
458         /* This barrier is coupled with smp_wmb() in tcp_reset() */
459         smp_rmb();
460         if (sk->sk_err)
461                 mask |= POLLERR;
462
463         return mask;
464 }
465 EXPORT_SYMBOL(tcp_poll);
466
467 int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg)
468 {
469         struct tcp_sock *tp = tcp_sk(sk);
470         int answ;
471
472         switch (cmd) {
473         case SIOCINQ:
474                 if (sk->sk_state == TCP_LISTEN)
475                         return -EINVAL;
476
477                 lock_sock(sk);
478                 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
479                         answ = 0;
480                 else if (sock_flag(sk, SOCK_URGINLINE) ||
481                          !tp->urg_data ||
482                          before(tp->urg_seq, tp->copied_seq) ||
483                          !before(tp->urg_seq, tp->rcv_nxt)) {
484
485                         answ = tp->rcv_nxt - tp->copied_seq;
486
487                         /* Subtract 1, if FIN was received */
488                         if (answ && sock_flag(sk, SOCK_DONE))
489                                 answ--;
490                 } else
491                         answ = tp->urg_seq - tp->copied_seq;
492                 release_sock(sk);
493                 break;
494         case SIOCATMARK:
495                 answ = tp->urg_data && tp->urg_seq == tp->copied_seq;
496                 break;
497         case SIOCOUTQ:
498                 if (sk->sk_state == TCP_LISTEN)
499                         return -EINVAL;
500
501                 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
502                         answ = 0;
503                 else
504                         answ = tp->write_seq - tp->snd_una;
505                 break;
506         case SIOCOUTQNSD:
507                 if (sk->sk_state == TCP_LISTEN)
508                         return -EINVAL;
509
510                 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
511                         answ = 0;
512                 else
513                         answ = tp->write_seq - tp->snd_nxt;
514                 break;
515         default:
516                 return -ENOIOCTLCMD;
517         }
518
519         return put_user(answ, (int __user *)arg);
520 }
521 EXPORT_SYMBOL(tcp_ioctl);
522
523 static inline void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb)
524 {
525         TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
526         tp->pushed_seq = tp->write_seq;
527 }
528
529 static inline int forced_push(const struct tcp_sock *tp)
530 {
531         return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1));
532 }
533
534 static inline void skb_entail(struct sock *sk, struct sk_buff *skb)
535 {
536         struct tcp_sock *tp = tcp_sk(sk);
537         struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
538
539         skb->csum    = 0;
540         tcb->seq     = tcb->end_seq = tp->write_seq;
541         tcb->tcp_flags = TCPHDR_ACK;
542         tcb->sacked  = 0;
543         skb_header_release(skb);
544         tcp_add_write_queue_tail(sk, skb);
545         sk->sk_wmem_queued += skb->truesize;
546         sk_mem_charge(sk, skb->truesize);
547         if (tp->nonagle & TCP_NAGLE_PUSH)
548                 tp->nonagle &= ~TCP_NAGLE_PUSH;
549 }
550
551 static inline void tcp_mark_urg(struct tcp_sock *tp, int flags)
552 {
553         if (flags & MSG_OOB)
554                 tp->snd_up = tp->write_seq;
555 }
556
557 static inline void tcp_push(struct sock *sk, int flags, int mss_now,
558                             int nonagle)
559 {
560         if (tcp_send_head(sk)) {
561                 struct tcp_sock *tp = tcp_sk(sk);
562
563                 if (!(flags & MSG_MORE) || forced_push(tp))
564                         tcp_mark_push(tp, tcp_write_queue_tail(sk));
565
566                 tcp_mark_urg(tp, flags);
567                 __tcp_push_pending_frames(sk, mss_now,
568                                           (flags & MSG_MORE) ? TCP_NAGLE_CORK : nonagle);
569         }
570 }
571
572 static int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
573                                 unsigned int offset, size_t len)
574 {
575         struct tcp_splice_state *tss = rd_desc->arg.data;
576         int ret;
577
578         ret = skb_splice_bits(skb, offset, tss->pipe, min(rd_desc->count, len),
579                               tss->flags);
580         if (ret > 0)
581                 rd_desc->count -= ret;
582         return ret;
583 }
584
585 static int __tcp_splice_read(struct sock *sk, struct tcp_splice_state *tss)
586 {
587         /* Store TCP splice context information in read_descriptor_t. */
588         read_descriptor_t rd_desc = {
589                 .arg.data = tss,
590                 .count    = tss->len,
591         };
592
593         return tcp_read_sock(sk, &rd_desc, tcp_splice_data_recv);
594 }
595
596 /**
597  *  tcp_splice_read - splice data from TCP socket to a pipe
598  * @sock:       socket to splice from
599  * @ppos:       position (not valid)
600  * @pipe:       pipe to splice to
601  * @len:        number of bytes to splice
602  * @flags:      splice modifier flags
603  *
604  * Description:
605  *    Will read pages from given socket and fill them into a pipe.
606  *
607  **/
608 ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,
609                         struct pipe_inode_info *pipe, size_t len,
610                         unsigned int flags)
611 {
612         struct sock *sk = sock->sk;
613         struct tcp_splice_state tss = {
614                 .pipe = pipe,
615                 .len = len,
616                 .flags = flags,
617         };
618         long timeo;
619         ssize_t spliced;
620         int ret;
621
622         sock_rps_record_flow(sk);
623         /*
624          * We can't seek on a socket input
625          */
626         if (unlikely(*ppos))
627                 return -ESPIPE;
628
629         ret = spliced = 0;
630
631         lock_sock(sk);
632
633         timeo = sock_rcvtimeo(sk, sock->file->f_flags & O_NONBLOCK);
634         while (tss.len) {
635                 ret = __tcp_splice_read(sk, &tss);
636                 if (ret < 0)
637                         break;
638                 else if (!ret) {
639                         if (spliced)
640                                 break;
641                         if (sock_flag(sk, SOCK_DONE))
642                                 break;
643                         if (sk->sk_err) {
644                                 ret = sock_error(sk);
645                                 break;
646                         }
647                         if (sk->sk_shutdown & RCV_SHUTDOWN)
648                                 break;
649                         if (sk->sk_state == TCP_CLOSE) {
650                                 /*
651                                  * This occurs when user tries to read
652                                  * from never connected socket.
653                                  */
654                                 if (!sock_flag(sk, SOCK_DONE))
655                                         ret = -ENOTCONN;
656                                 break;
657                         }
658                         if (!timeo) {
659                                 ret = -EAGAIN;
660                                 break;
661                         }
662                         sk_wait_data(sk, &timeo);
663                         if (signal_pending(current)) {
664                                 ret = sock_intr_errno(timeo);
665                                 break;
666                         }
667                         continue;
668                 }
669                 tss.len -= ret;
670                 spliced += ret;
671
672                 if (!timeo)
673                         break;
674                 release_sock(sk);
675                 lock_sock(sk);
676
677                 if (sk->sk_err || sk->sk_state == TCP_CLOSE ||
678                     (sk->sk_shutdown & RCV_SHUTDOWN) ||
679                     signal_pending(current))
680                         break;
681         }
682
683         release_sock(sk);
684
685         if (spliced)
686                 return spliced;
687
688         return ret;
689 }
690 EXPORT_SYMBOL(tcp_splice_read);
691
692 struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp)
693 {
694         struct sk_buff *skb;
695
696         /* The TCP header must be at least 32-bit aligned.  */
697         size = ALIGN(size, 4);
698
699         skb = alloc_skb_fclone(size + sk->sk_prot->max_header, gfp);
700         if (skb) {
701                 if (sk_wmem_schedule(sk, skb->truesize)) {
702                         skb_reserve(skb, sk->sk_prot->max_header);
703                         /*
704                          * Make sure that we have exactly size bytes
705                          * available to the caller, no more, no less.
706                          */
707                         skb->reserved_tailroom = skb->end - skb->tail - size;
708                         return skb;
709                 }
710                 __kfree_skb(skb);
711         } else {
712                 sk->sk_prot->enter_memory_pressure(sk);
713                 sk_stream_moderate_sndbuf(sk);
714         }
715         return NULL;
716 }
717
718 static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now,
719                                        int large_allowed)
720 {
721         struct tcp_sock *tp = tcp_sk(sk);
722         u32 xmit_size_goal, old_size_goal;
723
724         xmit_size_goal = mss_now;
725
726         if (large_allowed && sk_can_gso(sk)) {
727                 xmit_size_goal = ((sk->sk_gso_max_size - 1) -
728                                   inet_csk(sk)->icsk_af_ops->net_header_len -
729                                   inet_csk(sk)->icsk_ext_hdr_len -
730                                   tp->tcp_header_len);
731
732                 xmit_size_goal = tcp_bound_to_half_wnd(tp, xmit_size_goal);
733
734                 /* We try hard to avoid divides here */
735                 old_size_goal = tp->xmit_size_goal_segs * mss_now;
736
737                 if (likely(old_size_goal <= xmit_size_goal &&
738                            old_size_goal + mss_now > xmit_size_goal)) {
739                         xmit_size_goal = old_size_goal;
740                 } else {
741                         tp->xmit_size_goal_segs =
742                                 min_t(u16, xmit_size_goal / mss_now,
743                                       sk->sk_gso_max_segs);
744                         xmit_size_goal = tp->xmit_size_goal_segs * mss_now;
745                 }
746         }
747
748         return max(xmit_size_goal, mss_now);
749 }
750
751 static int tcp_send_mss(struct sock *sk, int *size_goal, int flags)
752 {
753         int mss_now;
754
755         mss_now = tcp_current_mss(sk);
756         *size_goal = tcp_xmit_size_goal(sk, mss_now, !(flags & MSG_OOB));
757
758         return mss_now;
759 }
760
761 static ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffset,
762                          size_t psize, int flags)
763 {
764         struct tcp_sock *tp = tcp_sk(sk);
765         int mss_now, size_goal;
766         int err;
767         ssize_t copied;
768         long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
769
770         /* Wait for a connection to finish. */
771         if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT))
772                 if ((err = sk_stream_wait_connect(sk, &timeo)) != 0)
773                         goto out_err;
774
775         clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
776
777         mss_now = tcp_send_mss(sk, &size_goal, flags);
778         copied = 0;
779
780         err = -EPIPE;
781         if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
782                 goto out_err;
783
784         while (psize > 0) {
785                 struct sk_buff *skb = tcp_write_queue_tail(sk);
786                 struct page *page = pages[poffset / PAGE_SIZE];
787                 int copy, i, can_coalesce;
788                 int offset = poffset % PAGE_SIZE;
789                 int size = min_t(size_t, psize, PAGE_SIZE - offset);
790
791                 if (!tcp_send_head(sk) || (copy = size_goal - skb->len) <= 0) {
792 new_segment:
793                         if (!sk_stream_memory_free(sk))
794                                 goto wait_for_sndbuf;
795
796                         skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation);
797                         if (!skb)
798                                 goto wait_for_memory;
799
800                         skb_entail(sk, skb);
801                         copy = size_goal;
802                 }
803
804                 if (copy > size)
805                         copy = size;
806
807                 i = skb_shinfo(skb)->nr_frags;
808                 can_coalesce = skb_can_coalesce(skb, i, page, offset);
809                 if (!can_coalesce && i >= MAX_SKB_FRAGS) {
810                         tcp_mark_push(tp, skb);
811                         goto new_segment;
812                 }
813                 if (!sk_wmem_schedule(sk, copy))
814                         goto wait_for_memory;
815
816                 if (can_coalesce) {
817                         skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
818                 } else {
819                         get_page(page);
820                         skb_fill_page_desc(skb, i, page, offset, copy);
821                 }
822
823                 skb->len += copy;
824                 skb->data_len += copy;
825                 skb->truesize += copy;
826                 sk->sk_wmem_queued += copy;
827                 sk_mem_charge(sk, copy);
828                 skb->ip_summed = CHECKSUM_PARTIAL;
829                 tp->write_seq += copy;
830                 TCP_SKB_CB(skb)->end_seq += copy;
831                 skb_shinfo(skb)->gso_segs = 0;
832
833                 if (!copied)
834                         TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
835
836                 copied += copy;
837                 poffset += copy;
838                 if (!(psize -= copy))
839                         goto out;
840
841                 if (skb->len < size_goal || (flags & MSG_OOB))
842                         continue;
843
844                 if (forced_push(tp)) {
845                         tcp_mark_push(tp, skb);
846                         __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
847                 } else if (skb == tcp_send_head(sk))
848                         tcp_push_one(sk, mss_now);
849                 continue;
850
851 wait_for_sndbuf:
852                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
853 wait_for_memory:
854                 tcp_push(sk, flags & ~MSG_MORE, mss_now, TCP_NAGLE_PUSH);
855
856                 if ((err = sk_stream_wait_memory(sk, &timeo)) != 0)
857                         goto do_error;
858
859                 mss_now = tcp_send_mss(sk, &size_goal, flags);
860         }
861
862 out:
863         if (copied && !(flags & MSG_SENDPAGE_NOTLAST))
864                 tcp_push(sk, flags, mss_now, tp->nonagle);
865         return copied;
866
867 do_error:
868         if (copied)
869                 goto out;
870 out_err:
871         return sk_stream_error(sk, flags, err);
872 }
873
874 int tcp_sendpage(struct sock *sk, struct page *page, int offset,
875                  size_t size, int flags)
876 {
877         ssize_t res;
878
879         if (!(sk->sk_route_caps & NETIF_F_SG) ||
880             !(sk->sk_route_caps & NETIF_F_ALL_CSUM))
881                 return sock_no_sendpage(sk->sk_socket, page, offset, size,
882                                         flags);
883
884         lock_sock(sk);
885         res = do_tcp_sendpages(sk, &page, offset, size, flags);
886         release_sock(sk);
887         return res;
888 }
889 EXPORT_SYMBOL(tcp_sendpage);
890
891 #define TCP_PAGE(sk)    (sk->sk_sndmsg_page)
892 #define TCP_OFF(sk)     (sk->sk_sndmsg_off)
893
894 static inline int select_size(const struct sock *sk, int sg)
895 {
896         const struct tcp_sock *tp = tcp_sk(sk);
897         int tmp = tp->mss_cache;
898
899         if (sg) {
900                 if (sk_can_gso(sk))
901                         tmp = 0;
902                 else {
903                         int pgbreak = SKB_MAX_HEAD(MAX_TCP_HEADER);
904
905                         if (tmp >= pgbreak &&
906                             tmp <= pgbreak + (MAX_SKB_FRAGS - 1) * PAGE_SIZE)
907                                 tmp = pgbreak;
908                 }
909         }
910
911         return tmp;
912 }
913
914 int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
915                 size_t size)
916 {
917         struct iovec *iov;
918         struct tcp_sock *tp = tcp_sk(sk);
919         struct sk_buff *skb;
920         int iovlen, flags;
921         int mss_now, size_goal;
922         int sg, err, copied;
923         long timeo;
924
925         lock_sock(sk);
926
927         flags = msg->msg_flags;
928         timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
929
930         /* Wait for a connection to finish. */
931         if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT))
932                 if ((err = sk_stream_wait_connect(sk, &timeo)) != 0)
933                         goto out_err;
934
935         /* This should be in poll */
936         clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
937
938         mss_now = tcp_send_mss(sk, &size_goal, flags);
939
940         /* Ok commence sending. */
941         iovlen = msg->msg_iovlen;
942         iov = msg->msg_iov;
943         copied = 0;
944
945         err = -EPIPE;
946         if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
947                 goto out_err;
948
949         sg = sk->sk_route_caps & NETIF_F_SG;
950
951         while (--iovlen >= 0) {
952                 size_t seglen = iov->iov_len;
953                 unsigned char __user *from = iov->iov_base;
954
955                 iov++;
956
957                 while (seglen > 0) {
958                         int copy = 0;
959                         int max = size_goal;
960
961                         skb = tcp_write_queue_tail(sk);
962                         if (tcp_send_head(sk)) {
963                                 if (skb->ip_summed == CHECKSUM_NONE)
964                                         max = mss_now;
965                                 copy = max - skb->len;
966                         }
967
968                         if (copy <= 0) {
969 new_segment:
970                                 /* Allocate new segment. If the interface is SG,
971                                  * allocate skb fitting to single page.
972                                  */
973                                 if (!sk_stream_memory_free(sk))
974                                         goto wait_for_sndbuf;
975
976                                 skb = sk_stream_alloc_skb(sk,
977                                                           select_size(sk, sg),
978                                                           sk->sk_allocation);
979                                 if (!skb)
980                                         goto wait_for_memory;
981
982                                 /*
983                                  * Check whether we can use HW checksum.
984                                  */
985                                 if (sk->sk_route_caps & NETIF_F_ALL_CSUM)
986                                         skb->ip_summed = CHECKSUM_PARTIAL;
987
988                                 skb_entail(sk, skb);
989                                 copy = size_goal;
990                                 max = size_goal;
991                         }
992
993                         /* Try to append data to the end of skb. */
994                         if (copy > seglen)
995                                 copy = seglen;
996
997                         /* Where to copy to? */
998                         if (skb_availroom(skb) > 0) {
999                                 /* We have some space in skb head. Superb! */
1000                                 copy = min_t(int, copy, skb_availroom(skb));
1001                                 err = skb_add_data_nocache(sk, skb, from, copy);
1002                                 if (err)
1003                                         goto do_fault;
1004                         } else {
1005                                 int merge = 0;
1006                                 int i = skb_shinfo(skb)->nr_frags;
1007                                 struct page *page = TCP_PAGE(sk);
1008                                 int off = TCP_OFF(sk);
1009
1010                                 if (skb_can_coalesce(skb, i, page, off) &&
1011                                     off != PAGE_SIZE) {
1012                                         /* We can extend the last page
1013                                          * fragment. */
1014                                         merge = 1;
1015                                 } else if (i == MAX_SKB_FRAGS || !sg) {
1016                                         /* Need to add new fragment and cannot
1017                                          * do this because interface is non-SG,
1018                                          * or because all the page slots are
1019                                          * busy. */
1020                                         tcp_mark_push(tp, skb);
1021                                         goto new_segment;
1022                                 } else if (page) {
1023                                         if (off == PAGE_SIZE) {
1024                                                 put_page(page);
1025                                                 TCP_PAGE(sk) = page = NULL;
1026                                                 off = 0;
1027                                         }
1028                                 } else
1029                                         off = 0;
1030
1031                                 if (copy > PAGE_SIZE - off)
1032                                         copy = PAGE_SIZE - off;
1033
1034                                 if (!sk_wmem_schedule(sk, copy))
1035                                         goto wait_for_memory;
1036
1037                                 if (!page) {
1038                                         /* Allocate new cache page. */
1039                                         if (!(page = sk_stream_alloc_page(sk)))
1040                                                 goto wait_for_memory;
1041                                 }
1042
1043                                 /* Time to copy data. We are close to
1044                                  * the end! */
1045                                 err = skb_copy_to_page_nocache(sk, from, skb,
1046                                                                page, off, copy);
1047                                 if (err) {
1048                                         /* If this page was new, give it to the
1049                                          * socket so it does not get leaked.
1050                                          */
1051                                         if (!TCP_PAGE(sk)) {
1052                                                 TCP_PAGE(sk) = page;
1053                                                 TCP_OFF(sk) = 0;
1054                                         }
1055                                         goto do_error;
1056                                 }
1057
1058                                 /* Update the skb. */
1059                                 if (merge) {
1060                                         skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1061                                 } else {
1062                                         skb_fill_page_desc(skb, i, page, off, copy);
1063                                         if (TCP_PAGE(sk)) {
1064                                                 get_page(page);
1065                                         } else if (off + copy < PAGE_SIZE) {
1066                                                 get_page(page);
1067                                                 TCP_PAGE(sk) = page;
1068                                         }
1069                                 }
1070
1071                                 TCP_OFF(sk) = off + copy;
1072                         }
1073
1074                         if (!copied)
1075                                 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
1076
1077                         tp->write_seq += copy;
1078                         TCP_SKB_CB(skb)->end_seq += copy;
1079                         skb_shinfo(skb)->gso_segs = 0;
1080
1081                         from += copy;
1082                         copied += copy;
1083                         if ((seglen -= copy) == 0 && iovlen == 0)
1084                                 goto out;
1085
1086                         if (skb->len < max || (flags & MSG_OOB))
1087                                 continue;
1088
1089                         if (forced_push(tp)) {
1090                                 tcp_mark_push(tp, skb);
1091                                 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
1092                         } else if (skb == tcp_send_head(sk))
1093                                 tcp_push_one(sk, mss_now);
1094                         continue;
1095
1096 wait_for_sndbuf:
1097                         set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1098 wait_for_memory:
1099                         if (copied)
1100                                 tcp_push(sk, flags & ~MSG_MORE, mss_now, TCP_NAGLE_PUSH);
1101
1102                         if ((err = sk_stream_wait_memory(sk, &timeo)) != 0)
1103                                 goto do_error;
1104
1105                         mss_now = tcp_send_mss(sk, &size_goal, flags);
1106                 }
1107         }
1108
1109 out:
1110         if (copied)
1111                 tcp_push(sk, flags, mss_now, tp->nonagle);
1112         release_sock(sk);
1113         return copied;
1114
1115 do_fault:
1116         if (!skb->len) {
1117                 tcp_unlink_write_queue(skb, sk);
1118                 /* It is the one place in all of TCP, except connection
1119                  * reset, where we can be unlinking the send_head.
1120                  */
1121                 tcp_check_send_head(sk, skb);
1122                 sk_wmem_free_skb(sk, skb);
1123         }
1124
1125 do_error:
1126         if (copied)
1127                 goto out;
1128 out_err:
1129         err = sk_stream_error(sk, flags, err);
1130         release_sock(sk);
1131         return err;
1132 }
1133 EXPORT_SYMBOL(tcp_sendmsg);
1134
1135 /*
1136  *      Handle reading urgent data. BSD has very simple semantics for
1137  *      this, no blocking and very strange errors 8)
1138  */
1139
1140 static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags)
1141 {
1142         struct tcp_sock *tp = tcp_sk(sk);
1143
1144         /* No URG data to read. */
1145         if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data ||
1146             tp->urg_data == TCP_URG_READ)
1147                 return -EINVAL; /* Yes this is right ! */
1148
1149         if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE))
1150                 return -ENOTCONN;
1151
1152         if (tp->urg_data & TCP_URG_VALID) {
1153                 int err = 0;
1154                 char c = tp->urg_data;
1155
1156                 if (!(flags & MSG_PEEK))
1157                         tp->urg_data = TCP_URG_READ;
1158
1159                 /* Read urgent data. */
1160                 msg->msg_flags |= MSG_OOB;
1161
1162                 if (len > 0) {
1163                         if (!(flags & MSG_TRUNC))
1164                                 err = memcpy_toiovec(msg->msg_iov, &c, 1);
1165                         len = 1;
1166                 } else
1167                         msg->msg_flags |= MSG_TRUNC;
1168
1169                 return err ? -EFAULT : len;
1170         }
1171
1172         if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN))
1173                 return 0;
1174
1175         /* Fixed the recv(..., MSG_OOB) behaviour.  BSD docs and
1176          * the available implementations agree in this case:
1177          * this call should never block, independent of the
1178          * blocking state of the socket.
1179          * Mike <pall@rz.uni-karlsruhe.de>
1180          */
1181         return -EAGAIN;
1182 }
1183
1184 /* Clean up the receive buffer for full frames taken by the user,
1185  * then send an ACK if necessary.  COPIED is the number of bytes
1186  * tcp_recvmsg has given to the user so far, it speeds up the
1187  * calculation of whether or not we must ACK for the sake of
1188  * a window update.
1189  */
1190 void tcp_cleanup_rbuf(struct sock *sk, int copied)
1191 {
1192         struct tcp_sock *tp = tcp_sk(sk);
1193         int time_to_ack = 0;
1194
1195         struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
1196
1197         WARN(skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq),
1198              "cleanup rbuf bug: copied %X seq %X rcvnxt %X\n",
1199              tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt);
1200
1201         if (inet_csk_ack_scheduled(sk)) {
1202                 const struct inet_connection_sock *icsk = inet_csk(sk);
1203                    /* Delayed ACKs frequently hit locked sockets during bulk
1204                     * receive. */
1205                 if (icsk->icsk_ack.blocked ||
1206                     /* Once-per-two-segments ACK was not sent by tcp_input.c */
1207                     tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss ||
1208                     /*
1209                      * If this read emptied read buffer, we send ACK, if
1210                      * connection is not bidirectional, user drained
1211                      * receive buffer and there was a small segment
1212                      * in queue.
1213                      */
1214                     (copied > 0 &&
1215                      ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) ||
1216                       ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) &&
1217                        !icsk->icsk_ack.pingpong)) &&
1218                       !atomic_read(&sk->sk_rmem_alloc)))
1219                         time_to_ack = 1;
1220         }
1221
1222         /* We send an ACK if we can now advertise a non-zero window
1223          * which has been raised "significantly".
1224          *
1225          * Even if window raised up to infinity, do not send window open ACK
1226          * in states, where we will not receive more. It is useless.
1227          */
1228         if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) {
1229                 __u32 rcv_window_now = tcp_receive_window(tp);
1230
1231                 /* Optimize, __tcp_select_window() is not cheap. */
1232                 if (2*rcv_window_now <= tp->window_clamp) {
1233                         __u32 new_window = __tcp_select_window(sk);
1234
1235                         /* Send ACK now, if this read freed lots of space
1236                          * in our buffer. Certainly, new_window is new window.
1237                          * We can advertise it now, if it is not less than current one.
1238                          * "Lots" means "at least twice" here.
1239                          */
1240                         if (new_window && new_window >= 2 * rcv_window_now)
1241                                 time_to_ack = 1;
1242                 }
1243         }
1244         if (time_to_ack)
1245                 tcp_send_ack(sk);
1246 }
1247
1248 static void tcp_prequeue_process(struct sock *sk)
1249 {
1250         struct sk_buff *skb;
1251         struct tcp_sock *tp = tcp_sk(sk);
1252
1253         NET_INC_STATS_USER(sock_net(sk), LINUX_MIB_TCPPREQUEUED);
1254
1255         /* RX process wants to run with disabled BHs, though it is not
1256          * necessary */
1257         local_bh_disable();
1258         while ((skb = __skb_dequeue(&tp->ucopy.prequeue)) != NULL)
1259                 sk_backlog_rcv(sk, skb);
1260         local_bh_enable();
1261
1262         /* Clear memory counter. */
1263         tp->ucopy.memory = 0;
1264 }
1265
1266 #ifdef CONFIG_NET_DMA
1267 static void tcp_service_net_dma(struct sock *sk, bool wait)
1268 {
1269         dma_cookie_t done, used;
1270         dma_cookie_t last_issued;
1271         struct tcp_sock *tp = tcp_sk(sk);
1272
1273         if (!tp->ucopy.dma_chan)
1274                 return;
1275
1276         last_issued = tp->ucopy.dma_cookie;
1277         dma_async_memcpy_issue_pending(tp->ucopy.dma_chan);
1278
1279         do {
1280                 if (dma_async_memcpy_complete(tp->ucopy.dma_chan,
1281                                               last_issued, &done,
1282                                               &used) == DMA_SUCCESS) {
1283                         /* Safe to free early-copied skbs now */
1284                         __skb_queue_purge(&sk->sk_async_wait_queue);
1285                         break;
1286                 } else {
1287                         struct sk_buff *skb;
1288                         while ((skb = skb_peek(&sk->sk_async_wait_queue)) &&
1289                                (dma_async_is_complete(skb->dma_cookie, done,
1290                                                       used) == DMA_SUCCESS)) {
1291                                 __skb_dequeue(&sk->sk_async_wait_queue);
1292                                 kfree_skb(skb);
1293                         }
1294                 }
1295         } while (wait);
1296 }
1297 #endif
1298
1299 static inline struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
1300 {
1301         struct sk_buff *skb;
1302         u32 offset;
1303
1304         skb_queue_walk(&sk->sk_receive_queue, skb) {
1305                 offset = seq - TCP_SKB_CB(skb)->seq;
1306                 if (tcp_hdr(skb)->syn)
1307                         offset--;
1308                 if (offset < skb->len || tcp_hdr(skb)->fin) {
1309                         *off = offset;
1310                         return skb;
1311                 }
1312         }
1313         return NULL;
1314 }
1315
1316 /*
1317  * This routine provides an alternative to tcp_recvmsg() for routines
1318  * that would like to handle copying from skbuffs directly in 'sendfile'
1319  * fashion.
1320  * Note:
1321  *      - It is assumed that the socket was locked by the caller.
1322  *      - The routine does not block.
1323  *      - At present, there is no support for reading OOB data
1324  *        or for 'peeking' the socket using this routine
1325  *        (although both would be easy to implement).
1326  */
1327 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
1328                   sk_read_actor_t recv_actor)
1329 {
1330         struct sk_buff *skb;
1331         struct tcp_sock *tp = tcp_sk(sk);
1332         u32 seq = tp->copied_seq;
1333         u32 offset;
1334         int copied = 0;
1335
1336         if (sk->sk_state == TCP_LISTEN)
1337                 return -ENOTCONN;
1338         while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1339                 if (offset < skb->len) {
1340                         int used;
1341                         size_t len;
1342
1343                         len = skb->len - offset;
1344                         /* Stop reading if we hit a patch of urgent data */
1345                         if (tp->urg_data) {
1346                                 u32 urg_offset = tp->urg_seq - seq;
1347                                 if (urg_offset < len)
1348                                         len = urg_offset;
1349                                 if (!len)
1350                                         break;
1351                         }
1352                         used = recv_actor(desc, skb, offset, len);
1353                         if (used < 0) {
1354                                 if (!copied)
1355                                         copied = used;
1356                                 break;
1357                         } else if (used <= len) {
1358                                 seq += used;
1359                                 copied += used;
1360                                 offset += used;
1361                         }
1362                         /*
1363                          * If recv_actor drops the lock (e.g. TCP splice
1364                          * receive) the skb pointer might be invalid when
1365                          * getting here: tcp_collapse might have deleted it
1366                          * while aggregating skbs from the socket queue.
1367                          */
1368                         skb = tcp_recv_skb(sk, seq-1, &offset);
1369                         if (!skb || (offset+1 != skb->len))
1370                                 break;
1371                 }
1372                 if (tcp_hdr(skb)->fin) {
1373                         sk_eat_skb(sk, skb, 0);
1374                         ++seq;
1375                         break;
1376                 }
1377                 sk_eat_skb(sk, skb, 0);
1378                 if (!desc->count)
1379                         break;
1380                 tp->copied_seq = seq;
1381         }
1382         tp->copied_seq = seq;
1383
1384         tcp_rcv_space_adjust(sk);
1385
1386         /* Clean up data we have read: This will do ACK frames. */
1387         if (copied > 0)
1388                 tcp_cleanup_rbuf(sk, copied);
1389         return copied;
1390 }
1391 EXPORT_SYMBOL(tcp_read_sock);
1392
1393 /*
1394  *      This routine copies from a sock struct into the user buffer.
1395  *
1396  *      Technical note: in 2.3 we work on _locked_ socket, so that
1397  *      tricks with *seq access order and skb->users are not required.
1398  *      Probably, code can be easily improved even more.
1399  */
1400
1401 int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
1402                 size_t len, int nonblock, int flags, int *addr_len)
1403 {
1404         struct tcp_sock *tp = tcp_sk(sk);
1405         int copied = 0;
1406         u32 peek_seq;
1407         u32 *seq;
1408         unsigned long used;
1409         int err;
1410         int target;             /* Read at least this many bytes */
1411         long timeo;
1412         struct task_struct *user_recv = NULL;
1413         int copied_early = 0;
1414         struct sk_buff *skb;
1415         u32 urg_hole = 0;
1416
1417         lock_sock(sk);
1418
1419         err = -ENOTCONN;
1420         if (sk->sk_state == TCP_LISTEN)
1421                 goto out;
1422
1423         timeo = sock_rcvtimeo(sk, nonblock);
1424
1425         /* Urgent data needs to be handled specially. */
1426         if (flags & MSG_OOB)
1427                 goto recv_urg;
1428
1429         seq = &tp->copied_seq;
1430         if (flags & MSG_PEEK) {
1431                 peek_seq = tp->copied_seq;
1432                 seq = &peek_seq;
1433         }
1434
1435         target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
1436
1437 #ifdef CONFIG_NET_DMA
1438         tp->ucopy.dma_chan = NULL;
1439         preempt_disable();
1440         skb = skb_peek_tail(&sk->sk_receive_queue);
1441         {
1442                 int available = 0;
1443
1444                 if (skb)
1445                         available = TCP_SKB_CB(skb)->seq + skb->len - (*seq);
1446                 if ((available < target) &&
1447                     (len > sysctl_tcp_dma_copybreak) && !(flags & MSG_PEEK) &&
1448                     !sysctl_tcp_low_latency &&
1449                     dma_find_channel(DMA_MEMCPY)) {
1450                         preempt_enable_no_resched();
1451                         tp->ucopy.pinned_list =
1452                                         dma_pin_iovec_pages(msg->msg_iov, len);
1453                 } else {
1454                         preempt_enable_no_resched();
1455                 }
1456         }
1457 #endif
1458
1459         do {
1460                 u32 offset;
1461
1462                 /* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */
1463                 if (tp->urg_data && tp->urg_seq == *seq) {
1464                         if (copied)
1465                                 break;
1466                         if (signal_pending(current)) {
1467                                 copied = timeo ? sock_intr_errno(timeo) : -EAGAIN;
1468                                 break;
1469                         }
1470                 }
1471
1472                 /* Next get a buffer. */
1473
1474                 skb_queue_walk(&sk->sk_receive_queue, skb) {
1475                         /* Now that we have two receive queues this
1476                          * shouldn't happen.
1477                          */
1478                         if (WARN(before(*seq, TCP_SKB_CB(skb)->seq),
1479                                  "recvmsg bug: copied %X seq %X rcvnxt %X fl %X\n",
1480                                  *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt,
1481                                  flags))
1482                                 break;
1483
1484                         offset = *seq - TCP_SKB_CB(skb)->seq;
1485                         if (tcp_hdr(skb)->syn)
1486                                 offset--;
1487                         if (offset < skb->len)
1488                                 goto found_ok_skb;
1489                         if (tcp_hdr(skb)->fin)
1490                                 goto found_fin_ok;
1491                         WARN(!(flags & MSG_PEEK),
1492                              "recvmsg bug 2: copied %X seq %X rcvnxt %X fl %X\n",
1493                              *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, flags);
1494                 }
1495
1496                 /* Well, if we have backlog, try to process it now yet. */
1497
1498                 if (copied >= target && !sk->sk_backlog.tail)
1499                         break;
1500
1501                 if (copied) {
1502                         if (sk->sk_err ||
1503                             sk->sk_state == TCP_CLOSE ||
1504                             (sk->sk_shutdown & RCV_SHUTDOWN) ||
1505                             !timeo ||
1506                             signal_pending(current))
1507                                 break;
1508                 } else {
1509                         if (sock_flag(sk, SOCK_DONE))
1510                                 break;
1511
1512                         if (sk->sk_err) {
1513                                 copied = sock_error(sk);
1514                                 break;
1515                         }
1516
1517                         if (sk->sk_shutdown & RCV_SHUTDOWN)
1518                                 break;
1519
1520                         if (sk->sk_state == TCP_CLOSE) {
1521                                 if (!sock_flag(sk, SOCK_DONE)) {
1522                                         /* This occurs when user tries to read
1523                                          * from never connected socket.
1524                                          */
1525                                         copied = -ENOTCONN;
1526                                         break;
1527                                 }
1528                                 break;
1529                         }
1530
1531                         if (!timeo) {
1532                                 copied = -EAGAIN;
1533                                 break;
1534                         }
1535
1536                         if (signal_pending(current)) {
1537                                 copied = sock_intr_errno(timeo);
1538                                 break;
1539                         }
1540                 }
1541
1542                 tcp_cleanup_rbuf(sk, copied);
1543
1544                 if (!sysctl_tcp_low_latency && tp->ucopy.task == user_recv) {
1545                         /* Install new reader */
1546                         if (!user_recv && !(flags & (MSG_TRUNC | MSG_PEEK))) {
1547                                 user_recv = current;
1548                                 tp->ucopy.task = user_recv;
1549                                 tp->ucopy.iov = msg->msg_iov;
1550                         }
1551
1552                         tp->ucopy.len = len;
1553
1554                         WARN_ON(tp->copied_seq != tp->rcv_nxt &&
1555                                 !(flags & (MSG_PEEK | MSG_TRUNC)));
1556
1557                         /* Ugly... If prequeue is not empty, we have to
1558                          * process it before releasing socket, otherwise
1559                          * order will be broken at second iteration.
1560                          * More elegant solution is required!!!
1561                          *
1562                          * Look: we have the following (pseudo)queues:
1563                          *
1564                          * 1. packets in flight
1565                          * 2. backlog
1566                          * 3. prequeue
1567                          * 4. receive_queue
1568                          *
1569                          * Each queue can be processed only if the next ones
1570                          * are empty. At this point we have empty receive_queue.
1571                          * But prequeue _can_ be not empty after 2nd iteration,
1572                          * when we jumped to start of loop because backlog
1573                          * processing added something to receive_queue.
1574                          * We cannot release_sock(), because backlog contains
1575                          * packets arrived _after_ prequeued ones.
1576                          *
1577                          * Shortly, algorithm is clear --- to process all
1578                          * the queues in order. We could make it more directly,
1579                          * requeueing packets from backlog to prequeue, if
1580                          * is not empty. It is more elegant, but eats cycles,
1581                          * unfortunately.
1582                          */
1583                         if (!skb_queue_empty(&tp->ucopy.prequeue))
1584                                 goto do_prequeue;
1585
1586                         /* __ Set realtime policy in scheduler __ */
1587                 }
1588
1589 #ifdef CONFIG_NET_DMA
1590                 if (tp->ucopy.dma_chan) {
1591                         if (tp->rcv_wnd == 0 &&
1592                             !skb_queue_empty(&sk->sk_async_wait_queue)) {
1593                                 tcp_service_net_dma(sk, true);
1594                                 tcp_cleanup_rbuf(sk, copied);
1595                         } else
1596                                 dma_async_memcpy_issue_pending(tp->ucopy.dma_chan);
1597                 }
1598 #endif
1599                 if (copied >= target) {
1600                         /* Do not sleep, just process backlog. */
1601                         release_sock(sk);
1602                         lock_sock(sk);
1603                 } else
1604                         sk_wait_data(sk, &timeo);
1605
1606 #ifdef CONFIG_NET_DMA
1607                 tcp_service_net_dma(sk, false);  /* Don't block */
1608                 tp->ucopy.wakeup = 0;
1609 #endif
1610
1611                 if (user_recv) {
1612                         int chunk;
1613
1614                         /* __ Restore normal policy in scheduler __ */
1615
1616                         if ((chunk = len - tp->ucopy.len) != 0) {
1617                                 NET_ADD_STATS_USER(sock_net(sk), LINUX_MIB_TCPDIRECTCOPYFROMBACKLOG, chunk);
1618                                 len -= chunk;
1619                                 copied += chunk;
1620                         }
1621
1622                         if (tp->rcv_nxt == tp->copied_seq &&
1623                             !skb_queue_empty(&tp->ucopy.prequeue)) {
1624 do_prequeue:
1625                                 tcp_prequeue_process(sk);
1626
1627                                 if ((chunk = len - tp->ucopy.len) != 0) {
1628                                         NET_ADD_STATS_USER(sock_net(sk), LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk);
1629                                         len -= chunk;
1630                                         copied += chunk;
1631                                 }
1632                         }
1633                 }
1634                 if ((flags & MSG_PEEK) &&
1635                     (peek_seq - copied - urg_hole != tp->copied_seq)) {
1636                         if (net_ratelimit())
1637                                 printk(KERN_DEBUG "TCP(%s:%d): Application bug, race in MSG_PEEK.\n",
1638                                        current->comm, task_pid_nr(current));
1639                         peek_seq = tp->copied_seq;
1640                 }
1641                 continue;
1642
1643         found_ok_skb:
1644                 /* Ok so how much can we use? */
1645                 used = skb->len - offset;
1646                 if (len < used)
1647                         used = len;
1648
1649                 /* Do we have urgent data here? */
1650                 if (tp->urg_data) {
1651                         u32 urg_offset = tp->urg_seq - *seq;
1652                         if (urg_offset < used) {
1653                                 if (!urg_offset) {
1654                                         if (!sock_flag(sk, SOCK_URGINLINE)) {
1655                                                 ++*seq;
1656                                                 urg_hole++;
1657                                                 offset++;
1658                                                 used--;
1659                                                 if (!used)
1660                                                         goto skip_copy;
1661                                         }
1662                                 } else
1663                                         used = urg_offset;
1664                         }
1665                 }
1666
1667                 if (!(flags & MSG_TRUNC)) {
1668 #ifdef CONFIG_NET_DMA
1669                         if (!tp->ucopy.dma_chan && tp->ucopy.pinned_list)
1670                                 tp->ucopy.dma_chan = dma_find_channel(DMA_MEMCPY);
1671
1672                         if (tp->ucopy.dma_chan) {
1673                                 tp->ucopy.dma_cookie = dma_skb_copy_datagram_iovec(
1674                                         tp->ucopy.dma_chan, skb, offset,
1675                                         msg->msg_iov, used,
1676                                         tp->ucopy.pinned_list);
1677
1678                                 if (tp->ucopy.dma_cookie < 0) {
1679
1680                                         printk(KERN_ALERT "dma_cookie < 0\n");
1681
1682                                         /* Exception. Bailout! */
1683                                         if (!copied)
1684                                                 copied = -EFAULT;
1685                                         break;
1686                                 }
1687
1688                                 dma_async_memcpy_issue_pending(tp->ucopy.dma_chan);
1689
1690                                 if ((offset + used) == skb->len)
1691                                         copied_early = 1;
1692
1693                         } else
1694 #endif
1695                         {
1696                                 err = skb_copy_datagram_iovec(skb, offset,
1697                                                 msg->msg_iov, used);
1698                                 if (err) {
1699                                         /* Exception. Bailout! */
1700                                         if (!copied)
1701                                                 copied = -EFAULT;
1702                                         break;
1703                                 }
1704                         }
1705                 }
1706
1707                 *seq += used;
1708                 copied += used;
1709                 len -= used;
1710
1711                 tcp_rcv_space_adjust(sk);
1712
1713 skip_copy:
1714                 if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) {
1715                         tp->urg_data = 0;
1716                         tcp_fast_path_check(sk);
1717                 }
1718                 if (used + offset < skb->len)
1719                         continue;
1720
1721                 if (tcp_hdr(skb)->fin)
1722                         goto found_fin_ok;
1723                 if (!(flags & MSG_PEEK)) {
1724                         sk_eat_skb(sk, skb, copied_early);
1725                         copied_early = 0;
1726                 }
1727                 continue;
1728
1729         found_fin_ok:
1730                 /* Process the FIN. */
1731                 ++*seq;
1732                 if (!(flags & MSG_PEEK)) {
1733                         sk_eat_skb(sk, skb, copied_early);
1734                         copied_early = 0;
1735                 }
1736                 break;
1737         } while (len > 0);
1738
1739         if (user_recv) {
1740                 if (!skb_queue_empty(&tp->ucopy.prequeue)) {
1741                         int chunk;
1742
1743                         tp->ucopy.len = copied > 0 ? len : 0;
1744
1745                         tcp_prequeue_process(sk);
1746
1747                         if (copied > 0 && (chunk = len - tp->ucopy.len) != 0) {
1748                                 NET_ADD_STATS_USER(sock_net(sk), LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk);
1749                                 len -= chunk;
1750                                 copied += chunk;
1751                         }
1752                 }
1753
1754                 tp->ucopy.task = NULL;
1755                 tp->ucopy.len = 0;
1756         }
1757
1758 #ifdef CONFIG_NET_DMA
1759         tcp_service_net_dma(sk, true);  /* Wait for queue to drain */
1760         tp->ucopy.dma_chan = NULL;
1761
1762         if (tp->ucopy.pinned_list) {
1763                 dma_unpin_iovec_pages(tp->ucopy.pinned_list);
1764                 tp->ucopy.pinned_list = NULL;
1765         }
1766 #endif
1767
1768         /* According to UNIX98, msg_name/msg_namelen are ignored
1769          * on connected socket. I was just happy when found this 8) --ANK
1770          */
1771
1772         /* Clean up data we have read: This will do ACK frames. */
1773         tcp_cleanup_rbuf(sk, copied);
1774
1775         release_sock(sk);
1776         return copied;
1777
1778 out:
1779         release_sock(sk);
1780         return err;
1781
1782 recv_urg:
1783         err = tcp_recv_urg(sk, msg, len, flags);
1784         goto out;
1785 }
1786 EXPORT_SYMBOL(tcp_recvmsg);
1787
1788 void tcp_set_state(struct sock *sk, int state)
1789 {
1790         int oldstate = sk->sk_state;
1791
1792         switch (state) {
1793         case TCP_ESTABLISHED:
1794                 if (oldstate != TCP_ESTABLISHED)
1795                         TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
1796                 break;
1797
1798         case TCP_CLOSE:
1799                 if (oldstate == TCP_CLOSE_WAIT || oldstate == TCP_ESTABLISHED)
1800                         TCP_INC_STATS(sock_net(sk), TCP_MIB_ESTABRESETS);
1801
1802                 sk->sk_prot->unhash(sk);
1803                 if (inet_csk(sk)->icsk_bind_hash &&
1804                     !(sk->sk_userlocks & SOCK_BINDPORT_LOCK))
1805                         inet_put_port(sk);
1806                 /* fall through */
1807         default:
1808                 if (oldstate == TCP_ESTABLISHED)
1809                         TCP_DEC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
1810         }
1811
1812         /* Change state AFTER socket is unhashed to avoid closed
1813          * socket sitting in hash tables.
1814          */
1815         sk->sk_state = state;
1816
1817 #ifdef STATE_TRACE
1818         SOCK_DEBUG(sk, "TCP sk=%p, State %s -> %s\n", sk, statename[oldstate], statename[state]);
1819 #endif
1820 }
1821 EXPORT_SYMBOL_GPL(tcp_set_state);
1822
1823 /*
1824  *      State processing on a close. This implements the state shift for
1825  *      sending our FIN frame. Note that we only send a FIN for some
1826  *      states. A shutdown() may have already sent the FIN, or we may be
1827  *      closed.
1828  */
1829
1830 static const unsigned char new_state[16] = {
1831   /* current state:        new state:      action:      */
1832   /* (Invalid)          */ TCP_CLOSE,
1833   /* TCP_ESTABLISHED    */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
1834   /* TCP_SYN_SENT       */ TCP_CLOSE,
1835   /* TCP_SYN_RECV       */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
1836   /* TCP_FIN_WAIT1      */ TCP_FIN_WAIT1,
1837   /* TCP_FIN_WAIT2      */ TCP_FIN_WAIT2,
1838   /* TCP_TIME_WAIT      */ TCP_CLOSE,
1839   /* TCP_CLOSE          */ TCP_CLOSE,
1840   /* TCP_CLOSE_WAIT     */ TCP_LAST_ACK  | TCP_ACTION_FIN,
1841   /* TCP_LAST_ACK       */ TCP_LAST_ACK,
1842   /* TCP_LISTEN         */ TCP_CLOSE,
1843   /* TCP_CLOSING        */ TCP_CLOSING,
1844 };
1845
1846 static int tcp_close_state(struct sock *sk)
1847 {
1848         int next = (int)new_state[sk->sk_state];
1849         int ns = next & TCP_STATE_MASK;
1850
1851         tcp_set_state(sk, ns);
1852
1853         return next & TCP_ACTION_FIN;
1854 }
1855
1856 /*
1857  *      Shutdown the sending side of a connection. Much like close except
1858  *      that we don't receive shut down or sock_set_flag(sk, SOCK_DEAD).
1859  */
1860
1861 void tcp_shutdown(struct sock *sk, int how)
1862 {
1863         /*      We need to grab some memory, and put together a FIN,
1864          *      and then put it into the queue to be sent.
1865          *              Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92.
1866          */
1867         if (!(how & SEND_SHUTDOWN))
1868                 return;
1869
1870         /* If we've already sent a FIN, or it's a closed state, skip this. */
1871         if ((1 << sk->sk_state) &
1872             (TCPF_ESTABLISHED | TCPF_SYN_SENT |
1873              TCPF_SYN_RECV | TCPF_CLOSE_WAIT)) {
1874                 /* Clear out any half completed packets.  FIN if needed. */
1875                 if (tcp_close_state(sk))
1876                         tcp_send_fin(sk);
1877         }
1878 }
1879 EXPORT_SYMBOL(tcp_shutdown);
1880
1881 void tcp_close(struct sock *sk, long timeout)
1882 {
1883         struct sk_buff *skb;
1884         int data_was_unread = 0;
1885         int state;
1886
1887         lock_sock(sk);
1888         sk->sk_shutdown = SHUTDOWN_MASK;
1889
1890         if (sk->sk_state == TCP_LISTEN) {
1891                 tcp_set_state(sk, TCP_CLOSE);
1892
1893                 /* Special case. */
1894                 inet_csk_listen_stop(sk);
1895
1896                 goto adjudge_to_death;
1897         }
1898
1899         /*  We need to flush the recv. buffs.  We do this only on the
1900          *  descriptor close, not protocol-sourced closes, because the
1901          *  reader process may not have drained the data yet!
1902          */
1903         while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
1904                 u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq -
1905                           tcp_hdr(skb)->fin;
1906                 data_was_unread += len;
1907                 __kfree_skb(skb);
1908         }
1909
1910         sk_mem_reclaim(sk);
1911
1912         /* If socket has been already reset (e.g. in tcp_reset()) - kill it. */
1913         if (sk->sk_state == TCP_CLOSE)
1914                 goto adjudge_to_death;
1915
1916         /* As outlined in RFC 2525, section 2.17, we send a RST here because
1917          * data was lost. To witness the awful effects of the old behavior of
1918          * always doing a FIN, run an older 2.1.x kernel or 2.0.x, start a bulk
1919          * GET in an FTP client, suspend the process, wait for the client to
1920          * advertise a zero window, then kill -9 the FTP client, wheee...
1921          * Note: timeout is always zero in such a case.
1922          */
1923         if (data_was_unread) {
1924                 /* Unread data was tossed, zap the connection. */
1925                 NET_INC_STATS_USER(sock_net(sk), LINUX_MIB_TCPABORTONCLOSE);
1926                 tcp_set_state(sk, TCP_CLOSE);
1927                 tcp_send_active_reset(sk, sk->sk_allocation);
1928         } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
1929                 /* Check zero linger _after_ checking for unread data. */
1930                 sk->sk_prot->disconnect(sk, 0);
1931                 NET_INC_STATS_USER(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
1932         } else if (tcp_close_state(sk)) {
1933                 /* We FIN if the application ate all the data before
1934                  * zapping the connection.
1935                  */
1936
1937                 /* RED-PEN. Formally speaking, we have broken TCP state
1938                  * machine. State transitions:
1939                  *
1940                  * TCP_ESTABLISHED -> TCP_FIN_WAIT1
1941                  * TCP_SYN_RECV -> TCP_FIN_WAIT1 (forget it, it's impossible)
1942                  * TCP_CLOSE_WAIT -> TCP_LAST_ACK
1943                  *
1944                  * are legal only when FIN has been sent (i.e. in window),
1945                  * rather than queued out of window. Purists blame.
1946                  *
1947                  * F.e. "RFC state" is ESTABLISHED,
1948                  * if Linux state is FIN-WAIT-1, but FIN is still not sent.
1949                  *
1950                  * The visible declinations are that sometimes
1951                  * we enter time-wait state, when it is not required really
1952                  * (harmless), do not send active resets, when they are
1953                  * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when
1954                  * they look as CLOSING or LAST_ACK for Linux)
1955                  * Probably, I missed some more holelets.
1956                  *                                              --ANK
1957                  */
1958                 tcp_send_fin(sk);
1959         }
1960
1961         sk_stream_wait_close(sk, timeout);
1962
1963 adjudge_to_death:
1964         state = sk->sk_state;
1965         sock_hold(sk);
1966         sock_orphan(sk);
1967
1968         /* It is the last release_sock in its life. It will remove backlog. */
1969         release_sock(sk);
1970
1971
1972         /* Now socket is owned by kernel and we acquire BH lock
1973            to finish close. No need to check for user refs.
1974          */
1975         local_bh_disable();
1976         bh_lock_sock(sk);
1977         WARN_ON(sock_owned_by_user(sk));
1978
1979         percpu_counter_inc(sk->sk_prot->orphan_count);
1980
1981         /* Have we already been destroyed by a softirq or backlog? */
1982         if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
1983                 goto out;
1984
1985         /*      This is a (useful) BSD violating of the RFC. There is a
1986          *      problem with TCP as specified in that the other end could
1987          *      keep a socket open forever with no application left this end.
1988          *      We use a 3 minute timeout (about the same as BSD) then kill
1989          *      our end. If they send after that then tough - BUT: long enough
1990          *      that we won't make the old 4*rto = almost no time - whoops
1991          *      reset mistake.
1992          *
1993          *      Nope, it was not mistake. It is really desired behaviour
1994          *      f.e. on http servers, when such sockets are useless, but
1995          *      consume significant resources. Let's do it with special
1996          *      linger2 option.                                 --ANK
1997          */
1998
1999         if (sk->sk_state == TCP_FIN_WAIT2) {
2000                 struct tcp_sock *tp = tcp_sk(sk);
2001                 if (tp->linger2 < 0) {
2002                         tcp_set_state(sk, TCP_CLOSE);
2003                         tcp_send_active_reset(sk, GFP_ATOMIC);
2004                         NET_INC_STATS_BH(sock_net(sk),
2005                                         LINUX_MIB_TCPABORTONLINGER);
2006                 } else {
2007                         const int tmo = tcp_fin_time(sk);
2008
2009                         if (tmo > TCP_TIMEWAIT_LEN) {
2010                                 inet_csk_reset_keepalive_timer(sk,
2011                                                 tmo - TCP_TIMEWAIT_LEN);
2012                         } else {
2013                                 tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
2014                                 goto out;
2015                         }
2016                 }
2017         }
2018         if (sk->sk_state != TCP_CLOSE) {
2019                 sk_mem_reclaim(sk);
2020                 if (tcp_too_many_orphans(sk, 0)) {
2021                         if (net_ratelimit())
2022                                 printk(KERN_INFO "TCP: too many of orphaned "
2023                                        "sockets\n");
2024                         tcp_set_state(sk, TCP_CLOSE);
2025                         tcp_send_active_reset(sk, GFP_ATOMIC);
2026                         NET_INC_STATS_BH(sock_net(sk),
2027                                         LINUX_MIB_TCPABORTONMEMORY);
2028                 }
2029         }
2030
2031         if (sk->sk_state == TCP_CLOSE)
2032                 inet_csk_destroy_sock(sk);
2033         /* Otherwise, socket is reprieved until protocol close. */
2034
2035 out:
2036         bh_unlock_sock(sk);
2037         local_bh_enable();
2038         sock_put(sk);
2039 }
2040 EXPORT_SYMBOL(tcp_close);
2041
2042 /* These states need RST on ABORT according to RFC793 */
2043
2044 static inline int tcp_need_reset(int state)
2045 {
2046         return (1 << state) &
2047                (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 |
2048                 TCPF_FIN_WAIT2 | TCPF_SYN_RECV);
2049 }
2050
2051 int tcp_disconnect(struct sock *sk, int flags)
2052 {
2053         struct inet_sock *inet = inet_sk(sk);
2054         struct inet_connection_sock *icsk = inet_csk(sk);
2055         struct tcp_sock *tp = tcp_sk(sk);
2056         int err = 0;
2057         int old_state = sk->sk_state;
2058
2059         if (old_state != TCP_CLOSE)
2060                 tcp_set_state(sk, TCP_CLOSE);
2061
2062         /* ABORT function of RFC793 */
2063         if (old_state == TCP_LISTEN) {
2064                 inet_csk_listen_stop(sk);
2065         } else if (tcp_need_reset(old_state) ||
2066                    (tp->snd_nxt != tp->write_seq &&
2067                     (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK))) {
2068                 /* The last check adjusts for discrepancy of Linux wrt. RFC
2069                  * states
2070                  */
2071                 tcp_send_active_reset(sk, gfp_any());
2072                 sk->sk_err = ECONNRESET;
2073         } else if (old_state == TCP_SYN_SENT)
2074                 sk->sk_err = ECONNRESET;
2075
2076         tcp_clear_xmit_timers(sk);
2077         __skb_queue_purge(&sk->sk_receive_queue);
2078         tcp_write_queue_purge(sk);
2079         __skb_queue_purge(&tp->out_of_order_queue);
2080 #ifdef CONFIG_NET_DMA
2081         __skb_queue_purge(&sk->sk_async_wait_queue);
2082 #endif
2083
2084         inet->inet_dport = 0;
2085
2086         if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK))
2087                 inet_reset_saddr(sk);
2088
2089         sk->sk_shutdown = 0;
2090         sock_reset_flag(sk, SOCK_DONE);
2091         tp->srtt = 0;
2092         if ((tp->write_seq += tp->max_window + 2) == 0)
2093                 tp->write_seq = 1;
2094         icsk->icsk_backoff = 0;
2095         tp->snd_cwnd = 2;
2096         icsk->icsk_probes_out = 0;
2097         tp->packets_out = 0;
2098         tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
2099         tp->snd_cwnd_cnt = 0;
2100         tp->bytes_acked = 0;
2101         tp->window_clamp = 0;
2102         tcp_set_ca_state(sk, TCP_CA_Open);
2103         tcp_clear_retrans(tp);
2104         inet_csk_delack_init(sk);
2105         tcp_init_send_head(sk);
2106         memset(&tp->rx_opt, 0, sizeof(tp->rx_opt));
2107         __sk_dst_reset(sk);
2108
2109         WARN_ON(inet->inet_num && !icsk->icsk_bind_hash);
2110
2111         sk->sk_error_report(sk);
2112         return err;
2113 }
2114 EXPORT_SYMBOL(tcp_disconnect);
2115
2116 /*
2117  *      Socket option code for TCP.
2118  */
2119 static int do_tcp_setsockopt(struct sock *sk, int level,
2120                 int optname, char __user *optval, unsigned int optlen)
2121 {
2122         struct tcp_sock *tp = tcp_sk(sk);
2123         struct inet_connection_sock *icsk = inet_csk(sk);
2124         int val;
2125         int err = 0;
2126
2127         /* These are data/string values, all the others are ints */
2128         switch (optname) {
2129         case TCP_CONGESTION: {
2130                 char name[TCP_CA_NAME_MAX];
2131
2132                 if (optlen < 1)
2133                         return -EINVAL;
2134
2135                 val = strncpy_from_user(name, optval,
2136                                         min_t(long, TCP_CA_NAME_MAX-1, optlen));
2137                 if (val < 0)
2138                         return -EFAULT;
2139                 name[val] = 0;
2140
2141                 lock_sock(sk);
2142                 err = tcp_set_congestion_control(sk, name);
2143                 release_sock(sk);
2144                 return err;
2145         }
2146         case TCP_COOKIE_TRANSACTIONS: {
2147                 struct tcp_cookie_transactions ctd;
2148                 struct tcp_cookie_values *cvp = NULL;
2149
2150                 if (sizeof(ctd) > optlen)
2151                         return -EINVAL;
2152                 if (copy_from_user(&ctd, optval, sizeof(ctd)))
2153                         return -EFAULT;
2154
2155                 if (ctd.tcpct_used > sizeof(ctd.tcpct_value) ||
2156                     ctd.tcpct_s_data_desired > TCP_MSS_DESIRED)
2157                         return -EINVAL;
2158
2159                 if (ctd.tcpct_cookie_desired == 0) {
2160                         /* default to global value */
2161                 } else if ((0x1 & ctd.tcpct_cookie_desired) ||
2162                            ctd.tcpct_cookie_desired > TCP_COOKIE_MAX ||
2163                            ctd.tcpct_cookie_desired < TCP_COOKIE_MIN) {
2164                         return -EINVAL;
2165                 }
2166
2167                 if (TCP_COOKIE_OUT_NEVER & ctd.tcpct_flags) {
2168                         /* Supercedes all other values */
2169                         lock_sock(sk);
2170                         if (tp->cookie_values != NULL) {
2171                                 kref_put(&tp->cookie_values->kref,
2172                                          tcp_cookie_values_release);
2173                                 tp->cookie_values = NULL;
2174                         }
2175                         tp->rx_opt.cookie_in_always = 0; /* false */
2176                         tp->rx_opt.cookie_out_never = 1; /* true */
2177                         release_sock(sk);
2178                         return err;
2179                 }
2180
2181                 /* Allocate ancillary memory before locking.
2182                  */
2183                 if (ctd.tcpct_used > 0 ||
2184                     (tp->cookie_values == NULL &&
2185                      (sysctl_tcp_cookie_size > 0 ||
2186                       ctd.tcpct_cookie_desired > 0 ||
2187                       ctd.tcpct_s_data_desired > 0))) {
2188                         cvp = kzalloc(sizeof(*cvp) + ctd.tcpct_used,
2189                                       GFP_KERNEL);
2190                         if (cvp == NULL)
2191                                 return -ENOMEM;
2192
2193                         kref_init(&cvp->kref);
2194                 }
2195                 lock_sock(sk);
2196                 tp->rx_opt.cookie_in_always =
2197                         (TCP_COOKIE_IN_ALWAYS & ctd.tcpct_flags);
2198                 tp->rx_opt.cookie_out_never = 0; /* false */
2199
2200                 if (tp->cookie_values != NULL) {
2201                         if (cvp != NULL) {
2202                                 /* Changed values are recorded by a changed
2203                                  * pointer, ensuring the cookie will differ,
2204                                  * without separately hashing each value later.
2205                                  */
2206                                 kref_put(&tp->cookie_values->kref,
2207                                          tcp_cookie_values_release);
2208                         } else {
2209                                 cvp = tp->cookie_values;
2210                         }
2211                 }
2212
2213                 if (cvp != NULL) {
2214                         cvp->cookie_desired = ctd.tcpct_cookie_desired;
2215
2216                         if (ctd.tcpct_used > 0) {
2217                                 memcpy(cvp->s_data_payload, ctd.tcpct_value,
2218                                        ctd.tcpct_used);
2219                                 cvp->s_data_desired = ctd.tcpct_used;
2220                                 cvp->s_data_constant = 1; /* true */
2221                         } else {
2222                                 /* No constant payload data. */
2223                                 cvp->s_data_desired = ctd.tcpct_s_data_desired;
2224                                 cvp->s_data_constant = 0; /* false */
2225                         }
2226
2227                         tp->cookie_values = cvp;
2228                 }
2229                 release_sock(sk);
2230                 return err;
2231         }
2232         default:
2233                 /* fallthru */
2234                 break;
2235         }
2236
2237         if (optlen < sizeof(int))
2238                 return -EINVAL;
2239
2240         if (get_user(val, (int __user *)optval))
2241                 return -EFAULT;
2242
2243         lock_sock(sk);
2244
2245         switch (optname) {
2246         case TCP_MAXSEG:
2247                 /* Values greater than interface MTU won't take effect. However
2248                  * at the point when this call is done we typically don't yet
2249                  * know which interface is going to be used */
2250                 if (val < TCP_MIN_MSS || val > MAX_TCP_WINDOW) {
2251                         err = -EINVAL;
2252                         break;
2253                 }
2254                 tp->rx_opt.user_mss = val;
2255                 break;
2256
2257         case TCP_NODELAY:
2258                 if (val) {
2259                         /* TCP_NODELAY is weaker than TCP_CORK, so that
2260                          * this option on corked socket is remembered, but
2261                          * it is not activated until cork is cleared.
2262                          *
2263                          * However, when TCP_NODELAY is set we make
2264                          * an explicit push, which overrides even TCP_CORK
2265                          * for currently queued segments.
2266                          */
2267                         tp->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH;
2268                         tcp_push_pending_frames(sk);
2269                 } else {
2270                         tp->nonagle &= ~TCP_NAGLE_OFF;
2271                 }
2272                 break;
2273
2274         case TCP_THIN_LINEAR_TIMEOUTS:
2275                 if (val < 0 || val > 1)
2276                         err = -EINVAL;
2277                 else
2278                         tp->thin_lto = val;
2279                 break;
2280
2281         case TCP_THIN_DUPACK:
2282                 if (val < 0 || val > 1)
2283                         err = -EINVAL;
2284                 else
2285                         tp->thin_dupack = val;
2286                 break;
2287
2288         case TCP_CORK:
2289                 /* When set indicates to always queue non-full frames.
2290                  * Later the user clears this option and we transmit
2291                  * any pending partial frames in the queue.  This is
2292                  * meant to be used alongside sendfile() to get properly
2293                  * filled frames when the user (for example) must write
2294                  * out headers with a write() call first and then use
2295                  * sendfile to send out the data parts.
2296                  *
2297                  * TCP_CORK can be set together with TCP_NODELAY and it is
2298                  * stronger than TCP_NODELAY.
2299                  */
2300                 if (val) {
2301                         tp->nonagle |= TCP_NAGLE_CORK;
2302                 } else {
2303                         tp->nonagle &= ~TCP_NAGLE_CORK;
2304                         if (tp->nonagle&TCP_NAGLE_OFF)
2305                                 tp->nonagle |= TCP_NAGLE_PUSH;
2306                         tcp_push_pending_frames(sk);
2307                 }
2308                 break;
2309
2310         case TCP_KEEPIDLE:
2311                 if (val < 1 || val > MAX_TCP_KEEPIDLE)
2312                         err = -EINVAL;
2313                 else {
2314                         tp->keepalive_time = val * HZ;
2315                         if (sock_flag(sk, SOCK_KEEPOPEN) &&
2316                             !((1 << sk->sk_state) &
2317                               (TCPF_CLOSE | TCPF_LISTEN))) {
2318                                 u32 elapsed = keepalive_time_elapsed(tp);
2319                                 if (tp->keepalive_time > elapsed)
2320                                         elapsed = tp->keepalive_time - elapsed;
2321                                 else
2322                                         elapsed = 0;
2323                                 inet_csk_reset_keepalive_timer(sk, elapsed);
2324                         }
2325                 }
2326                 break;
2327         case TCP_KEEPINTVL:
2328                 if (val < 1 || val > MAX_TCP_KEEPINTVL)
2329                         err = -EINVAL;
2330                 else
2331                         tp->keepalive_intvl = val * HZ;
2332                 break;
2333         case TCP_KEEPCNT:
2334                 if (val < 1 || val > MAX_TCP_KEEPCNT)
2335                         err = -EINVAL;
2336                 else
2337                         tp->keepalive_probes = val;
2338                 break;
2339         case TCP_SYNCNT:
2340                 if (val < 1 || val > MAX_TCP_SYNCNT)
2341                         err = -EINVAL;
2342                 else
2343                         icsk->icsk_syn_retries = val;
2344                 break;
2345
2346         case TCP_LINGER2:
2347                 if (val < 0)
2348                         tp->linger2 = -1;
2349                 else if (val > sysctl_tcp_fin_timeout / HZ)
2350                         tp->linger2 = 0;
2351                 else
2352                         tp->linger2 = val * HZ;
2353                 break;
2354
2355         case TCP_DEFER_ACCEPT:
2356                 /* Translate value in seconds to number of retransmits */
2357                 icsk->icsk_accept_queue.rskq_defer_accept =
2358                         secs_to_retrans(val, TCP_TIMEOUT_INIT / HZ,
2359                                         TCP_RTO_MAX / HZ);
2360                 break;
2361
2362         case TCP_WINDOW_CLAMP:
2363                 if (!val) {
2364                         if (sk->sk_state != TCP_CLOSE) {
2365                                 err = -EINVAL;
2366                                 break;
2367                         }
2368                         tp->window_clamp = 0;
2369                 } else
2370                         tp->window_clamp = val < SOCK_MIN_RCVBUF / 2 ?
2371                                                 SOCK_MIN_RCVBUF / 2 : val;
2372                 break;
2373
2374         case TCP_QUICKACK:
2375                 if (!val) {
2376                         icsk->icsk_ack.pingpong = 1;
2377                 } else {
2378                         icsk->icsk_ack.pingpong = 0;
2379                         if ((1 << sk->sk_state) &
2380                             (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) &&
2381                             inet_csk_ack_scheduled(sk)) {
2382                                 icsk->icsk_ack.pending |= ICSK_ACK_PUSHED;
2383                                 tcp_cleanup_rbuf(sk, 1);
2384                                 if (!(val & 1))
2385                                         icsk->icsk_ack.pingpong = 1;
2386                         }
2387                 }
2388                 break;
2389
2390 #ifdef CONFIG_TCP_MD5SIG
2391         case TCP_MD5SIG:
2392                 /* Read the IP->Key mappings from userspace */
2393                 err = tp->af_specific->md5_parse(sk, optval, optlen);
2394                 break;
2395 #endif
2396         case TCP_USER_TIMEOUT:
2397                 /* Cap the max timeout in ms TCP will retry/retrans
2398                  * before giving up and aborting (ETIMEDOUT) a connection.
2399                  */
2400                 if (val < 0)
2401                         err = -EINVAL;
2402                 else
2403                         icsk->icsk_user_timeout = msecs_to_jiffies(val);
2404                 break;
2405         default:
2406                 err = -ENOPROTOOPT;
2407                 break;
2408         }
2409
2410         release_sock(sk);
2411         return err;
2412 }
2413
2414 int tcp_setsockopt(struct sock *sk, int level, int optname, char __user *optval,
2415                    unsigned int optlen)
2416 {
2417         const struct inet_connection_sock *icsk = inet_csk(sk);
2418
2419         if (level != SOL_TCP)
2420                 return icsk->icsk_af_ops->setsockopt(sk, level, optname,
2421                                                      optval, optlen);
2422         return do_tcp_setsockopt(sk, level, optname, optval, optlen);
2423 }
2424 EXPORT_SYMBOL(tcp_setsockopt);
2425
2426 #ifdef CONFIG_COMPAT
2427 int compat_tcp_setsockopt(struct sock *sk, int level, int optname,
2428                           char __user *optval, unsigned int optlen)
2429 {
2430         if (level != SOL_TCP)
2431                 return inet_csk_compat_setsockopt(sk, level, optname,
2432                                                   optval, optlen);
2433         return do_tcp_setsockopt(sk, level, optname, optval, optlen);
2434 }
2435 EXPORT_SYMBOL(compat_tcp_setsockopt);
2436 #endif
2437
2438 /* Return information about state of tcp endpoint in API format. */
2439 void tcp_get_info(const struct sock *sk, struct tcp_info *info)
2440 {
2441         const struct tcp_sock *tp = tcp_sk(sk);
2442         const struct inet_connection_sock *icsk = inet_csk(sk);
2443         u32 now = tcp_time_stamp;
2444
2445         memset(info, 0, sizeof(*info));
2446
2447         info->tcpi_state = sk->sk_state;
2448         info->tcpi_ca_state = icsk->icsk_ca_state;
2449         info->tcpi_retransmits = icsk->icsk_retransmits;
2450         info->tcpi_probes = icsk->icsk_probes_out;
2451         info->tcpi_backoff = icsk->icsk_backoff;
2452
2453         if (tp->rx_opt.tstamp_ok)
2454                 info->tcpi_options |= TCPI_OPT_TIMESTAMPS;
2455         if (tcp_is_sack(tp))
2456                 info->tcpi_options |= TCPI_OPT_SACK;
2457         if (tp->rx_opt.wscale_ok) {
2458                 info->tcpi_options |= TCPI_OPT_WSCALE;
2459                 info->tcpi_snd_wscale = tp->rx_opt.snd_wscale;
2460                 info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale;
2461         }
2462
2463         if (tp->ecn_flags & TCP_ECN_OK)
2464                 info->tcpi_options |= TCPI_OPT_ECN;
2465         if (tp->ecn_flags & TCP_ECN_SEEN)
2466                 info->tcpi_options |= TCPI_OPT_ECN_SEEN;
2467
2468         info->tcpi_rto = jiffies_to_usecs(icsk->icsk_rto);
2469         info->tcpi_ato = jiffies_to_usecs(icsk->icsk_ack.ato);
2470         info->tcpi_snd_mss = tp->mss_cache;
2471         info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss;
2472
2473         if (sk->sk_state == TCP_LISTEN) {
2474                 info->tcpi_unacked = sk->sk_ack_backlog;
2475                 info->tcpi_sacked = sk->sk_max_ack_backlog;
2476         } else {
2477                 info->tcpi_unacked = tp->packets_out;
2478                 info->tcpi_sacked = tp->sacked_out;
2479         }
2480         info->tcpi_lost = tp->lost_out;
2481         info->tcpi_retrans = tp->retrans_out;
2482         info->tcpi_fackets = tp->fackets_out;
2483
2484         info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime);
2485         info->tcpi_last_data_recv = jiffies_to_msecs(now - icsk->icsk_ack.lrcvtime);
2486         info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp);
2487
2488         info->tcpi_pmtu = icsk->icsk_pmtu_cookie;
2489         info->tcpi_rcv_ssthresh = tp->rcv_ssthresh;
2490         info->tcpi_rtt = jiffies_to_usecs(tp->srtt)>>3;
2491         info->tcpi_rttvar = jiffies_to_usecs(tp->mdev)>>2;
2492         info->tcpi_snd_ssthresh = tp->snd_ssthresh;
2493         info->tcpi_snd_cwnd = tp->snd_cwnd;
2494         info->tcpi_advmss = tp->advmss;
2495         info->tcpi_reordering = tp->reordering;
2496
2497         info->tcpi_rcv_rtt = jiffies_to_usecs(tp->rcv_rtt_est.rtt)>>3;
2498         info->tcpi_rcv_space = tp->rcvq_space.space;
2499
2500         info->tcpi_total_retrans = tp->total_retrans;
2501 }
2502 EXPORT_SYMBOL_GPL(tcp_get_info);
2503
2504 static int do_tcp_getsockopt(struct sock *sk, int level,
2505                 int optname, char __user *optval, int __user *optlen)
2506 {
2507         struct inet_connection_sock *icsk = inet_csk(sk);
2508         struct tcp_sock *tp = tcp_sk(sk);
2509         int val, len;
2510
2511         if (get_user(len, optlen))
2512                 return -EFAULT;
2513
2514         len = min_t(unsigned int, len, sizeof(int));
2515
2516         if (len < 0)
2517                 return -EINVAL;
2518
2519         switch (optname) {
2520         case TCP_MAXSEG:
2521                 val = tp->mss_cache;
2522                 if (!val && ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
2523                         val = tp->rx_opt.user_mss;
2524                 break;
2525         case TCP_NODELAY:
2526                 val = !!(tp->nonagle&TCP_NAGLE_OFF);
2527                 break;
2528         case TCP_CORK:
2529                 val = !!(tp->nonagle&TCP_NAGLE_CORK);
2530                 break;
2531         case TCP_KEEPIDLE:
2532                 val = keepalive_time_when(tp) / HZ;
2533                 break;
2534         case TCP_KEEPINTVL:
2535                 val = keepalive_intvl_when(tp) / HZ;
2536                 break;
2537         case TCP_KEEPCNT:
2538                 val = keepalive_probes(tp);
2539                 break;
2540         case TCP_SYNCNT:
2541                 val = icsk->icsk_syn_retries ? : sysctl_tcp_syn_retries;
2542                 break;
2543         case TCP_LINGER2:
2544                 val = tp->linger2;
2545                 if (val >= 0)
2546                         val = (val ? : sysctl_tcp_fin_timeout) / HZ;
2547                 break;
2548         case TCP_DEFER_ACCEPT:
2549                 val = retrans_to_secs(icsk->icsk_accept_queue.rskq_defer_accept,
2550                                       TCP_TIMEOUT_INIT / HZ, TCP_RTO_MAX / HZ);
2551                 break;
2552         case TCP_WINDOW_CLAMP:
2553                 val = tp->window_clamp;
2554                 break;
2555         case TCP_INFO: {
2556                 struct tcp_info info;
2557
2558                 if (get_user(len, optlen))
2559                         return -EFAULT;
2560
2561                 tcp_get_info(sk, &info);
2562
2563                 len = min_t(unsigned int, len, sizeof(info));
2564                 if (put_user(len, optlen))
2565                         return -EFAULT;
2566                 if (copy_to_user(optval, &info, len))
2567                         return -EFAULT;
2568                 return 0;
2569         }
2570         case TCP_QUICKACK:
2571                 val = !icsk->icsk_ack.pingpong;
2572                 break;
2573
2574         case TCP_CONGESTION:
2575                 if (get_user(len, optlen))
2576                         return -EFAULT;
2577                 len = min_t(unsigned int, len, TCP_CA_NAME_MAX);
2578                 if (put_user(len, optlen))
2579                         return -EFAULT;
2580                 if (copy_to_user(optval, icsk->icsk_ca_ops->name, len))
2581                         return -EFAULT;
2582                 return 0;
2583
2584         case TCP_COOKIE_TRANSACTIONS: {
2585                 struct tcp_cookie_transactions ctd;
2586                 struct tcp_cookie_values *cvp = tp->cookie_values;
2587
2588                 if (get_user(len, optlen))
2589                         return -EFAULT;
2590                 if (len < sizeof(ctd))
2591                         return -EINVAL;
2592
2593                 memset(&ctd, 0, sizeof(ctd));
2594                 ctd.tcpct_flags = (tp->rx_opt.cookie_in_always ?
2595                                    TCP_COOKIE_IN_ALWAYS : 0)
2596                                 | (tp->rx_opt.cookie_out_never ?
2597                                    TCP_COOKIE_OUT_NEVER : 0);
2598
2599                 if (cvp != NULL) {
2600                         ctd.tcpct_flags |= (cvp->s_data_in ?
2601                                             TCP_S_DATA_IN : 0)
2602                                          | (cvp->s_data_out ?
2603                                             TCP_S_DATA_OUT : 0);
2604
2605                         ctd.tcpct_cookie_desired = cvp->cookie_desired;
2606                         ctd.tcpct_s_data_desired = cvp->s_data_desired;
2607
2608                         memcpy(&ctd.tcpct_value[0], &cvp->cookie_pair[0],
2609                                cvp->cookie_pair_size);
2610                         ctd.tcpct_used = cvp->cookie_pair_size;
2611                 }
2612
2613                 if (put_user(sizeof(ctd), optlen))
2614                         return -EFAULT;
2615                 if (copy_to_user(optval, &ctd, sizeof(ctd)))
2616                         return -EFAULT;
2617                 return 0;
2618         }
2619         case TCP_THIN_LINEAR_TIMEOUTS:
2620                 val = tp->thin_lto;
2621                 break;
2622         case TCP_THIN_DUPACK:
2623                 val = tp->thin_dupack;
2624                 break;
2625
2626         case TCP_USER_TIMEOUT:
2627                 val = jiffies_to_msecs(icsk->icsk_user_timeout);
2628                 break;
2629         default:
2630                 return -ENOPROTOOPT;
2631         }
2632
2633         if (put_user(len, optlen))
2634                 return -EFAULT;
2635         if (copy_to_user(optval, &val, len))
2636                 return -EFAULT;
2637         return 0;
2638 }
2639
2640 int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
2641                    int __user *optlen)
2642 {
2643         struct inet_connection_sock *icsk = inet_csk(sk);
2644
2645         if (level != SOL_TCP)
2646                 return icsk->icsk_af_ops->getsockopt(sk, level, optname,
2647                                                      optval, optlen);
2648         return do_tcp_getsockopt(sk, level, optname, optval, optlen);
2649 }
2650 EXPORT_SYMBOL(tcp_getsockopt);
2651
2652 #ifdef CONFIG_COMPAT
2653 int compat_tcp_getsockopt(struct sock *sk, int level, int optname,
2654                           char __user *optval, int __user *optlen)
2655 {
2656         if (level != SOL_TCP)
2657                 return inet_csk_compat_getsockopt(sk, level, optname,
2658                                                   optval, optlen);
2659         return do_tcp_getsockopt(sk, level, optname, optval, optlen);
2660 }
2661 EXPORT_SYMBOL(compat_tcp_getsockopt);
2662 #endif
2663
2664 struct sk_buff *tcp_tso_segment(struct sk_buff *skb, u32 features)
2665 {
2666         struct sk_buff *segs = ERR_PTR(-EINVAL);
2667         struct tcphdr *th;
2668         unsigned thlen;
2669         unsigned int seq;
2670         __be32 delta;
2671         unsigned int oldlen;
2672         unsigned int mss;
2673
2674         if (!pskb_may_pull(skb, sizeof(*th)))
2675                 goto out;
2676
2677         th = tcp_hdr(skb);
2678         thlen = th->doff * 4;
2679         if (thlen < sizeof(*th))
2680                 goto out;
2681
2682         if (!pskb_may_pull(skb, thlen))
2683                 goto out;
2684
2685         oldlen = (u16)~skb->len;
2686         __skb_pull(skb, thlen);
2687
2688         mss = skb_shinfo(skb)->gso_size;
2689         if (unlikely(skb->len <= mss))
2690                 goto out;
2691
2692         if (skb_gso_ok(skb, features | NETIF_F_GSO_ROBUST)) {
2693                 /* Packet is from an untrusted source, reset gso_segs. */
2694                 int type = skb_shinfo(skb)->gso_type;
2695
2696                 if (unlikely(type &
2697                              ~(SKB_GSO_TCPV4 |
2698                                SKB_GSO_DODGY |
2699                                SKB_GSO_TCP_ECN |
2700                                SKB_GSO_TCPV6 |
2701                                0) ||
2702                              !(type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))))
2703                         goto out;
2704
2705                 skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(skb->len, mss);
2706
2707                 segs = NULL;
2708                 goto out;
2709         }
2710
2711         segs = skb_segment(skb, features);
2712         if (IS_ERR(segs))
2713                 goto out;
2714
2715         delta = htonl(oldlen + (thlen + mss));
2716
2717         skb = segs;
2718         th = tcp_hdr(skb);
2719         seq = ntohl(th->seq);
2720
2721         do {
2722                 th->fin = th->psh = 0;
2723
2724                 th->check = ~csum_fold((__force __wsum)((__force u32)th->check +
2725                                        (__force u32)delta));
2726                 if (skb->ip_summed != CHECKSUM_PARTIAL)
2727                         th->check =
2728                              csum_fold(csum_partial(skb_transport_header(skb),
2729                                                     thlen, skb->csum));
2730
2731                 seq += mss;
2732                 skb = skb->next;
2733                 th = tcp_hdr(skb);
2734
2735                 th->seq = htonl(seq);
2736                 th->cwr = 0;
2737         } while (skb->next);
2738
2739         delta = htonl(oldlen + (skb->tail - skb->transport_header) +
2740                       skb->data_len);
2741         th->check = ~csum_fold((__force __wsum)((__force u32)th->check +
2742                                 (__force u32)delta));
2743         if (skb->ip_summed != CHECKSUM_PARTIAL)
2744                 th->check = csum_fold(csum_partial(skb_transport_header(skb),
2745                                                    thlen, skb->csum));
2746
2747 out:
2748         return segs;
2749 }
2750 EXPORT_SYMBOL(tcp_tso_segment);
2751
2752 struct sk_buff **tcp_gro_receive(struct sk_buff **head, struct sk_buff *skb)
2753 {
2754         struct sk_buff **pp = NULL;
2755         struct sk_buff *p;
2756         struct tcphdr *th;
2757         struct tcphdr *th2;
2758         unsigned int len;
2759         unsigned int thlen;
2760         __be32 flags;
2761         unsigned int mss = 1;
2762         unsigned int hlen;
2763         unsigned int off;
2764         int flush = 1;
2765         int i;
2766
2767         off = skb_gro_offset(skb);
2768         hlen = off + sizeof(*th);
2769         th = skb_gro_header_fast(skb, off);
2770         if (skb_gro_header_hard(skb, hlen)) {
2771                 th = skb_gro_header_slow(skb, hlen, off);
2772                 if (unlikely(!th))
2773                         goto out;
2774         }
2775
2776         thlen = th->doff * 4;
2777         if (thlen < sizeof(*th))
2778                 goto out;
2779
2780         hlen = off + thlen;
2781         if (skb_gro_header_hard(skb, hlen)) {
2782                 th = skb_gro_header_slow(skb, hlen, off);
2783                 if (unlikely(!th))
2784                         goto out;
2785         }
2786
2787         skb_gro_pull(skb, thlen);
2788
2789         len = skb_gro_len(skb);
2790         flags = tcp_flag_word(th);
2791
2792         for (; (p = *head); head = &p->next) {
2793                 if (!NAPI_GRO_CB(p)->same_flow)
2794                         continue;
2795
2796                 th2 = tcp_hdr(p);
2797
2798                 if (*(u32 *)&th->source ^ *(u32 *)&th2->source) {
2799                         NAPI_GRO_CB(p)->same_flow = 0;
2800                         continue;
2801                 }
2802
2803                 goto found;
2804         }
2805
2806         goto out_check_final;
2807
2808 found:
2809         flush = NAPI_GRO_CB(p)->flush;
2810         flush |= (__force int)(flags & TCP_FLAG_CWR);
2811         flush |= (__force int)((flags ^ tcp_flag_word(th2)) &
2812                   ~(TCP_FLAG_CWR | TCP_FLAG_FIN | TCP_FLAG_PSH));
2813         flush |= (__force int)(th->ack_seq ^ th2->ack_seq);
2814         for (i = sizeof(*th); i < thlen; i += 4)
2815                 flush |= *(u32 *)((u8 *)th + i) ^
2816                          *(u32 *)((u8 *)th2 + i);
2817
2818         mss = skb_shinfo(p)->gso_size;
2819
2820         flush |= (len - 1) >= mss;
2821         flush |= (ntohl(th2->seq) + skb_gro_len(p)) ^ ntohl(th->seq);
2822
2823         if (flush || skb_gro_receive(head, skb)) {
2824                 mss = 1;
2825                 goto out_check_final;
2826         }
2827
2828         p = *head;
2829         th2 = tcp_hdr(p);
2830         tcp_flag_word(th2) |= flags & (TCP_FLAG_FIN | TCP_FLAG_PSH);
2831
2832 out_check_final:
2833         flush = len < mss;
2834         flush |= (__force int)(flags & (TCP_FLAG_URG | TCP_FLAG_PSH |
2835                                         TCP_FLAG_RST | TCP_FLAG_SYN |
2836                                         TCP_FLAG_FIN));
2837
2838         if (p && (!NAPI_GRO_CB(skb)->same_flow || flush))
2839                 pp = head;
2840
2841 out:
2842         NAPI_GRO_CB(skb)->flush |= flush;
2843
2844         return pp;
2845 }
2846 EXPORT_SYMBOL(tcp_gro_receive);
2847
2848 int tcp_gro_complete(struct sk_buff *skb)
2849 {
2850         struct tcphdr *th = tcp_hdr(skb);
2851
2852         skb->csum_start = skb_transport_header(skb) - skb->head;
2853         skb->csum_offset = offsetof(struct tcphdr, check);
2854         skb->ip_summed = CHECKSUM_PARTIAL;
2855
2856         skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
2857
2858         if (th->cwr)
2859                 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
2860
2861         return 0;
2862 }
2863 EXPORT_SYMBOL(tcp_gro_complete);
2864
2865 #ifdef CONFIG_TCP_MD5SIG
2866 static unsigned long tcp_md5sig_users;
2867 static struct tcp_md5sig_pool __percpu *tcp_md5sig_pool;
2868 static DEFINE_SPINLOCK(tcp_md5sig_pool_lock);
2869
2870 static void __tcp_free_md5sig_pool(struct tcp_md5sig_pool __percpu *pool)
2871 {
2872         int cpu;
2873
2874         for_each_possible_cpu(cpu) {
2875                 struct tcp_md5sig_pool *p = per_cpu_ptr(pool, cpu);
2876
2877                 if (p->md5_desc.tfm)
2878                         crypto_free_hash(p->md5_desc.tfm);
2879         }
2880         free_percpu(pool);
2881 }
2882
2883 void tcp_free_md5sig_pool(void)
2884 {
2885         struct tcp_md5sig_pool __percpu *pool = NULL;
2886
2887         spin_lock_bh(&tcp_md5sig_pool_lock);
2888         if (--tcp_md5sig_users == 0) {
2889                 pool = tcp_md5sig_pool;
2890                 tcp_md5sig_pool = NULL;
2891         }
2892         spin_unlock_bh(&tcp_md5sig_pool_lock);
2893         if (pool)
2894                 __tcp_free_md5sig_pool(pool);
2895 }
2896 EXPORT_SYMBOL(tcp_free_md5sig_pool);
2897
2898 static struct tcp_md5sig_pool __percpu *
2899 __tcp_alloc_md5sig_pool(struct sock *sk)
2900 {
2901         int cpu;
2902         struct tcp_md5sig_pool __percpu *pool;
2903
2904         pool = alloc_percpu(struct tcp_md5sig_pool);
2905         if (!pool)
2906                 return NULL;
2907
2908         for_each_possible_cpu(cpu) {
2909                 struct crypto_hash *hash;
2910
2911                 hash = crypto_alloc_hash("md5", 0, CRYPTO_ALG_ASYNC);
2912                 if (!hash || IS_ERR(hash))
2913                         goto out_free;
2914
2915                 per_cpu_ptr(pool, cpu)->md5_desc.tfm = hash;
2916         }
2917         return pool;
2918 out_free:
2919         __tcp_free_md5sig_pool(pool);
2920         return NULL;
2921 }
2922
2923 struct tcp_md5sig_pool __percpu *tcp_alloc_md5sig_pool(struct sock *sk)
2924 {
2925         struct tcp_md5sig_pool __percpu *pool;
2926         int alloc = 0;
2927
2928 retry:
2929         spin_lock_bh(&tcp_md5sig_pool_lock);
2930         pool = tcp_md5sig_pool;
2931         if (tcp_md5sig_users++ == 0) {
2932                 alloc = 1;
2933                 spin_unlock_bh(&tcp_md5sig_pool_lock);
2934         } else if (!pool) {
2935                 tcp_md5sig_users--;
2936                 spin_unlock_bh(&tcp_md5sig_pool_lock);
2937                 cpu_relax();
2938                 goto retry;
2939         } else
2940                 spin_unlock_bh(&tcp_md5sig_pool_lock);
2941
2942         if (alloc) {
2943                 /* we cannot hold spinlock here because this may sleep. */
2944                 struct tcp_md5sig_pool __percpu *p;
2945
2946                 p = __tcp_alloc_md5sig_pool(sk);
2947                 spin_lock_bh(&tcp_md5sig_pool_lock);
2948                 if (!p) {
2949                         tcp_md5sig_users--;
2950                         spin_unlock_bh(&tcp_md5sig_pool_lock);
2951                         return NULL;
2952                 }
2953                 pool = tcp_md5sig_pool;
2954                 if (pool) {
2955                         /* oops, it has already been assigned. */
2956                         spin_unlock_bh(&tcp_md5sig_pool_lock);
2957                         __tcp_free_md5sig_pool(p);
2958                 } else {
2959                         tcp_md5sig_pool = pool = p;
2960                         spin_unlock_bh(&tcp_md5sig_pool_lock);
2961                 }
2962         }
2963         return pool;
2964 }
2965 EXPORT_SYMBOL(tcp_alloc_md5sig_pool);
2966
2967
2968 /**
2969  *      tcp_get_md5sig_pool - get md5sig_pool for this user
2970  *
2971  *      We use percpu structure, so if we succeed, we exit with preemption
2972  *      and BH disabled, to make sure another thread or softirq handling
2973  *      wont try to get same context.
2974  */
2975 struct tcp_md5sig_pool *tcp_get_md5sig_pool(void)
2976 {
2977         struct tcp_md5sig_pool __percpu *p;
2978
2979         local_bh_disable();
2980
2981         spin_lock(&tcp_md5sig_pool_lock);
2982         p = tcp_md5sig_pool;
2983         if (p)
2984                 tcp_md5sig_users++;
2985         spin_unlock(&tcp_md5sig_pool_lock);
2986
2987         if (p)
2988             &nbs