[src/trunk]: src/sys/netinet Fix unsigned wraparound on window s...

[uwe <uwe%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/c02a1472c722
branches:  trunk
changeset: 318866:c02a1472c722
user:      uwe <uwe%NetBSD.org@localhost>
date:      Mon May 07 23:42:13 2018 +0000
description:
Fix unsigned wraparound on window size calculations.

This is another instance where tp->rcv_adv - tp->rcv_nxt can wrap
around after successful zero-window probe from the peer.  The first
one was fixed by chs@ in revision 1.112 on 2004-05-08.

While here, CSE and de-obfuscate the code a bit.

diffstat:

 sys/netinet/tcp_output.c |  27 +++++++++++++++++++--------
 1 files changed, 19 insertions(+), 8 deletions(-)

diffs (52 lines):

diff -r 519f79dc3fc7 -r c02a1472c722 sys/netinet/tcp_output.c
--- a/sys/netinet/tcp_output.c  Mon May 07 21:03:45 2018 +0000
+++ b/sys/netinet/tcp_output.c  Mon May 07 23:42:13 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: tcp_output.c,v 1.206 2018/05/03 07:13:48 maxv Exp $    */
+/*     $NetBSD: tcp_output.c,v 1.207 2018/05/07 23:42:13 uwe Exp $     */
 
 /*
  * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -135,7 +135,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: tcp_output.c,v 1.206 2018/05/03 07:13:48 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: tcp_output.c,v 1.207 2018/05/07 23:42:13 uwe Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
@@ -987,16 +987,27 @@
                 * taking into account that we are limited by
                 * TCP_MAXWIN << tp->rcv_scale.
                 */
-               long adv = min(win, (long)TCP_MAXWIN << tp->rcv_scale) -
-                       (tp->rcv_adv - tp->rcv_nxt);
+               long recwin = min(win, (long)TCP_MAXWIN << tp->rcv_scale);
+               long oldwin, adv;
 
                /*
-                * If the new window size ends up being the same as the old
-                * size when it is scaled, then don't force a window update.
+                * rcv_nxt may overtake rcv_adv when we accept a
+                * zero-window probe.
                 */
-               if ((tp->rcv_adv - tp->rcv_nxt) >> tp->rcv_scale ==
-                   (adv + tp->rcv_adv - tp->rcv_nxt) >> tp->rcv_scale)
+               if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt))
+                       oldwin = tp->rcv_adv - tp->rcv_nxt;
+               else
+                       oldwin = 0;
+
+               /*
+                * If the new window size ends up being the same as or
+                * less than the old size when it is scaled, then
+                * don't force a window update.
+                */
+               if (recwin >> tp->rcv_scale <= oldwin >> tp->rcv_scale)
                        goto dontupdate;
+
+               adv = recwin - oldwin;
                if (adv >= (long) (2 * rxsegsize))
                        goto send;
                if (2 * adv >= (long) so->so_rcv.sb_hiwat)