Detecting buffer overflows

[Roy Marples <roy%marples.name@localhost>]

Hi

I'm currently dealing with an issue where a route(4) socket is flooded 
with RTM_MISS on my router when the default route vanishes.
The kernel will silently discard messages all route(4) messages once the 
internal buffer overflows. Side effects of this include ifwatchd and 
dhcpcd not seeing carrier up/down or address addition/removal/dad 
completion events. wpa_supplicant will also miss various IEEE80211 events.

Adding a knob to disable RTM_MISS might seem the easy answer, but it's 
also trivial to flood RTM_NEWADDR messages as well from userland 
(provided you have permissions to add them), resulting in the similar 
DoS attack against listeners.

We have the undocumented SO_OVERFLOW getsockopt facility to get the 
number of overflows on a socket but means an extra syscall per read just 
to work out if we overflowed or not.
Also, this isn't enabled for RAW sockets which is what route(4) uses.

Attached is a patch that introduces the kevent(2) NOTE_OVERFLOW which is 
set when the socket receives buffer overflows.

ifwatchd and dhcpcd can then spot the overflow, flush the buffer (like 
closing and re-opening the socket), re-examine the current state of 
affairs and do the right thing.
wpa_supplicant can't do much other than just log that it happened.

Commentary on this welcome, especially the kqueue specific parts.

Roy

Index: sys/kern/uipc_socket.c
===================================================================
RCS file: /cvsroot/src/sys/kern/uipc_socket.c,v
retrieving revision 1.259
diff -u -p -r1.259 uipc_socket.c
--- sys/kern/uipc_socket.c	4 Jan 2018 01:42:25 -0000	1.259
+++ sys/kern/uipc_socket.c	7 Mar 2018 21:29:25 -0000
@@ -2247,6 +2247,8 @@ filt_soread(struct knote *kn, long hint)
 	if (hint != NOTE_SUBMIT)
 		solock(so);
 	kn->kn_data = so->so_rcv.sb_cc;
+	if (so->so_rcv.sb_overflowed)
+		kn->kn_fflags = NOTE_OVERFLOW;
 	if (so->so_state & SS_CANTRCVMORE) {
 		kn->kn_flags |= EV_EOF;
 		kn->kn_fflags = so->so_error;
Index: sys/kern/uipc_socket2.c
===================================================================
RCS file: /cvsroot/src/sys/kern/uipc_socket2.c,v
retrieving revision 1.126
diff -u -p -r1.126 uipc_socket2.c
--- sys/kern/uipc_socket2.c	6 Jul 2017 17:42:39 -0000	1.126
+++ sys/kern/uipc_socket2.c	7 Mar 2018 21:29:25 -0000
@@ -495,6 +495,19 @@ socantrcvmore(struct socket *so)
 }
 
 /*
+ * soroverflow(): indicates that data was attempted to be sent
+ * but the receiving buffer overflowed.
+ */
+void
+soroverflow(struct socket *so)
+{
+	KASSERT(solocked(so));
+
+	so->so_rcv.sb_overflowed++;
+	sorwakeup(so);
+}
+
+/*
  * Wait for data to arrive at/drain from a socket buffer.
  */
 int
Index: sys/kern/uipc_usrreq.c
===================================================================
RCS file: /cvsroot/src/sys/kern/uipc_usrreq.c,v
retrieving revision 1.183
diff -u -p -r1.183 uipc_usrreq.c
--- sys/kern/uipc_usrreq.c	17 Feb 2018 20:19:36 -0000	1.183
+++ sys/kern/uipc_usrreq.c	7 Mar 2018 21:29:25 -0000
@@ -342,10 +342,10 @@ unp_output(struct mbuf *m, struct mbuf *
 #endif
 	if (sbappendaddr(&so2->so_rcv, (const struct sockaddr *)sun, m,
 	    control) == 0) {
-		so2->so_rcv.sb_overflowed++;
 		unp_dispose(control);
 		m_freem(control);
 		m_freem(m);
+		soroverflow(so2);
 		return (ENOBUFS);
 	} else {
 		sorwakeup(so2);
Index: sys/net/raw_usrreq.c
===================================================================
RCS file: /cvsroot/src/sys/net/raw_usrreq.c,v
retrieving revision 1.58
diff -u -p -r1.58 raw_usrreq.c
--- sys/net/raw_usrreq.c	25 Sep 2017 01:57:54 -0000	1.58
+++ sys/net/raw_usrreq.c	7 Mar 2018 21:29:25 -0000
@@ -106,21 +106,26 @@ raw_input(struct mbuf *m0, ...)
 			continue;
 		if (last != NULL) {
 			struct mbuf *n;
-			if ((n = m_copy(m, 0, M_COPYALL)) == NULL)
-				;
-			else if (sbappendaddr(&last->so_rcv, src, n, NULL) == 0)
-				/* should notify about lost packet */
-				m_freem(n);
-			else {
+
+			if ((n = m_copy(m, 0, M_COPYALL)) == NULL ||
+			    sbappendaddr(&last->so_rcv, src, n, NULL) == 0)
+			{
+				if (n != NULL)
+					m_freem(n);
+				soroverflow(last);
+			} else
 				sorwakeup(last);
-			}
 		}
 		last = rp->rcb_socket;
 	}
-	if (last == NULL || sbappendaddr(&last->so_rcv, src, m, NULL) == 0)
-		m_freem(m);
-	else {
-		sorwakeup(last);
+	if (last != NULL) {
+		if (sbappendaddr(&last->so_rcv, src, m, NULL) == 0) {
+			m_free(m);
+			soroverflow(last);
+		} else
+			sorwakeup(last);
+	} else {
+		m_free(m);
 	}
 }
 
Index: sys/netinet/udp_usrreq.c
===================================================================
RCS file: /cvsroot/src/sys/netinet/udp_usrreq.c,v
retrieving revision 1.245
diff -u -p -r1.245 udp_usrreq.c
--- sys/netinet/udp_usrreq.c	28 Feb 2018 11:23:24 -0000	1.245
+++ sys/netinet/udp_usrreq.c	7 Mar 2018 21:29:25 -0000
@@ -498,8 +498,8 @@ udp4_sendup(struct mbuf *m, int off /* o
 			m_freem(n);
 			if (opts)
 				m_freem(opts);
-			so->so_rcv.sb_overflowed++;
 			UDP_STATINC(UDP_STAT_FULLSOCK);
+			soroverflow(so);
 		} else
 			sorwakeup(so);
 	}
Index: sys/netinet6/udp6_usrreq.c
===================================================================
RCS file: /cvsroot/src/sys/netinet6/udp6_usrreq.c,v
retrieving revision 1.137
diff -u -p -r1.137 udp6_usrreq.c
--- sys/netinet6/udp6_usrreq.c	28 Feb 2018 11:23:24 -0000	1.137
+++ sys/netinet6/udp6_usrreq.c	7 Mar 2018 21:29:26 -0000
@@ -372,8 +372,8 @@ udp6_sendup(struct mbuf *m, int off /* o
 			m_freem(n);
 			if (opts)
 				m_freem(opts);
-			so->so_rcv.sb_overflowed++;
 			UDP6_STATINC(UDP6_STAT_FULLSOCK);
+			soroverflow(so);
 		} else
 			sorwakeup(so);
 	}
Index: sys/netipsec/keysock.c
===================================================================
RCS file: /cvsroot/src/sys/netipsec/keysock.c,v
retrieving revision 1.62
diff -u -p -r1.62 keysock.c
--- sys/netipsec/keysock.c	28 Sep 2017 17:21:42 -0000	1.62
+++ sys/netipsec/keysock.c	7 Mar 2018 21:29:26 -0000
@@ -207,11 +207,12 @@ key_sendup0(
 		    __func__);
 		PFKEY_STATINC(PFKEY_STAT_IN_NOMEM);
 		m_freem(m);
+		soroverflow(rp->rcb_socket);
 		error = ENOBUFS;
-		rp->rcb_socket->so_rcv.sb_overflowed++;
-	} else
+	} else {
+		sorwakeup(rp->rcb_socket);
 		error = 0;
-	sorwakeup(rp->rcb_socket);
+	}
 	return error;
 }
 
Index: sys/sys/event.h
===================================================================
RCS file: /cvsroot/src/sys/sys/event.h,v
retrieving revision 1.32
diff -u -p -r1.32 event.h
--- sys/sys/event.h	9 Jan 2018 03:31:13 -0000	1.32
+++ sys/sys/event.h	7 Mar 2018 21:29:26 -0000
@@ -100,6 +100,7 @@ _EV_SET(struct kevent *_kevp, uintptr_t 
  * data/hint flags for EVFILT_{READ|WRITE}, shared with userspace
  */
 #define	NOTE_LOWAT	0x0001U			/* low water mark */
+#define	NOTE_OVERFLOW	0x0002U			/* buffer overflowed */
 
 /*
  * data/hint flags for EVFILT_VNODE, shared with userspace
Index: sys/sys/socketvar.h
===================================================================
RCS file: /cvsroot/src/sys/sys/socketvar.h,v
retrieving revision 1.146
diff -u -p -r1.146 socketvar.h
--- sys/sys/socketvar.h	4 Jan 2018 01:42:25 -0000	1.146
+++ sys/sys/socketvar.h	7 Mar 2018 21:29:26 -0000
@@ -300,6 +300,7 @@ int	sofamily(const struct socket *);
 int	sobind(struct socket *, struct sockaddr *, struct lwp *);
 void	socantrcvmore(struct socket *);
 void	socantsendmore(struct socket *);
+void	soroverflow(struct socket *);
 int	soclose(struct socket *);
 int	soconnect(struct socket *, struct sockaddr *, struct lwp *);
 int	soconnect2(struct socket *, struct socket *);