[src/trunk]: src/sys/sys Introduce KCOV remote support. This allows to collec...

[maxv <maxv%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/2d91237ba44d
branches:  trunk
changeset: 972081:2d91237ba44d
user:      maxv <maxv%NetBSD.org@localhost>
date:      Fri May 15 12:34:52 2020 +0000

description:
Introduce KCOV remote support. This allows to collect KCOV coverage on
threads other than curlwp, which is useful when fuzzing components that
defer processing, such as the network stack (partially runs in softints)
and the USB stack (partially runs in uhub kthreads).

A subsystem that whishes to provide coverage for its threads creates a
"mailbox" via kcov_remote_register() and gives it a (subsystem, id)
identifier. There is one mailbox per "target lwp". The target lwp(s)
must then call kcov_remote_enter() and kcov_remote_leave() with the
identifier, to respectively enable and disable coverage within the
thread.

On the userland side, the fuzzer has access to the mailboxes on the
system with the KCOV_IOC_REMOTE_ATTACH and KCOV_IOC_REMOTE_DETACH ioctls.
When attached to a mailbox with a given identifier, the KCOV_IOC_ENABLE,
KCOV_IOC_DISABLE and mmap() operations will affect the mailbox.

As a demonstrator, the vHCI subsystem is changed to use KCOV mailboxes.
When the vHCI bus attaches it creates as many mailboxes as it has USB
ports, each mailbox being associated with a distinct port. Uhub is changed
to enable KCOV coverage in usbd_new_device(). With that in place, all of
the USB enumeration procedure can be traced with KCOV.

diffstat:

 sys/dev/usb/uhub.c     |   13 +-
 sys/dev/usb/usbdivar.h |   11 +-
 sys/dev/usb/vhci.c     |    7 +-
 sys/kern/subr_kcov.c   |  305 ++++++++++++++++++++++++++++++++++++++++--------
 sys/sys/kcov.h         |   18 ++-
 5 files changed, 294 insertions(+), 60 deletions(-)

diffs (truncated from 546 to 300 lines):

diff -r 2c59a4e779f0 -r 2d91237ba44d sys/dev/usb/uhub.c
--- a/sys/dev/usb/uhub.c        Fri May 15 12:31:03 2020 +0000
+++ b/sys/dev/usb/uhub.c        Fri May 15 12:34:52 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: uhub.c,v 1.144 2020/01/07 06:42:26 maxv Exp $  */
+/*     $NetBSD: uhub.c,v 1.145 2020/05/15 12:34:52 maxv Exp $  */
 /*     $FreeBSD: src/sys/dev/usb/uhub.c,v 1.18 1999/11/17 22:33:43 n_hibma Exp $       */
 /*     $OpenBSD: uhub.c,v 1.86 2015/06/29 18:27:40 mpi Exp $ */
 
@@ -37,7 +37,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uhub.c,v 1.144 2020/01/07 06:42:26 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uhub.c,v 1.145 2020/05/15 12:34:52 maxv Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_usb.h"
@@ -52,7 +52,7 @@
 #include <sys/proc.h>
 #include <sys/sysctl.h>
 #include <sys/systm.h>
-
+#include <sys/kcov.h>
 
 #include <dev/usb/usb.h>
 #include <dev/usb/usbdi.h>
@@ -755,9 +755,16 @@
                                    port);
                }
 
+               if (dev->ud_bus->ub_hctype == USBHCTYPE_VHCI)
+                       kcov_remote_enter(KCOV_REMOTE_VHCI, port);
+
                /* Get device info and set its address. */
                err = usbd_new_device(sc->sc_dev, dev->ud_bus,
                          dev->ud_depth + 1, speed, port, up);
+
+               if (dev->ud_bus->ub_hctype == USBHCTYPE_VHCI)
+                       kcov_remote_leave(KCOV_REMOTE_VHCI, port);
+
                /* XXX retry a few times? */
                if (err) {
                        DPRINTF("uhub%jd: usbd_new_device failed, error %jd",
diff -r 2c59a4e779f0 -r 2d91237ba44d sys/dev/usb/usbdivar.h
--- a/sys/dev/usb/usbdivar.h    Fri May 15 12:31:03 2020 +0000
+++ b/sys/dev/usb/usbdivar.h    Fri May 15 12:34:52 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: usbdivar.h,v 1.122 2020/02/12 16:01:00 riastradh Exp $ */
+/*     $NetBSD: usbdivar.h,v 1.123 2020/05/15 12:34:52 maxv Exp $      */
 
 /*
  * Copyright (c) 1998, 2012 The NetBSD Foundation, Inc.
@@ -157,7 +157,14 @@
 #define USBREV_3_0     5
 #define USBREV_3_1     6
 #define USBREV_STR { "unknown", "pre 1.0", "1.0", "1.1", "2.0", "3.0", "3.1" }
-
+       int                     ub_hctype;
+#define USBHCTYPE_UNKNOWN      0
+#define USBHCTYPE_MOTG         1
+#define USBHCTYPE_OHCI         2
+#define USBHCTYPE_UHCI         3
+#define USBHCTYPE_EHCI         4
+#define USBHCTYPE_XHCI         5
+#define USBHCTYPE_VHCI         6
        const struct usbd_bus_methods
                               *ub_methods;
        uint32_t                ub_pipesize;    /* size of a pipe struct */
diff -r 2c59a4e779f0 -r 2d91237ba44d sys/dev/usb/vhci.c
--- a/sys/dev/usb/vhci.c        Fri May 15 12:31:03 2020 +0000
+++ b/sys/dev/usb/vhci.c        Fri May 15 12:34:52 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: vhci.c,v 1.16 2020/03/31 16:34:25 maxv Exp $ */
+/*     $NetBSD: vhci.c,v 1.17 2020/05/15 12:34:52 maxv Exp $ */
 
 /*
  * Copyright (c) 2019-2020 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vhci.c,v 1.16 2020/03/31 16:34:25 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vhci.c,v 1.17 2020/05/15 12:34:52 maxv Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_usb.h"
@@ -51,6 +51,7 @@
 #include <sys/mman.h>
 #include <sys/file.h>
 #include <sys/filedesc.h>
+#include <sys/kcov.h>
 
 #include <machine/endian.h>
 
@@ -1282,6 +1283,7 @@
 
        sc->sc_dev = self;
        sc->sc_bus.ub_revision = USBREV_2_0;
+       sc->sc_bus.ub_hctype = USBHCTYPE_VHCI;
        sc->sc_bus.ub_usedma = false;
        sc->sc_bus.ub_methods = &vhci_bus_methods;
        sc->sc_bus.ub_pipesize = sizeof(vhci_pipe_t);
@@ -1297,6 +1299,7 @@
                        TAILQ_INIT(&port->endpoints[addr].usb_to_host);
                        TAILQ_INIT(&port->endpoints[addr].host_to_usb);
                }
+               kcov_remote_register(KCOV_REMOTE_VHCI, i);
        }
 
        sc->sc_child = config_found(self, &sc->sc_bus, usbctlprint);
diff -r 2c59a4e779f0 -r 2d91237ba44d sys/kern/subr_kcov.c
--- a/sys/kern/subr_kcov.c      Fri May 15 12:31:03 2020 +0000
+++ b/sys/kern/subr_kcov.c      Fri May 15 12:34:52 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: subr_kcov.c,v 1.12 2020/04/04 06:51:46 maxv Exp $      */
+/*     $NetBSD: subr_kcov.c,v 1.13 2020/05/15 12:34:52 maxv Exp $      */
 
 /*
  * Copyright (c) 2019-2020 The NetBSD Foundation, Inc.
@@ -105,16 +105,24 @@
  */
 
 typedef struct kcov_desc {
+       /* Local only */
        kmutex_t lock;
+       bool lwpfree;
+
+       /* Pointer to the end of the structure, if any */
+       struct kcov_desc *remote;
+
+       /* Can be remote */
        kcov_int_t *buf;
        struct uvm_object *uobj;
        size_t bufnent;
        size_t bufsize;
        int mode;
        bool enabled;
-       bool lwpfree;
 } kcov_t;
 
+/* -------------------------------------------------------------------------- */
+
 static void
 kcov_lock(kcov_t *kd)
 {
@@ -129,6 +137,21 @@
        mutex_exit(&kd->lock);
 }
 
+static bool
+kcov_mode_is_valid(int mode)
+{
+       switch (mode) {
+       case KCOV_MODE_NONE:
+       case KCOV_MODE_TRACE_PC:
+       case KCOV_MODE_TRACE_CMP:
+               return true;
+       default:
+               return false;
+       }
+}
+
+/* -------------------------------------------------------------------------- */
+
 static void
 kcov_free(kcov_t *kd)
 {
@@ -194,6 +217,212 @@
 
 /* -------------------------------------------------------------------------- */
 
+typedef struct kcov_remote {
+       LIST_ENTRY(kcov_remote) list;
+       uint64_t subsystem;
+       uint64_t id;
+       u_int refcount;
+       kcov_t kd;
+} kcov_remote_t;
+
+typedef LIST_HEAD(, kcov_remote) kcov_remote_list_t;
+
+static kcov_remote_list_t kcov_remote_list;
+
+static kcov_remote_t *
+kcov_remote_find(uint64_t subsystem, uint64_t id)
+{
+       kcov_remote_t *kr;
+
+       LIST_FOREACH(kr, &kcov_remote_list, list) {
+               if (kr->subsystem == subsystem && kr->id == id)
+                       return kr;
+       }
+
+       return NULL;
+}
+
+void
+kcov_remote_register(uint64_t subsystem, uint64_t id)
+{
+       kcov_remote_t *kr;
+       kcov_t *kd;
+       int error;
+
+       if (kcov_remote_find(subsystem, id) != NULL) {
+               panic("%s: kr already exists", __func__);
+       }
+
+       kr = kmem_zalloc(sizeof(*kr), KM_SLEEP);
+       kr->subsystem = subsystem;
+       kr->id = id;
+       kr->refcount = 0;
+       kd = &kr->kd;
+
+       mutex_init(&kd->lock, MUTEX_DEFAULT, IPL_NONE);
+       error = kcov_allocbuf(kd, KCOV_BUF_MAX_ENTRIES);
+       if (error != 0)
+               panic("%s: failed to allocate buffer", __func__);
+
+       LIST_INSERT_HEAD(&kcov_remote_list, kr, list);
+}
+
+void
+kcov_remote_enter(uint64_t subsystem, uint64_t id)
+{
+       struct lwp *l = curlwp;
+       kcov_remote_t *kr;
+       kcov_t *kd;
+       u_int refs;
+
+       kr = kcov_remote_find(subsystem, id);
+       if (__predict_false(kr == NULL)) {
+               panic("%s: unable to find kr", __func__);
+       }
+
+       refs = atomic_inc_uint_nv(&kr->refcount);
+       KASSERT(refs == 1);
+
+       KASSERT(l->l_kcov == NULL);
+       kd = &kr->kd;
+       if (atomic_load_relaxed(&kd->enabled)) {
+               l->l_kcov = kd;
+       }
+}
+
+void
+kcov_remote_leave(uint64_t subsystem, uint64_t id)
+{
+       struct lwp *l = curlwp;
+       kcov_remote_t *kr;
+       u_int refs;
+
+       kr = kcov_remote_find(subsystem, id);
+       if (__predict_false(kr == NULL)) {
+               panic("%s: unable to find kr", __func__);
+       }
+
+       refs = atomic_dec_uint_nv(&kr->refcount);
+       KASSERT(refs == 0);
+
+       l->l_kcov = NULL;
+}
+
+static int
+kcov_remote_enable(kcov_t *kd, int mode)
+{
+       kcov_lock(kd);
+       if (kd->enabled) {
+               kcov_unlock(kd);
+               return EBUSY;
+       }
+       kd->mode = mode;
+       atomic_store_relaxed(&kd->enabled, true);
+       kcov_unlock(kd);
+
+       return 0;
+}
+
+static int
+kcov_remote_disable(kcov_t *kd)
+{
+       kcov_lock(kd);
+       if (!kd->enabled) {
+               kcov_unlock(kd);
+               return ENOENT;
+       }
+       atomic_store_relaxed(&kd->enabled, false);
+       kcov_unlock(kd);
+
+       return 0;
+}
+
+static int
+kcov_remote_attach(kcov_t *kd, struct kcov_ioc_remote_attach *args)
+{
+       kcov_remote_t *kr;
+
+       if (kd->enabled)
+               return EEXIST;
+