[src/netbsd-9]: src/sys/dev Pull up following revision(s) (requested by riast...

[martin <martin%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/36ad91e383c3
branches:  netbsd-9
changeset: 963767:36ad91e383c3
user:      martin <martin%NetBSD.org@localhost>
date:      Mon Apr 06 14:57:42 2020 +0000

description:
Pull up following revision(s) (requested by riastradh in ticket #823):

        sys/dev/cgdvar.h: revision 1.19
        sys/dev/cgd.c: revision 1.122
        sys/dev/cgd.c: revision 1.123
        sys/dev/cgd.c: revision 1.124

Defer crypto operations to a workqueue and make it utilize all CPUs.

Make device mpsafe.

Some code cleanup.

Don't wait for data buffer.

cgd: switch from malloc(9) to kmem(9)
XXX might be worthwhile to use pool_cache(9) in the write path

diffstat:

 sys/dev/cgd.c    |  716 ++++++++++++++++++++++++++++++++++++++++--------------
 sys/dev/cgdvar.h |   32 ++-
 2 files changed, 552 insertions(+), 196 deletions(-)

diffs (truncated from 1306 to 300 lines):

diff -r 3c42352794d5 -r 36ad91e383c3 sys/dev/cgd.c
--- a/sys/dev/cgd.c     Mon Apr 06 14:53:33 2020 +0000
+++ b/sys/dev/cgd.c     Mon Apr 06 14:57:42 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cgd.c,v 1.116.10.2 2020/03/21 15:52:09 martin Exp $ */
+/* $NetBSD: cgd.c,v 1.116.10.3 2020/04/06 14:57:42 martin Exp $ */
 
 /*-
  * Copyright (c) 2002 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cgd.c,v 1.116.10.2 2020/03/21 15:52:09 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cgd.c,v 1.116.10.3 2020/04/06 14:57:42 martin Exp $");
 
 #include <sys/types.h>
 #include <sys/param.h>
@@ -39,7 +39,7 @@
 #include <sys/errno.h>
 #include <sys/buf.h>
 #include <sys/bufq.h>
-#include <sys/malloc.h>
+#include <sys/kmem.h>
 #include <sys/module.h>
 #include <sys/pool.h>
 #include <sys/ioctl.h>
@@ -51,6 +51,8 @@
 #include <sys/vnode.h>
 #include <sys/conf.h>
 #include <sys/syslog.h>
+#include <sys/workqueue.h>
+#include <sys/cpu.h>
 
 #include <dev/dkvar.h>
 #include <dev/cgdvar.h>
@@ -90,7 +92,7 @@
        .d_dump = cgddump,
        .d_psize = cgdsize,
        .d_discard = nodiscard,
-       .d_flag = D_DISK
+       .d_flag = D_DISK | D_MPSAFE
 };
 
 const struct cdevsw cgd_cdevsw = {
@@ -105,7 +107,7 @@
        .d_mmap = nommap,
        .d_kqfilter = nokqfilter,
        .d_discard = nodiscard,
-       .d_flag = D_DISK
+       .d_flag = D_DISK | D_MPSAFE
 };
 
 /*
@@ -207,12 +209,20 @@
 static void cgd_attach(device_t, device_t, void *);
 static int cgd_detach(device_t, int);
 static struct cgd_softc        *cgd_spawn(int);
+static struct cgd_worker *cgd_create_one_worker(void);
+static void cgd_destroy_one_worker(struct cgd_worker *);
+static struct cgd_worker *cgd_create_worker(void);
+static void cgd_destroy_worker(struct cgd_worker *);
 static int cgd_destroy(device_t);
 
 /* Internal Functions */
 
 static int     cgd_diskstart(device_t, struct buf *);
+static void    cgd_diskstart2(struct cgd_softc *, struct cgd_xfer *);
 static void    cgdiodone(struct buf *);
+static void    cgd_iodone2(struct cgd_softc *, struct cgd_xfer *);
+static void    cgd_enqueue(struct cgd_softc *, struct cgd_xfer *);
+static void    cgd_process(struct work *, void *);
 static int     cgd_dumpblocks(device_t, void *, daddr_t, int);
 
 static int     cgd_ioctl_set(struct cgd_softc *, void *, struct lwp *);
@@ -272,25 +282,49 @@
 
 /* Global variables */
 
+static kmutex_t cgd_spawning_mtx;
+static kcondvar_t cgd_spawning_cv;
+static bool cgd_spawning;
+static struct cgd_worker *cgd_worker;
+static u_int cgd_refcnt;       /* number of users of cgd_worker */
+
 /* Utility Functions */
 
 #define CGDUNIT(x)             DISKUNIT(x)
-#define GETCGD_SOFTC(_cs, x)   if (!((_cs) = getcgd_softc(x))) return ENXIO
 
 /* The code */
 
+static int
+cgd_lock(bool intr)
+{
+       int error = 0;
+
+       mutex_enter(&cgd_spawning_mtx);
+       while (cgd_spawning) {
+               if (intr)
+                       error = cv_wait_sig(&cgd_spawning_cv, &cgd_spawning_mtx);
+               else
+                       cv_wait(&cgd_spawning_cv, &cgd_spawning_mtx);
+       }
+       if (error == 0)
+               cgd_spawning = true;
+       mutex_exit(&cgd_spawning_mtx);
+       return error;
+}
+
+static void
+cgd_unlock(void)
+{
+       mutex_enter(&cgd_spawning_mtx);
+       cgd_spawning = false;
+       cv_broadcast(&cgd_spawning_cv);
+       mutex_exit(&cgd_spawning_mtx);
+}
+
 static struct cgd_softc *
 getcgd_softc(dev_t dev)
 {
-       int     unit = CGDUNIT(dev);
-       struct cgd_softc *sc;
-
-       DPRINTF_FOLLOW(("getcgd_softc(0x%"PRIx64"): unit = %d\n", dev, unit));
-
-       sc = device_lookup_private(&cgd_cd, unit);
-       if (sc == NULL)
-               sc = cgd_spawn(unit);
-       return sc;
+       return device_lookup_private(&cgd_cd, CGDUNIT(dev));
 }
 
 static int
@@ -306,6 +340,7 @@
        struct cgd_softc *sc = device_private(self);
 
        mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_BIO);
+       cv_init(&sc->sc_cv, "cgdcv");
        dk_init(&sc->sc_dksc, self, DKTYPE_CGD);
        disk_init(&sc->sc_dksc.sc_dkdev, sc->sc_dksc.sc_xname, &cgddkdriver);
 
@@ -331,6 +366,7 @@
                return ret;
 
        disk_destroy(&dksc->sc_dkdev);
+       cv_destroy(&sc->sc_cv);
        mutex_destroy(&sc->sc_lock);
 
        return 0;
@@ -339,89 +375,224 @@
 void
 cgdattach(int num)
 {
+#ifndef _MODULE
        int error;
 
+       mutex_init(&cgd_spawning_mtx, MUTEX_DEFAULT, IPL_NONE);
+       cv_init(&cgd_spawning_cv, "cgspwn");
+
        error = config_cfattach_attach(cgd_cd.cd_name, &cgd_ca);
        if (error != 0)
                aprint_error("%s: unable to register cfattach\n",
                    cgd_cd.cd_name);
+#endif
 }
 
 static struct cgd_softc *
 cgd_spawn(int unit)
 {
        cfdata_t cf;
+       struct cgd_worker *cw;
+       struct cgd_softc *sc;
 
-       cf = malloc(sizeof(*cf), M_DEVBUF, M_WAITOK);
+       cf = kmem_alloc(sizeof(*cf), KM_SLEEP);
        cf->cf_name = cgd_cd.cd_name;
        cf->cf_atname = cgd_cd.cd_name;
        cf->cf_unit = unit;
        cf->cf_fstate = FSTATE_STAR;
 
-       return device_private(config_attach_pseudo(cf));
+       cw = cgd_create_one_worker();
+       if (cw == NULL) {
+               kmem_free(cf, sizeof(*cf));
+               return NULL;
+       }
+
+       sc = device_private(config_attach_pseudo(cf));
+       if (sc == NULL) {
+               cgd_destroy_one_worker(cw);
+               return NULL;
+       }
+
+       sc->sc_worker = cw;
+
+       return sc;
 }
 
 static int
 cgd_destroy(device_t dev)
 {
+       struct cgd_softc *sc = device_private(dev);
+       struct cgd_worker *cw = sc->sc_worker;
+       cfdata_t cf;
        int error;
-       cfdata_t cf;
 
        cf = device_cfdata(dev);
        error = config_detach(dev, DETACH_QUIET);
        if (error)
                return error;
-       free(cf, M_DEVBUF);
+
+       cgd_destroy_one_worker(cw);
+
+       kmem_free(cf, sizeof(*cf));
        return 0;
 }
 
+static void
+cgd_busy(struct cgd_softc *sc)
+{
+
+       mutex_enter(&sc->sc_lock);
+       while (sc->sc_busy)
+               cv_wait(&sc->sc_cv, &sc->sc_lock);
+       sc->sc_busy = true;
+       mutex_exit(&sc->sc_lock);
+}
+
+static void
+cgd_unbusy(struct cgd_softc *sc)
+{
+
+       mutex_enter(&sc->sc_lock);
+       sc->sc_busy = false;
+       cv_broadcast(&sc->sc_cv);
+       mutex_exit(&sc->sc_lock);
+}
+
+static struct cgd_worker *
+cgd_create_one_worker(void)
+{
+       KASSERT(cgd_spawning);
+
+       if (cgd_refcnt++ == 0) {
+               KASSERT(cgd_worker == NULL);
+               cgd_worker = cgd_create_worker();
+       }
+
+       KASSERT(cgd_worker != NULL);
+       return cgd_worker;
+}
+
+static void
+cgd_destroy_one_worker(struct cgd_worker *cw)
+{
+       KASSERT(cgd_spawning);
+       KASSERT(cw == cgd_worker);
+
+       if (--cgd_refcnt == 0) {
+               cgd_destroy_worker(cgd_worker);
+               cgd_worker = NULL;
+       }
+}
+
+static struct cgd_worker *
+cgd_create_worker(void)
+{
+       struct cgd_worker *cw;
+       struct workqueue *wq;
+       struct pool *cp;
+       int error;
+
+       cw = kmem_alloc(sizeof(struct cgd_worker), KM_SLEEP);
+       cp = kmem_alloc(sizeof(struct pool), KM_SLEEP);
+
+       error = workqueue_create(&wq, "cgd", cgd_process, NULL,
+                                PRI_BIO, IPL_BIO, WQ_MPSAFE | WQ_PERCPU);
+       if (error) {
+               kmem_free(cp, sizeof(struct pool));
+               kmem_free(cw, sizeof(struct cgd_worker));
+               return NULL;
+       }
+
+       cw->cw_cpool = cp;
+       cw->cw_wq = wq;
+       pool_init(cw->cw_cpool, sizeof(struct cgd_xfer), 0,
+           0, 0, "cgdcpl", NULL, IPL_BIO);
+
+       mutex_init(&cw->cw_lock, MUTEX_DEFAULT, IPL_BIO);
+       
+       return cw;
+}
+
+static void
+cgd_destroy_worker(struct cgd_worker *cw)
+{