[src/trunk]: src/sys/dev Defer crypto operations to a workqueue and make it u...

[mlelstv <mlelstv%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/a03984973721
branches:  trunk
changeset: 970031:a03984973721
user:      mlelstv <mlelstv%NetBSD.org@localhost>
date:      Mon Mar 09 08:33:15 2020 +0000

description:
Defer crypto operations to a workqueue and make it utilize all CPUs.
Make device mpsafe.
Some code cleanup.

diffstat:

 sys/dev/cgd.c    |  711 ++++++++++++++++++++++++++++++++++++++++--------------
 sys/dev/cgdvar.h |   32 ++-
 2 files changed, 549 insertions(+), 194 deletions(-)

diffs (truncated from 1313 to 300 lines):

diff -r ee4d8e7ea737 -r a03984973721 sys/dev/cgd.c
--- a/sys/dev/cgd.c     Mon Mar 09 08:29:11 2020 +0000
+++ b/sys/dev/cgd.c     Mon Mar 09 08:33:15 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cgd.c,v 1.121 2020/03/02 16:01:56 riastradh Exp $ */
+/* $NetBSD: cgd.c,v 1.122 2020/03/09 08:33:15 mlelstv Exp $ */
 
 /*-
  * Copyright (c) 2002 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cgd.c,v 1.121 2020/03/02 16:01:56 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cgd.c,v 1.122 2020/03/09 08:33:15 mlelstv Exp $");
 
 #include <sys/types.h>
 #include <sys/param.h>
@@ -39,6 +39,7 @@
 #include <sys/errno.h>
 #include <sys/buf.h>
 #include <sys/bufq.h>
+#include <sys/kmem.h>
 #include <sys/malloc.h>
 #include <sys/module.h>
 #include <sys/pool.h>
@@ -51,6 +52,8 @@
 #include <sys/vnode.h>
 #include <sys/conf.h>
 #include <sys/syslog.h>
+#include <sys/workqueue.h>
+#include <sys/cpu.h>
 
 #include <dev/dkvar.h>
 #include <dev/cgdvar.h>
@@ -90,7 +93,7 @@
        .d_dump = cgddump,
        .d_psize = cgdsize,
        .d_discard = nodiscard,
-       .d_flag = D_DISK
+       .d_flag = D_DISK | D_MPSAFE
 };
 
 const struct cdevsw cgd_cdevsw = {
@@ -105,7 +108,7 @@
        .d_mmap = nommap,
        .d_kqfilter = nokqfilter,
        .d_discard = nodiscard,
-       .d_flag = D_DISK
+       .d_flag = D_DISK | D_MPSAFE
 };
 
 /*
@@ -207,12 +210,20 @@
 static void cgd_attach(device_t, device_t, void *);
 static int cgd_detach(device_t, int);
 static struct cgd_softc        *cgd_spawn(int);
+static struct cgd_worker *cgd_create_one_worker(void);
+static void cgd_destroy_one_worker(struct cgd_worker *);
+static struct cgd_worker *cgd_create_worker(void);
+static void cgd_destroy_worker(struct cgd_worker *);
 static int cgd_destroy(device_t);
 
 /* Internal Functions */
 
 static int     cgd_diskstart(device_t, struct buf *);
+static void    cgd_diskstart2(struct cgd_softc *, struct cgd_xfer *);
 static void    cgdiodone(struct buf *);
+static void    cgd_iodone2(struct cgd_softc *, struct cgd_xfer *);
+static void    cgd_enqueue(struct cgd_softc *, struct cgd_xfer *);
+static void    cgd_process(struct work *, void *);
 static int     cgd_dumpblocks(device_t, void *, daddr_t, int);
 
 static int     cgd_ioctl_set(struct cgd_softc *, void *, struct lwp *);
@@ -264,25 +275,49 @@
 
 /* Global variables */
 
+static kmutex_t cgd_spawning_mtx;
+static kcondvar_t cgd_spawning_cv;
+static bool cgd_spawning;
+static struct cgd_worker *cgd_worker;
+static u_int cgd_refcnt;       /* number of users of cgd_worker */
+
 /* Utility Functions */
 
 #define CGDUNIT(x)             DISKUNIT(x)
-#define GETCGD_SOFTC(_cs, x)   if (!((_cs) = getcgd_softc(x))) return ENXIO
 
 /* The code */
 
+static int
+cgd_lock(bool intr)
+{
+       int error = 0;
+
+       mutex_enter(&cgd_spawning_mtx);
+       while (cgd_spawning) {
+               if (intr)
+                       error = cv_wait_sig(&cgd_spawning_cv, &cgd_spawning_mtx);
+               else
+                       cv_wait(&cgd_spawning_cv, &cgd_spawning_mtx);
+       }
+       if (error == 0)
+               cgd_spawning = true;
+       mutex_exit(&cgd_spawning_mtx);
+       return error;
+}
+
+static void
+cgd_unlock(void)
+{
+       mutex_enter(&cgd_spawning_mtx);
+       cgd_spawning = false;
+       cv_broadcast(&cgd_spawning_cv);
+       mutex_exit(&cgd_spawning_mtx);
+}
+
 static struct cgd_softc *
 getcgd_softc(dev_t dev)
 {
-       int     unit = CGDUNIT(dev);
-       struct cgd_softc *sc;
-
-       DPRINTF_FOLLOW(("getcgd_softc(0x%"PRIx64"): unit = %d\n", dev, unit));
-
-       sc = device_lookup_private(&cgd_cd, unit);
-       if (sc == NULL)
-               sc = cgd_spawn(unit);
-       return sc;
+       return device_lookup_private(&cgd_cd, CGDUNIT(dev));
 }
 
 static int
@@ -298,6 +333,7 @@
        struct cgd_softc *sc = device_private(self);
 
        mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_BIO);
+       cv_init(&sc->sc_cv, "cgdcv");
        dk_init(&sc->sc_dksc, self, DKTYPE_CGD);
        disk_init(&sc->sc_dksc.sc_dkdev, sc->sc_dksc.sc_xname, &cgddkdriver);
 
@@ -323,6 +359,7 @@
                return ret;
 
        disk_destroy(&dksc->sc_dkdev);
+       cv_destroy(&sc->sc_cv);
        mutex_destroy(&sc->sc_lock);
 
        return 0;
@@ -331,89 +368,224 @@
 void
 cgdattach(int num)
 {
+#ifndef _MODULE
        int error;
 
+       mutex_init(&cgd_spawning_mtx, MUTEX_DEFAULT, IPL_NONE);
+       cv_init(&cgd_spawning_cv, "cgspwn");
+
        error = config_cfattach_attach(cgd_cd.cd_name, &cgd_ca);
        if (error != 0)
                aprint_error("%s: unable to register cfattach\n",
                    cgd_cd.cd_name);
+#endif
 }
 
 static struct cgd_softc *
 cgd_spawn(int unit)
 {
        cfdata_t cf;
+       struct cgd_worker *cw;
+       struct cgd_softc *sc;
 
-       cf = malloc(sizeof(*cf), M_DEVBUF, M_WAITOK);
+       cf = kmem_alloc(sizeof(*cf), KM_SLEEP);
        cf->cf_name = cgd_cd.cd_name;
        cf->cf_atname = cgd_cd.cd_name;
        cf->cf_unit = unit;
        cf->cf_fstate = FSTATE_STAR;
 
-       return device_private(config_attach_pseudo(cf));
+       cw = cgd_create_one_worker();
+       if (cw == NULL) {
+               kmem_free(cf, sizeof(*cf));
+               return NULL;
+       }
+
+       sc = device_private(config_attach_pseudo(cf));
+       if (sc == NULL) {
+               cgd_destroy_one_worker(cw);
+               return NULL;
+       }
+
+       sc->sc_worker = cw;
+
+       return sc;
 }
 
 static int
 cgd_destroy(device_t dev)
 {
+       struct cgd_softc *sc = device_private(dev);
+       struct cgd_worker *cw = sc->sc_worker;
+       cfdata_t cf;
        int error;
-       cfdata_t cf;
 
        cf = device_cfdata(dev);
        error = config_detach(dev, DETACH_QUIET);
        if (error)
                return error;
-       free(cf, M_DEVBUF);
+
+       cgd_destroy_one_worker(cw);
+
+       kmem_free(cf, sizeof(*cf));
        return 0;
 }
 
+static void
+cgd_busy(struct cgd_softc *sc)
+{
+
+       mutex_enter(&sc->sc_lock);
+       while (sc->sc_busy)
+               cv_wait(&sc->sc_cv, &sc->sc_lock);
+       sc->sc_busy = true;
+       mutex_exit(&sc->sc_lock);
+}
+
+static void
+cgd_unbusy(struct cgd_softc *sc)
+{
+
+       mutex_enter(&sc->sc_lock);
+       sc->sc_busy = false;
+       cv_broadcast(&sc->sc_cv);
+       mutex_exit(&sc->sc_lock);
+}
+
+static struct cgd_worker *
+cgd_create_one_worker(void)
+{
+       KASSERT(cgd_spawning);
+
+       if (cgd_refcnt++ == 0) {
+               KASSERT(cgd_worker == NULL);
+               cgd_worker = cgd_create_worker();
+       }
+
+       KASSERT(cgd_worker != NULL);
+       return cgd_worker;
+}
+
+static void
+cgd_destroy_one_worker(struct cgd_worker *cw)
+{
+       KASSERT(cgd_spawning);
+       KASSERT(cw == cgd_worker);
+
+       if (--cgd_refcnt == 0) {
+               cgd_destroy_worker(cgd_worker);
+               cgd_worker = NULL;
+       }
+}
+
+static struct cgd_worker *
+cgd_create_worker(void)
+{
+       struct cgd_worker *cw;
+       struct workqueue *wq;
+       struct pool *cp;
+       int error;
+
+       cw = kmem_alloc(sizeof(struct cgd_worker), KM_SLEEP);
+       cp = kmem_alloc(sizeof(struct pool), KM_SLEEP);
+
+       error = workqueue_create(&wq, "cgd", cgd_process, NULL,
+                                PRI_BIO, IPL_BIO, WQ_MPSAFE | WQ_PERCPU);
+       if (error) {
+               kmem_free(cp, sizeof(struct pool));
+               kmem_free(cw, sizeof(struct cgd_worker));
+               return NULL;
+       }
+
+       cw->cw_cpool = cp;
+       cw->cw_wq = wq;
+       pool_init(cw->cw_cpool, sizeof(struct cgd_xfer), 0,
+           0, 0, "cgdcpl", NULL, IPL_BIO);
+
+       mutex_init(&cw->cw_lock, MUTEX_DEFAULT, IPL_BIO);
+       
+       return cw;
+}
+
+static void
+cgd_destroy_worker(struct cgd_worker *cw)
+{
+       mutex_destroy(&cw->cw_lock);