[src/netbsd-8]: src/sys/opencrypto Pull up following revision(s) (requested b...

[snj <snj%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/740db8ecd691
branches:  netbsd-8
changeset: 850903:740db8ecd691
user:      snj <snj%NetBSD.org@localhost>
date:      Sat Aug 05 03:59:21 2017 +0000

description:
Pull up following revision(s) (requested by knakahara in ticket #178):
        sys/opencrypto/crypto.c: 1.92-1.100
        sys/opencrypto/cryptodev.h: 1.38-1.39
        sys/opencrypto/ocryptodev.c: 1.9-1.11
        sys/opencrypto/xform.c: revision 1.29
        sys/opencrypto/xform.h: revision 1.20
KNF
--
Apply C99-style struct initialization to enc_xform, auth_hash and comp_algo
--
make cryptoret() context softint to balance dequeuing crypto_ret_q with enqueuing it.
--
fix panic when using ecryption devices attached earlier than ipi_sysinit().
pointed out and tested by martin@n.o, thanks.
--
fix typo
--
make crp_{,k}q percpu to scale crypto_dispatch().
update locking note later.
--
divide crp_ret_{,k}q by CPU to avoid reordering.
update locking note later.
--
update locking notes of opencrypto(9)
--
Don't disclose uninitialized 32-bit word if cryptodev_session fails.
>From Ilja Van Sprundel.
--
Avert userland-controlled integer overflow.
>From Ilja Van Sprundel.
--
Avoid another userland-controlled integer overflow.
>From Ilja Van Sprundel.
--
refactor: remove glue macros for FreeBSD code.
--
pack crypto_drivers variables to struct and add cacheline_aligned qualifier.
--
use kmem_alloc KPI instead of malloc KPI.
--
use pool_cache(9) instead of pool(9) as they can be called concurrently.

diffstat:

 sys/opencrypto/crypto.c     |  714 +++++++++++++++++++++++++++++--------------
 sys/opencrypto/cryptodev.h  |   32 +-
 sys/opencrypto/ocryptodev.c |   18 +-
 sys/opencrypto/xform.c      |  259 +++++++++++----
 sys/opencrypto/xform.h      |    8 +-
 5 files changed, 713 insertions(+), 318 deletions(-)

diffs (truncated from 1747 to 300 lines):

diff -r c0e466c0f81b -r 740db8ecd691 sys/opencrypto/crypto.c
--- a/sys/opencrypto/crypto.c   Sat Aug 05 03:52:57 2017 +0000
+++ b/sys/opencrypto/crypto.c   Sat Aug 05 03:59:21 2017 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: crypto.c,v 1.78.2.2 2017/07/05 20:19:21 snj Exp $ */
+/*     $NetBSD: crypto.c,v 1.78.2.3 2017/08/05 03:59:21 snj Exp $ */
 /*     $FreeBSD: src/sys/opencrypto/crypto.c,v 1.4.2.5 2003/02/26 00:14:05 sam Exp $   */
 /*     $OpenBSD: crypto.c,v 1.41 2002/07/17 23:52:38 art Exp $ */
 
@@ -53,12 +53,11 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: crypto.c,v 1.78.2.2 2017/07/05 20:19:21 snj Exp $");
+__KERNEL_RCSID(0, "$NetBSD: crypto.c,v 1.78.2.3 2017/08/05 03:59:21 snj Exp $");
 
 #include <sys/param.h>
 #include <sys/reboot.h>
 #include <sys/systm.h>
-#include <sys/malloc.h>
 #include <sys/proc.h>
 #include <sys/pool.h>
 #include <sys/kthread.h>
@@ -67,6 +66,11 @@
 #include <sys/intr.h>
 #include <sys/errno.h>
 #include <sys/module.h>
+#include <sys/xcall.h>
+#include <sys/device.h>
+#include <sys/cpu.h>
+#include <sys/percpu.h>
+#include <sys/kmem.h>
 
 #if defined(_KERNEL_OPT)
 #include "opt_ocf.h"
@@ -75,45 +79,75 @@
 #include <opencrypto/cryptodev.h>
 #include <opencrypto/xform.h>                  /* XXX for M_XDATA */
 
-static kmutex_t crypto_q_mtx;
-static kmutex_t crypto_ret_q_mtx;
-static kcondvar_t cryptoret_cv;
-
-/* below are kludges for residual code wrtitten to FreeBSD interfaces */
-  #define SWI_CRYPTO 17
-  #define register_swi(lvl, fn)  \
-  softint_establish(SOFTINT_NET|SOFTINT_MPSAFE, (void (*)(void *))fn, NULL)
-  #define unregister_swi(lvl, fn)  softint_disestablish(softintr_cookie)
-  #define setsoftcrypto(x)                     \
-       do{                                     \
-               kpreempt_disable();             \
-               softint_schedule(x);            \
-               kpreempt_enable();              \
-       }while(0)
-
-int crypto_ret_q_check(struct cryptop *);
-
 /*
  * Crypto drivers register themselves by allocating a slot in the
  * crypto_drivers table with crypto_get_driverid() and then registering
  * each algorithm they support with crypto_register() and crypto_kregister().
  */
-static kmutex_t crypto_drv_mtx;
 /* Don't directly access crypto_drivers[i], use crypto_checkdriver(i). */
-static struct cryptocap *crypto_drivers;
-static int crypto_drivers_num;
-static void *softintr_cookie;
-static int crypto_exit_flag;
+static struct {
+       kmutex_t mtx;
+       int num;
+       struct cryptocap *list;
+} crypto_drv __cacheline_aligned;
+#define crypto_drv_mtx         (crypto_drv.mtx)
+#define crypto_drivers_num     (crypto_drv.num)
+#define crypto_drivers         (crypto_drv.list)
+
+static void *crypto_q_si;
+static void *crypto_ret_si;
 
 /*
  * There are two queues for crypto requests; one for symmetric (e.g.
  * cipher) operations and one for asymmetric (e.g. MOD) operations.
  * See below for how synchronization is handled.
  */
-static TAILQ_HEAD(,cryptop) crp_q =            /* request queues */
-               TAILQ_HEAD_INITIALIZER(crp_q);
-static TAILQ_HEAD(,cryptkop) crp_kq =
-               TAILQ_HEAD_INITIALIZER(crp_kq);
+TAILQ_HEAD(crypto_crp_q, cryptop);
+TAILQ_HEAD(crypto_crp_kq, cryptkop);
+struct crypto_crp_qs {
+       struct crypto_crp_q crp_q;
+       struct crypto_crp_kq crp_kq;
+};
+static percpu_t *crypto_crp_qs_percpu;
+
+static inline struct crypto_crp_qs *
+crypto_get_crp_qs(int *s)
+{
+
+       KASSERT(s != NULL);
+
+       *s = splsoftnet();
+       return percpu_getref(crypto_crp_qs_percpu);
+}
+
+static inline void
+crypto_put_crp_qs(int *s)
+{
+
+       KASSERT(s != NULL);
+
+       percpu_putref(crypto_crp_qs_percpu);
+       splx(*s);
+}
+
+static void
+crypto_crp_q_is_busy_pc(void *p, void *arg, struct cpu_info *ci __unused)
+{
+       struct crypto_crp_qs *qs_pc = p;
+       bool *isempty = arg;
+
+       if (!TAILQ_EMPTY(&qs_pc->crp_q) || !TAILQ_EMPTY(&qs_pc->crp_kq))
+               *isempty = true;
+}
+
+static void
+crypto_crp_qs_init_pc(void *p, void *arg __unused, struct cpu_info *ci __unused)
+{
+       struct crypto_crp_qs *qs = p;
+
+       TAILQ_INIT(&qs->crp_q);
+       TAILQ_INIT(&qs->crp_kq);
+}
 
 /*
  * There are two queues for processing completed crypto requests; one
@@ -121,50 +155,51 @@
  * but have two to avoid type futzing (cryptop vs. cryptkop).  See below
  * for how synchronization is handled.
  */
-static TAILQ_HEAD(crprethead, cryptop) crp_ret_q =     /* callback queues */
-               TAILQ_HEAD_INITIALIZER(crp_ret_q);
-static TAILQ_HEAD(krprethead, cryptkop) crp_ret_kq =
-               TAILQ_HEAD_INITIALIZER(crp_ret_kq);
-
-#define DEFINIT_CRYPTO_Q_LEN(name)             \
-       static int crypto_##name##_len = 0
+TAILQ_HEAD(crypto_crp_ret_q, cryptop);
+TAILQ_HEAD(crypto_crp_ret_kq, cryptkop);
+struct crypto_crp_ret_qs {
+       kmutex_t crp_ret_q_mtx;
+       bool crp_ret_q_exit_flag;
 
-#define DEFINIT_CRYPTO_Q_DROPS(name)           \
-       static int crypto_##name##_drops = 0
+       struct crypto_crp_ret_q crp_ret_q;
+       int crp_ret_q_len;
+       int crp_ret_q_maxlen; /* queue length limit. <=0 means unlimited. */
+       int crp_ret_q_drops;
 
-#define DEFINIT_CRYPTO_Q_MAXLEN(name, defval)          \
-       static int crypto_##name##_maxlen = defval
+       struct crypto_crp_ret_kq crp_ret_kq;
+       int crp_ret_kq_len;
+       int crp_ret_kq_maxlen; /* queue length limit. <=0 means unlimited. */
+       int crp_ret_kq_drops;
+};
+struct crypto_crp_ret_qs **crypto_crp_ret_qs_list;
 
-#define CRYPTO_Q_INC(name)                     \
-       do {                                    \
-               crypto_##name##_len++;          \
-       } while(0);
 
-#define CRYPTO_Q_DEC(name)                     \
-       do {                                    \
-               crypto_##name##_len--;          \
-       } while(0);
+static inline struct crypto_crp_ret_qs *
+crypto_get_crp_ret_qs(struct cpu_info *ci)
+{
+       u_int cpuid;
+       struct crypto_crp_ret_qs *qs;
 
-#define CRYPTO_Q_INC_DROPS(name)               \
-       do {                                    \
-               crypto_##name##_drops++;        \
-       } while(0);
+       KASSERT(ci != NULL);
 
-#define CRYPTO_Q_IS_FULL(name)                                 \
-       (crypto_##name##_maxlen > 0                             \
-           && (crypto_##name##_len > crypto_##name##_maxlen))
+       cpuid = cpu_index(ci);
+       qs = crypto_crp_ret_qs_list[cpuid];
+       mutex_enter(&qs->crp_ret_q_mtx);
+       return qs;
+}
 
-/*
- * current queue length.
- */
-DEFINIT_CRYPTO_Q_LEN(crp_ret_q);
-DEFINIT_CRYPTO_Q_LEN(crp_ret_kq);
+static inline void
+crypto_put_crp_ret_qs(struct cpu_info *ci)
+{
+       u_int cpuid;
+       struct crypto_crp_ret_qs *qs;
 
-/*
- * queue dropped count.
- */
-DEFINIT_CRYPTO_Q_DROPS(crp_ret_q);
-DEFINIT_CRYPTO_Q_DROPS(crp_ret_kq);
+       KASSERT(ci != NULL);
+
+       cpuid = cpu_index(ci);
+       qs = crypto_crp_ret_qs_list[cpuid];
+       mutex_exit(&qs->crp_ret_q_mtx);
+}
 
 #ifndef CRYPTO_RET_Q_MAXLEN
 #define CRYPTO_RET_Q_MAXLEN 0
@@ -172,67 +207,162 @@
 #ifndef CRYPTO_RET_KQ_MAXLEN
 #define CRYPTO_RET_KQ_MAXLEN 0
 #endif
-/*
- * queue length limit.
- * default value is 0. <=0 means unlimited.
- */
-DEFINIT_CRYPTO_Q_MAXLEN(crp_ret_q, CRYPTO_RET_Q_MAXLEN);
-DEFINIT_CRYPTO_Q_MAXLEN(crp_ret_kq, CRYPTO_RET_KQ_MAXLEN);
 
-/*
- * TODO:
- * make percpu
- */
 static int
 sysctl_opencrypto_q_len(SYSCTLFN_ARGS)
 {
-       int error;
+       int error, len = 0;
+       struct sysctlnode node = *rnode;
+
+       for (int i = 0; i < ncpu; i++) {
+               struct crypto_crp_ret_qs *qs;
+               struct cpu_info *ci = cpu_lookup(i);
+
+               qs = crypto_get_crp_ret_qs(ci);
+               len += qs->crp_ret_q_len;
+               crypto_put_crp_ret_qs(ci);
+       }
+
+       node.sysctl_data = &len;
+       error = sysctl_lookup(SYSCTLFN_CALL(&node));
+       if (error || newp == NULL)
+               return error;
 
-       error = sysctl_lookup(SYSCTLFN_CALL(rnode));
+       return 0;
+}
+
+static int
+sysctl_opencrypto_q_drops(SYSCTLFN_ARGS)
+{
+       int error, drops = 0;
+       struct sysctlnode node = *rnode;
+
+       for (int i = 0; i < ncpu; i++) {
+               struct crypto_crp_ret_qs *qs;
+               struct cpu_info *ci = cpu_lookup(i);
+
+               qs = crypto_get_crp_ret_qs(ci);
+               drops += qs->crp_ret_q_drops;
+               crypto_put_crp_ret_qs(ci);
+       }
+
+       node.sysctl_data = &drops;
+       error = sysctl_lookup(SYSCTLFN_CALL(&node));
        if (error || newp == NULL)
                return error;
 
        return 0;
 }
 
-/*
- * TODO:
- * make percpu
- */
 static int
-sysctl_opencrypto_q_drops(SYSCTLFN_ARGS)
+sysctl_opencrypto_q_maxlen(SYSCTLFN_ARGS)
 {
-       int error;
+       int error, maxlen;
+       struct crypto_crp_ret_qs *qs;