[src/trunk]: src/sys/dev/ic nvme: Make sure that q_ccb_list is always accesse...

[jmcneill <jmcneill%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/2a02f951b30d
branches:  trunk
changeset: 368960:2a02f951b30d
user:      jmcneill <jmcneill%NetBSD.org@localhost>
date:      Sun Aug 14 12:08:57 2022 +0000

description:
nvme: Make sure that q_ccb_list is always accessed with the q lock held.

diffstat:

 sys/dev/ic/nvme.c    |  49 +++++++++++++++++++++++++++++++++++++++++++------
 sys/dev/ic/nvmevar.h |  17 +++--------------
 2 files changed, 46 insertions(+), 20 deletions(-)

diffs (134 lines):

diff -r 1e8254c7f4d3 -r 2a02f951b30d sys/dev/ic/nvme.c
--- a/sys/dev/ic/nvme.c Sun Aug 14 11:26:41 2022 +0000
+++ b/sys/dev/ic/nvme.c Sun Aug 14 12:08:57 2022 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: nvme.c,v 1.61 2022/07/31 12:02:28 mlelstv Exp $        */
+/*     $NetBSD: nvme.c,v 1.62 2022/08/14 12:08:57 jmcneill Exp $       */
 /*     $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */
 
 /*
@@ -18,7 +18,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.61 2022/07/31 12:02:28 mlelstv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.62 2022/08/14 12:08:57 jmcneill Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -88,6 +88,9 @@
 
 static struct nvme_ccb *
                nvme_ccb_get(struct nvme_queue *, bool);
+static struct nvme_ccb *
+               nvme_ccb_get_bio(struct nvme_softc *, struct buf *,
+                   struct nvme_queue **);
 static void    nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *);
 
 static int     nvme_poll(struct nvme_softc *, struct nvme_queue *,
@@ -768,12 +771,12 @@
     struct buf *bp, void *data, size_t datasize,
     int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done)
 {
-       struct nvme_queue *q = nvme_get_q(sc, bp, false);
+       struct nvme_queue *q;
        struct nvme_ccb *ccb;
        bus_dmamap_t dmap;
        int i, error;
 
-       ccb = nvme_ccb_get(q, false);
+       ccb = nvme_ccb_get_bio(sc, bp, &q);
        if (ccb == NULL)
                return EAGAIN;
 
@@ -910,7 +913,7 @@
 int
 nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, int flags)
 {
-       struct nvme_queue *q = nvme_get_q(sc, NULL, true);
+       struct nvme_queue *q = nvme_get_q(sc);
        struct nvme_ccb *ccb;
        int result = 0;
 
@@ -1277,7 +1280,7 @@
            (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts)))
                return EINVAL;
 
-       q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc, NULL, true);
+       q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc);
        ccb = nvme_ccb_get(q, true);
        KASSERT(ccb != NULL);
 
@@ -1852,6 +1855,40 @@
        return ccb;
 }
 
+static struct nvme_ccb *
+nvme_ccb_get_bio(struct nvme_softc *sc, struct buf *bp,
+    struct nvme_queue **selq)
+{
+       u_int cpuindex = cpu_index(bp->b_ci);
+
+       /*
+        * Find a queue with available ccbs, preferring the originating
+        * CPU's queue.
+        */
+
+       for (u_int qoff = 0; qoff < sc->sc_nq; qoff++) {
+               struct nvme_queue *q = sc->sc_q[(cpuindex + qoff) % sc->sc_nq];
+               struct nvme_ccb *ccb;
+
+               mutex_enter(&q->q_ccb_mtx);
+               ccb = SIMPLEQ_FIRST(&q->q_ccb_list);
+               if (ccb != NULL) {
+                       SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry);
+#ifdef DEBUG
+                       ccb->ccb_cookie = NULL;
+#endif
+               }
+               mutex_exit(&q->q_ccb_mtx);
+
+               if (ccb != NULL) {
+                       *selq = q;
+                       return ccb;
+               }
+       }
+
+       return NULL;
+}
+
 static void
 nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb)
 {
diff -r 1e8254c7f4d3 -r 2a02f951b30d sys/dev/ic/nvmevar.h
--- a/sys/dev/ic/nvmevar.h      Sun Aug 14 11:26:41 2022 +0000
+++ b/sys/dev/ic/nvmevar.h      Sun Aug 14 12:08:57 2022 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: nvmevar.h,v 1.27 2022/08/01 08:09:30 mlelstv Exp $     */
+/*     $NetBSD: nvmevar.h,v 1.28 2022/08/14 12:08:57 jmcneill Exp $    */
 /*     $OpenBSD: nvmevar.h,v 1.8 2016/04/14 11:18:32 dlg Exp $ */
 
 /*
@@ -197,20 +197,9 @@
 void   nvme_softintr_msi(void *);
 
 static __inline struct nvme_queue *
-nvme_get_q(struct nvme_softc *sc, struct buf *bp, bool waitok)
+nvme_get_q(struct nvme_softc *sc)
 {
-       struct cpu_info *ci = (bp && bp->b_ci) ? bp->b_ci : curcpu();
-
-       /*
-        * Find a queue with available ccbs, preferring the originating CPU's queue.
-        */
-
-       for (u_int qoff = 0; qoff < sc->sc_nq; qoff++) {
-               struct nvme_queue *q = sc->sc_q[(cpu_index(ci) + qoff) % sc->sc_nq];
-               if (!SIMPLEQ_EMPTY(&q->q_ccb_list) || waitok)
-                       return q;
-       }
-       return NULL;
+       return sc->sc_q[cpu_index(curcpu()) % sc->sc_nq];
 }
 
 /*