[src/trunk]: src/lib/librumpuser Unschedule from CPU for out-of-kernel blocki...

[pooka <pooka%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/19325e5da33b
branches:  trunk
changeset: 759016:19325e5da33b
user:      pooka <pooka%NetBSD.org@localhost>
date:      Wed Nov 24 17:00:10 2010 +0000

description:
Unschedule from CPU for out-of-kernel blocking ops.  Otherwise we
might even deadlock if the thread that wakes us up wants a CPU.

diffstat:

 lib/librumpuser/rumpuser_sp.c |  65 ++++++++++++++++++++++++++++--------------
 lib/librumpuser/sp_common.c   |  22 ++++++++++++-
 2 files changed, 62 insertions(+), 25 deletions(-)

diffs (211 lines):

diff -r bb97117703a3 -r 19325e5da33b lib/librumpuser/rumpuser_sp.c
--- a/lib/librumpuser/rumpuser_sp.c     Wed Nov 24 16:31:12 2010 +0000
+++ b/lib/librumpuser/rumpuser_sp.c     Wed Nov 24 17:00:10 2010 +0000
@@ -1,4 +1,4 @@
-/*      $NetBSD: rumpuser_sp.c,v 1.12 2010/11/24 15:17:46 pooka Exp $  */
+/*      $NetBSD: rumpuser_sp.c,v 1.13 2010/11/24 17:00:10 pooka Exp $  */
 
 /*
  * Copyright (c) 2010 Antti Kantee.  All Rights Reserved.
@@ -38,7 +38,7 @@
  */
 
 #include <sys/cdefs.h>
-__RCSID("$NetBSD: rumpuser_sp.c,v 1.12 2010/11/24 15:17:46 pooka Exp $");
+__RCSID("$NetBSD: rumpuser_sp.c,v 1.13 2010/11/24 17:00:10 pooka Exp $");
 
 #include <sys/types.h>
 #include <sys/atomic.h>
@@ -61,6 +61,7 @@
 #include <unistd.h>
 
 #include <rump/rumpuser.h>
+#include "rumpuser_int.h"
 
 #include "sp_common.c"
 
@@ -211,13 +212,12 @@
        copydata.rcp_len = dlen;
 
        putwait(spc, &rw, &rhdr);
-
-       sendlock(spc);
        rv = dosend(spc, &rhdr, sizeof(rhdr));
        rv = dosend(spc, &copydata, sizeof(copydata));
-       sendunlock(spc);
-       if (rv)
-               return rv; /* XXX: unputwait */
+       if (rv) {
+               unputwait(spc, &rw);
+               return rv;
+       }
 
        rv = waitresp(spc, &rw);
 
@@ -271,15 +271,15 @@
        rhdr.rsp_sysnum = 0;
 
        putwait(spc, &rw, &rhdr);
-
-       sendlock(spc);
        rv = dosend(spc, &rhdr, sizeof(rhdr));
        rv = dosend(spc, &howmuch, sizeof(howmuch));
-       sendunlock(spc);
-       if (rv)
-               return rv; /* XXX: unputwait */
+       if (rv) {
+               unputwait(spc, &rw);
+               return rv;
+       }
 
        rv = waitresp(spc, &rw);
+
        *resp = rw.rw_data;
 
        DPRINTF(("anonmmap: mapped at %p\n", **(void ***)resp));
@@ -310,7 +310,7 @@
 
        DPRINTF(("spcrelease: spc %p fd %d\n", spc, spc->spc_fd));
 
-       _DIAGASSERT(TAILQ_EMPTY(spc->spc_respwait));
+       _DIAGASSERT(TAILQ_EMPTY(&spc->spc_respwait));
        _DIAGASSERT(spc->spc_buf == NULL);
 
        lwproc_switch(spc->spc_mainlwp);
@@ -394,8 +394,8 @@
 
        TAILQ_INIT(&spclist[i].spc_respwait);
 
-       DPRINTF(("rump_sp: added new connection at idx %u, pid %d\n",
-           i, lwproc_getpid()));
+       DPRINTF(("rump_sp: added new connection fd %d at idx %u, pid %d\n",
+           newfd, i, lwproc_getpid()));
 
        lwproc_switch(NULL);
 
@@ -444,15 +444,21 @@
 {
        struct spclient *spc = arg;
        void *rdata = NULL; /* XXXuninit */
-       int rv;
+       int rv, nlocks;
+
+       rumpuser__kunlock(0, &nlocks, NULL);
 
        rv = copyin_req(spc, uaddr, len, &rdata);
        if (rv)
-               return EFAULT;
+               goto out;
 
        memcpy(kaddr, rdata, len);
        free(rdata);
 
+ out:
+       rumpuser__klock(nlocks, NULL);
+       if (rv)
+               return EFAULT;
        return 0;
 }
 
@@ -460,8 +466,13 @@
 rumpuser_sp_copyout(void *arg, const void *kaddr, void *uaddr, size_t dlen)
 {
        struct spclient *spc = arg;
+       int nlocks, rv;
 
-       if (send_copyout_req(spc, uaddr, kaddr, dlen) != 0)
+       rumpuser__kunlock(0, &nlocks, NULL);
+       rv = send_copyout_req(spc, uaddr, kaddr, dlen);
+       rumpuser__klock(nlocks, NULL);
+
+       if (rv)
                return EFAULT;
        return 0;
 }
@@ -471,20 +482,30 @@
 {
        struct spclient *spc = arg;
        void *resp, *rdata;
-       int rv;
+       int nlocks, rv;
+
+       rumpuser__kunlock(0, &nlocks, NULL);
 
        rv = anonmmap_req(spc, howmuch, &rdata);
-       if (rv)
-               return rv;
+       if (rv) {
+               rv = EFAULT;
+               goto out;
+       }
 
        resp = *(void **)rdata;
        free(rdata);
 
        if (resp == NULL) {
-               return ENOMEM;
+               rv = ENOMEM;
        }
 
        *addr = resp;
+
+ out:
+       rumpuser__klock(nlocks, NULL);
+
+       if (rv)
+               return rv;
        return 0;
 }
 
diff -r bb97117703a3 -r 19325e5da33b lib/librumpuser/sp_common.c
--- a/lib/librumpuser/sp_common.c       Wed Nov 24 16:31:12 2010 +0000
+++ b/lib/librumpuser/sp_common.c       Wed Nov 24 17:00:10 2010 +0000
@@ -1,4 +1,4 @@
-/*      $NetBSD: sp_common.c,v 1.7 2010/11/24 14:32:42 pooka Exp $     */
+/*      $NetBSD: sp_common.c,v 1.8 2010/11/24 17:00:10 pooka Exp $     */
 
 /*
  * Copyright (c) 2010 Antti Kantee.  All Rights Reserved.
@@ -216,7 +216,15 @@
        pthread_mutex_lock(&spc->spc_mtx);
        rw->rw_reqno = rhdr->rsp_reqno = spc->spc_nextreq++;
        TAILQ_INSERT_TAIL(&spc->spc_respwait, rw, rw_entries);
+}
+
+static void
+unputwait(struct spclient *spc, struct respwait *rw)
+{
+
+       TAILQ_REMOVE(&spc->spc_respwait, rw, rw_entries);
        pthread_mutex_unlock(&spc->spc_mtx);
+       pthread_cond_destroy(&rw->rw_cv);
 }
 
 static void
@@ -258,7 +266,6 @@
        struct pollfd pfd;
        int rv = 0;
 
-       pthread_mutex_lock(&spc->spc_mtx);
        while (rw->rw_data == NULL && spc->spc_dying == 0) {
                /* are we free to receive? */
                if (spc->spc_istatus == SPCSTATUS_FREE) {
@@ -271,8 +278,17 @@
                        pfd.events = POLLIN;
 
                        for (gotresp = 0; !gotresp; ) {
-                               while (readframe(spc) < 1)
+                               rv = readframe(spc);
+                               switch (rv) {
+                               case 0:
                                        poll(&pfd, 1, INFTIM);
+                                       continue;
+                               case -1:
+                                       spc->spc_dying = 1;
+                                       break;
+                               default:
+                                       break;
+                               }
 
                                switch (spc->spc_hdr.rsp_class) {
                                case RUMPSP_RESP: