[src/trunk]: src/lib Use sendmsg() instead multiple calls to sendto(). It be...

[pooka <pooka%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/0d5d52161aeb
branches:  trunk
changeset: 763023:0d5d52161aeb
user:      pooka <pooka%NetBSD.org@localhost>
date:      Tue Mar 08 15:34:37 2011 +0000

description:
Use sendmsg() instead multiple calls to sendto().  It behaves
slightly better in the multithreaded exec case in terms of partial
frames sent.  Plus, it's theoretically a little cheaper.

There's still a gray area with partial transfer from sendmsg(),
but I'm not sure I care enough about some academic scenario to
start fixing it (it basically needs an OOB exec signaller).

diffstat:

 lib/librumpclient/rumpclient.c |  55 ++++++++++++++++++++++--------------
 lib/librumpuser/rumpuser_sp.c  |  63 ++++++++++++++++++++++++++++++-----------
 lib/librumpuser/sp_common.c    |  41 ++++++++++++++++++++------
 3 files changed, 111 insertions(+), 48 deletions(-)

diffs (truncated from 481 to 300 lines):

diff -r 20870274a734 -r 0d5d52161aeb lib/librumpclient/rumpclient.c
--- a/lib/librumpclient/rumpclient.c    Tue Mar 08 15:12:46 2011 +0000
+++ b/lib/librumpclient/rumpclient.c    Tue Mar 08 15:34:37 2011 +0000
@@ -1,4 +1,4 @@
-/*      $NetBSD: rumpclient.c,v 1.38 2011/02/27 12:58:29 pooka Exp $   */
+/*      $NetBSD: rumpclient.c,v 1.39 2011/03/08 15:34:37 pooka Exp $   */
 
 /*
  * Copyright (c) 2010, 2011 Antti Kantee.  All Rights Reserved.
@@ -30,7 +30,7 @@
  */
 
 #include <sys/cdefs.h>
-__RCSID("$NetBSD: rumpclient.c,v 1.38 2011/02/27 12:58:29 pooka Exp $");
+__RCSID("$NetBSD: rumpclient.c,v 1.39 2011/03/08 15:34:37 pooka Exp $");
 
 #include <sys/param.h>
 #include <sys/event.h>
@@ -66,8 +66,7 @@
 int    (*host_fcntl)(int, int, ...);
 int    (*host_poll)(struct pollfd *, nfds_t, int);
 ssize_t        (*host_read)(int, void *, size_t);
-ssize_t (*host_sendto)(int, const void *, size_t, int,
-                      const struct sockaddr *, socklen_t);
+ssize_t (*host_sendmsg)(int, const struct msghdr *, int);
 int    (*host_setsockopt)(int, int, int, const void *, socklen_t);
 int    (*host_dup)(int);
 
@@ -96,7 +95,7 @@
 static time_t retrytimo = 0;
 
 static int
-send_with_recon(struct spclient *spc, const void *data, size_t dlen)
+send_with_recon(struct spclient *spc, struct iovec *iov, size_t iovlen)
 {
        struct timeval starttime, curtime;
        time_t prevreconmsg;
@@ -104,7 +103,7 @@
        int rv;
 
        for (prevreconmsg = 0, reconretries = 0;;) {
-               rv = dosend(spc, data, dlen);
+               rv = dosend(spc, iov, iovlen);
                if (__predict_false(rv == ENOTCONN || rv == EBADF)) {
                        /* no persistent connections */
                        if (retrytimo == 0) {
@@ -287,6 +286,7 @@
 {
        struct rsp_hdr rhdr;
        struct respwait rw;
+       struct iovec iov[2];
        int rv;
 
        rhdr.rsp_len = sizeof(rhdr) + dlen;
@@ -294,13 +294,12 @@
        rhdr.rsp_type = RUMPSP_SYSCALL;
        rhdr.rsp_sysnum = sysnum;
 
+       IOVPUT(iov[0], rhdr);
+       IOVPUT_WITHSIZE(iov[1], __UNCONST(data), dlen);
+
        do {
                putwait(spc, &rw, &rhdr);
-               if ((rv = send_with_recon(spc, &rhdr, sizeof(rhdr))) != 0) {
-                       unputwait(spc, &rw);
-                       continue;
-               }
-               if ((rv = send_with_recon(spc, data, dlen)) != 0) {
+               if ((rv = send_with_recon(spc, iov, __arraycount(iov))) != 0) {
                        unputwait(spc, &rw);
                        continue;
                }
@@ -319,16 +318,19 @@
        int cancel, bool haslock)
 {
        struct handshake_fork rf;
+       const char *myprogname;
        struct rsp_hdr rhdr;
        struct respwait rw;
        sigset_t omask;
        size_t bonus;
+       struct iovec iov[2];
        int rv;
 
        if (type == HANDSHAKE_FORK) {
                bonus = sizeof(rf);
        } else {
-               bonus = strlen(getprogname())+1;
+               myprogname = getprogname();
+               bonus = strlen(myprogname)+1;
        }
 
        /* performs server handshake */
@@ -337,19 +339,21 @@
        rhdr.rsp_type = RUMPSP_HANDSHAKE;
        rhdr.rsp_handshake = type;
 
+       IOVPUT(iov[0], rhdr);
+
        pthread_sigmask(SIG_SETMASK, &fullset, &omask);
        if (haslock)
                putwait_locked(spc, &rw, &rhdr);
        else
                putwait(spc, &rw, &rhdr);
-       rv = dosend(spc, &rhdr, sizeof(rhdr));
        if (type == HANDSHAKE_FORK) {
                memcpy(rf.rf_auth, data, sizeof(rf.rf_auth)); /* uh, why? */
                rf.rf_cancel = cancel;
-               rv = send_with_recon(spc, &rf, sizeof(rf));
+               IOVPUT(iov[1], rf);
        } else {
-               rv = dosend(spc, getprogname(), strlen(getprogname())+1);
+               IOVPUT_WITHSIZE(iov[1], __UNCONST(getprogname()), bonus);
        }
+       rv = send_with_recon(spc, iov, __arraycount(iov));
        if (rv || cancel) {
                if (haslock)
                        unputwait_locked(spc, &rw);
@@ -377,6 +381,7 @@
 {
        struct rsp_hdr rhdr;
        struct respwait rw;
+       struct iovec iov[1];
        int rv;
 
        rhdr.rsp_len = sizeof(rhdr);
@@ -384,9 +389,11 @@
        rhdr.rsp_type = RUMPSP_PREFORK;
        rhdr.rsp_error = 0;
 
+       IOVPUT(iov[0], rhdr);
+
        do {
                putwait(spc, &rw, &rhdr);
-               rv = send_with_recon(spc, &rhdr, sizeof(rhdr));
+               rv = send_with_recon(spc, iov, __arraycount(iov));
                if (rv != 0) {
                        unputwait(spc, &rw);
                        continue;
@@ -430,6 +437,7 @@
        int wantstr)
 {
        struct rsp_hdr rhdr;
+       struct iovec iov[2];
 
        if (wantstr)
                dlen = MIN(dlen, strlen(data)+1);
@@ -440,10 +448,12 @@
        rhdr.rsp_type = RUMPSP_COPYIN;
        rhdr.rsp_sysnum = 0;
 
+       IOVPUT(iov[0], rhdr);
+       IOVPUT_WITHSIZE(iov[1], data, dlen);
+
        if (resp_sendlock(spc) != 0)
                return;
-       (void)dosend(spc, &rhdr, sizeof(rhdr));
-       (void)dosend(spc, data, dlen);
+       (void)SENDIOV(spc, iov);
        sendunlock(spc);
 }
 
@@ -451,6 +461,7 @@
 send_anonmmap_resp(struct spclient *spc, uint64_t reqno, void *addr)
 {
        struct rsp_hdr rhdr;
+       struct iovec iov[2];
 
        rhdr.rsp_len = sizeof(rhdr) + sizeof(addr);
        rhdr.rsp_reqno = reqno;
@@ -458,10 +469,12 @@
        rhdr.rsp_type = RUMPSP_ANONMMAP;
        rhdr.rsp_sysnum = 0;
 
+       IOVPUT(iov[0], rhdr);
+       IOVPUT(iov[1], addr);
+
        if (resp_sendlock(spc) != 0)
                return;
-       (void)dosend(spc, &rhdr, sizeof(rhdr));
-       (void)dosend(spc, &addr, sizeof(addr));
+       (void)SENDIOV(spc, iov);
        sendunlock(spc);
 }
 
@@ -775,7 +788,7 @@
        FINDSYM(fcntl)
        FINDSYM(poll)
        FINDSYM(read)
-       FINDSYM(sendto)
+       FINDSYM(sendmsg)
        FINDSYM(setsockopt)
        FINDSYM(dup)
        FINDSYM(kqueue)
diff -r 20870274a734 -r 0d5d52161aeb lib/librumpuser/rumpuser_sp.c
--- a/lib/librumpuser/rumpuser_sp.c     Tue Mar 08 15:12:46 2011 +0000
+++ b/lib/librumpuser/rumpuser_sp.c     Tue Mar 08 15:34:37 2011 +0000
@@ -1,4 +1,4 @@
-/*      $NetBSD: rumpuser_sp.c,v 1.44 2011/03/08 12:39:28 pooka Exp $  */
+/*      $NetBSD: rumpuser_sp.c,v 1.45 2011/03/08 15:34:37 pooka Exp $  */
 
 /*
  * Copyright (c) 2010, 2011 Antti Kantee.  All Rights Reserved.
@@ -35,7 +35,7 @@
  */
 
 #include <sys/cdefs.h>
-__RCSID("$NetBSD: rumpuser_sp.c,v 1.44 2011/03/08 12:39:28 pooka Exp $");
+__RCSID("$NetBSD: rumpuser_sp.c,v 1.45 2011/03/08 15:34:37 pooka Exp $");
 
 #include <sys/types.h>
 #include <sys/atomic.h>
@@ -238,10 +238,16 @@
        return nw;
 }
 
+/*
+ * XXX: we send responses with "blocking" I/O.  This is not
+ * ok for the main thread.  XXXFIXME
+ */
+
 static void
 send_error_resp(struct spclient *spc, uint64_t reqno, int error)
 {
        struct rsp_hdr rhdr;
+       struct iovec iov[1];
 
        rhdr.rsp_len = sizeof(rhdr);
        rhdr.rsp_reqno = reqno;
@@ -249,8 +255,10 @@
        rhdr.rsp_type = 0;
        rhdr.rsp_error = error;
 
+       IOVPUT(iov[0], rhdr);
+
        sendlock(spc);
-       (void)dosend(spc, &rhdr, sizeof(rhdr));
+       (void)SENDIOV(spc, iov);
        sendunlock(spc);
 }
 
@@ -258,6 +266,7 @@
 send_handshake_resp(struct spclient *spc, uint64_t reqno, int error)
 {
        struct rsp_hdr rhdr;
+       struct iovec iov[2];
        int rv;
 
        rhdr.rsp_len = sizeof(rhdr) + sizeof(error);
@@ -266,9 +275,11 @@
        rhdr.rsp_type = RUMPSP_HANDSHAKE;
        rhdr.rsp_error = 0;
 
+       IOVPUT(iov[0], rhdr);
+       IOVPUT(iov[1], error);
+
        sendlock(spc);
-       rv = dosend(spc, &rhdr, sizeof(rhdr));
-       rv = dosend(spc, &error, sizeof(error));
+       rv = SENDIOV(spc, iov);
        sendunlock(spc);
 
        return rv;
@@ -280,6 +291,7 @@
 {
        struct rsp_hdr rhdr;
        struct rsp_sysresp sysresp;
+       struct iovec iov[2];
        int rv;
 
        rhdr.rsp_len = sizeof(rhdr) + sizeof(sysresp);
@@ -291,9 +303,11 @@
        sysresp.rsys_error = error;
        memcpy(sysresp.rsys_retval, retval, sizeof(sysresp.rsys_retval));
 
+       IOVPUT(iov[0], rhdr);
+       IOVPUT(iov[1], sysresp);
+
        sendlock(spc);
-       rv = dosend(spc, &rhdr, sizeof(rhdr));
-       rv = dosend(spc, &sysresp, sizeof(sysresp));
+       rv = SENDIOV(spc, iov);
        sendunlock(spc);
 
        return rv;
@@ -303,6 +317,7 @@
 send_prefork_resp(struct spclient *spc, uint64_t reqno, uint32_t *auth)
 {
        struct rsp_hdr rhdr;
+       struct iovec iov[2];
        int rv;
 
        rhdr.rsp_len = sizeof(rhdr) + AUTHLEN*sizeof(*auth);
@@ -311,9 +326,11 @@
        rhdr.rsp_type = RUMPSP_PREFORK;
        rhdr.rsp_sysnum = 0;
 
+       IOVPUT(iov[0], rhdr);
+       IOVPUT_WITHSIZE(iov[1], auth, AUTHLEN*sizeof(*auth));
+