[src/trunk]: src/external/bsd/fetch/dist/libfetch libfetch-2.30:

[joerg <joerg%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/79f690a83760
branches:  trunk
changeset: 751267:79f690a83760
user:      joerg <joerg%NetBSD.org@localhost>
date:      Sat Jan 30 21:26:09 2010 +0000

description:
libfetch-2.30:
- Revamped connection cache, allowing more than one active session
- HTTP keep-alive support

diffstat:

 external/bsd/fetch/dist/libfetch/common.c |  221 +++++++++++++++++++----------
 external/bsd/fetch/dist/libfetch/common.h |   22 +-
 external/bsd/fetch/dist/libfetch/fetch.3  |   20 ++-
 external/bsd/fetch/dist/libfetch/fetch.h  |    6 +-
 external/bsd/fetch/dist/libfetch/ftp.c    |  159 +++++++--------------
 external/bsd/fetch/dist/libfetch/http.c   |  105 +++++++++++---
 6 files changed, 309 insertions(+), 224 deletions(-)

diffs (truncated from 1169 to 300 lines):

diff -r a04619177d37 -r 79f690a83760 external/bsd/fetch/dist/libfetch/common.c
--- a/external/bsd/fetch/dist/libfetch/common.c Sat Jan 30 21:23:46 2010 +0000
+++ b/external/bsd/fetch/dist/libfetch/common.c Sat Jan 30 21:26:09 2010 +0000
@@ -1,7 +1,7 @@
-/*     $NetBSD: common.c,v 1.1.1.7 2009/10/15 12:59:57 joerg Exp $     */
+/*     $NetBSD: common.c,v 1.1.1.8 2010/01/30 21:26:09 joerg Exp $     */
 /*-
  * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
- * Copyright (c) 2008 Joerg Sonnenberger <joerg%NetBSD.org@localhost>
+ * Copyright (c) 2008, 2010 Joerg Sonnenberger <joerg%NetBSD.org@localhost>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -62,12 +62,13 @@
 #include <string.h>
 #include <unistd.h>
 
+#ifndef MSG_NOSIGNAL
+#include <signal.h>
+#endif
+
 #include "fetch.h"
 #include "common.h"
 
-#define DECONST(x,y) ((x)(uintptr_t)(y))
-
-
 /*** Local data **************************************************************/
 
 /*
@@ -83,10 +84,6 @@
        { -1,           FETCH_UNKNOWN,  "Unknown resolver error" }
 };
 
-/* End-of-Line */
-static const char ENDL[2] = "\r\n";
-
-
 /*** Error-reporting functions ***********************************************/
 
 /*
@@ -234,23 +231,10 @@
        /* allocate and fill connection structure */
        if ((conn = calloc(1, sizeof(*conn))) == NULL)
                return (NULL);
+       conn->cache_url = NULL;
        conn->next_buf = NULL;
        conn->next_len = 0;
        conn->sd = sd;
-       conn->is_active = 0;
-       ++conn->ref;
-       return (conn);
-}
-
-
-/*
- * Bump a connection's reference count.
- */
-conn_t *
-fetch_ref(conn_t *conn)
-{
-
-       ++conn->ref;
        return (conn);
 }
 
@@ -281,7 +265,7 @@
  * Establish a TCP connection to the specified port on the specified host.
  */
 conn_t *
-fetch_connect(const char *host, int port, int af, int verbose)
+fetch_connect(struct url *url, int af, int verbose)
 {
        conn_t *conn;
        char pbuf[10];
@@ -290,22 +274,22 @@
        int sd, error;
 
        if (verbose)
-               fetch_info("looking up %s", host);
+               fetch_info("looking up %s", url->host);
 
        /* look up host name and set up socket address structure */
-       snprintf(pbuf, sizeof(pbuf), "%d", port);
+       snprintf(pbuf, sizeof(pbuf), "%d", url->port);
        memset(&hints, 0, sizeof(hints));
        hints.ai_family = af;
        hints.ai_socktype = SOCK_STREAM;
        hints.ai_protocol = 0;
-       if ((error = getaddrinfo(host, pbuf, &hints, &res0)) != 0) {
+       if ((error = getaddrinfo(url->host, pbuf, &hints, &res0)) != 0) {
                netdb_seterr(error);
                return (NULL);
        }
        bindaddr = getenv("FETCH_BIND_ADDRESS");
 
        if (verbose)
-               fetch_info("connecting to %s:%d", host, port);
+               fetch_info("connecting to %s:%d", url->host, url->port);
 
        /* try to connect */
        for (sd = -1, res = res0; res; sd = -1, res = res->ai_next) {
@@ -332,9 +316,114 @@
                fetch_syserr();
                close(sd);
        }
+       conn->cache_url = fetchCopyURL(url);
+       conn->cache_af = af;
        return (conn);
 }
 
+static conn_t *connection_cache;
+static int cache_global_limit = 0;
+static int cache_per_host_limit = 0;
+
+/*
+ * Initialise cache with the given limits.
+ */
+void
+fetchConnectionCacheInit(int global_limit, int per_host_limit)
+{
+
+       if (global_limit < 0)
+               cache_global_limit = INT_MAX;
+       else if (per_host_limit > global_limit)
+               cache_global_limit = per_host_limit;
+       else
+               cache_global_limit = global_limit;
+       if (per_host_limit < 0)
+               cache_per_host_limit = INT_MAX;
+       else
+               cache_per_host_limit = per_host_limit;
+}
+
+/*
+ * Flush cache and free all associated resources.
+ */
+void
+fetchConnectionCacheClose(void)
+{
+       conn_t *conn;
+
+       while ((conn = connection_cache) != NULL) {
+               connection_cache = conn->next_cached;
+               (*conn->cache_close)(conn);
+       }
+}
+
+/*
+ * Check connection cache for an existing entry matching
+ * protocol/host/port/user/password/family.
+ */
+conn_t *
+fetch_cache_get(const struct url *url, int af)
+{
+       conn_t *conn, *last_conn = NULL;
+
+       for (conn = connection_cache; conn; conn = conn->next_cached) {
+               if (conn->cache_url->port == url->port &&
+                   strcmp(conn->cache_url->scheme, url->scheme) == 0 &&
+                   strcmp(conn->cache_url->host, url->host) == 0 &&
+                   strcmp(conn->cache_url->user, url->user) == 0 &&
+                   strcmp(conn->cache_url->pwd, url->pwd) == 0 &&
+                   (conn->cache_af == AF_UNSPEC || af == AF_UNSPEC ||
+                    conn->cache_af == af)) {
+                       if (last_conn != NULL)
+                               last_conn->next_cached = conn->next_cached;
+                       else
+                               connection_cache = conn->next_cached;
+                       return conn;
+               }
+       }
+
+       return NULL;
+}
+
+/*
+ * Put the connection back into the cache for reuse.
+ * If the connection is freed due to LRU or if the cache
+ * is explicitly closed, the given callback is called.
+ */
+void
+fetch_cache_put(conn_t *conn, int (*closecb)(conn_t *))
+{
+       conn_t *iter, *last;
+       int global_count, host_count;
+
+       if (conn->cache_url == NULL || cache_global_limit == 0) {
+               (*closecb)(conn);
+               return;
+       }
+
+       global_count = host_count = 0;
+       last = NULL;
+       for (iter = connection_cache; iter;
+           last = iter, iter = iter->next_cached) {
+               ++global_count;
+               if (strcmp(conn->cache_url->host, iter->cache_url->host) == 0)
+                       ++host_count;
+               if (global_count < cache_global_limit &&
+                   host_count < cache_per_host_limit)
+                       continue;
+               --global_count;
+               if (last != NULL)
+                       last->next_cached = iter->next_cached;
+               else
+                       connection_cache = iter->next_cached;
+               (*iter->cache_close)(iter);
+       }
+
+       conn->cache_close = closecb;
+       conn->next_cached = connection_cache;
+       connection_cache = conn;
+}
 
 /*
  * Enable SSL on a connection.
@@ -528,31 +617,28 @@
        return (0);
 }
 
-
-/*
- * Write to a connection w/ timeout
- */
-ssize_t
-fetch_write(conn_t *conn, const char *buf, size_t len)
-{
-       struct iovec iov;
-
-       iov.iov_base = DECONST(char *, buf);
-       iov.iov_len = len;
-       return fetch_writev(conn, &iov, 1);
-}
-
 /*
  * Write a vector to a connection w/ timeout
  * Note: can modify the iovec.
  */
 ssize_t
-fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt)
+fetch_write(conn_t *conn, const void *buf, size_t len)
 {
        struct timeval now, timeout, waittv;
        fd_set writefds;
        ssize_t wlen, total;
        int r;
+#ifndef MSG_NOSIGNAL
+       static int killed_sigpipe;
+#endif
+
+#ifndef MSG_NOSIGNAL
+       if (!killed_sigpipe) {
+               signal(SIGPIPE, SIG_IGN);
+               killed_sigpipe = 1;
+       }
+#endif
+
 
        if (fetchTimeout) {
                FD_ZERO(&writefds);
@@ -561,7 +647,7 @@
        }
 
        total = 0;
-       while (iovcnt > 0) {
+       while (len) {
                while (fetchTimeout && !FD_ISSET(conn->sd, &writefds)) {
                        FD_SET(conn->sd, &writefds);
                        gettimeofday(&now, NULL);
@@ -587,11 +673,14 @@
                errno = 0;
 #ifdef WITH_SSL
                if (conn->ssl != NULL)
-                       wlen = SSL_write(conn->ssl,
-                           iov->iov_base, iov->iov_len);
+                       wlen = SSL_write(conn->ssl, buf, len);
                else
 #endif
-                       wlen = writev(conn->sd, iov, iovcnt);
+#ifndef MSG_NOSIGNAL
+                       wlen = send(conn->sd, buf, len, 0);
+#else
+                       wlen = send(conn->sd, buf, len, MSG_NOSIGNAL);
+#endif
                if (wlen == 0) {
                        /* we consider a short write a failure */
                        errno = EPIPE;
@@ -604,44 +693,14 @@
                        return (-1);
                }
                total += wlen;
-               while (iovcnt > 0 && wlen >= (ssize_t)iov->iov_len) {
-                       wlen -= iov->iov_len;
-                       iov++;
-                       iovcnt--;
-               }
-               if (iovcnt > 0) {
-                       iov->iov_len -= wlen;
-                       iov->iov_base = DECONST(char *, iov->iov_base) + wlen;