This is a multi-part message in MIME format.
--------------000702070409040908010405
Content-Type: text/plain; charset=us-ascii; format=flowed
Content-Transfer-Encoding: 7bit

Greetings, I am not sure if this question is appropriate on this list, 
but I thought somebody might know the answer. I wrote a simple TCP/IP 
server, which listens for client requests and then prints out a text 
message from the client. The server uses Posix threads to service 
requests from clients concurrently. I start a client, which creates many 
concurrent connections to the server and for each connection it sends a 
text message. The server pre threads a certain number of threads when it 
starts, which then process client requests, the problem is, after N 
requests have been processed, there is a 3 second wait, and then the 
next N requests are processed, then there is another 3 second wait, etc. 
To me it seems that accept() blocks for longer than it should. I did 
tests and found out that the number of requests processed before a 3 
second wait is proportional to backlog number, as in when server calls 
listen(int sockfd, int backlog). A similar non threaded server (the one 
that simply forks new children instead of creating threads) behaves much 
better, even for small backlog numbers. I looked in my code to see where 
the error could be, but I just can't see it.

I am writing and testing this code on NetBSD-2.0F, if anyone has any 
hints, I would really appreciate your help. If you're interested I've 
attached the sources for the server.

--------------000702070409040908010405
Content-Type: text/plain;
 name="tcpserv_thr.c"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="tcpserv_thr.c"

/*
** Multi-threaded implementation of a simple server
*/

#include <stdio.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <unistd.h>
#include <pthread.h>

#define SERV_PORT 8000
#define BACKLOGNUM 2
#define NUM_THREADS 64 

void *serv_req(void *);

/* Structure for thread synchronization */
struct {
	pthread_mutex_t mutex;
	int listenfd;
} tsync;

int main(int argc, char *argv[]) {
	struct sockaddr_in servaddr;
	int pthread_retval, i;
	pthread_t tid[NUM_THREADS];
	int tnum[NUM_THREADS]; /* thread number */
	
	/* stdio is unbuffed */
	setbuf(stdout, NULL);

	pthread_retval = pthread_mutex_init(&(tsync.mutex), NULL);
	if(pthread_retval != 0) {
		fprintf(stderr, "pthread_mutex_init() error: errno %d, current line %d, file %s\n", pthread_retval, __LINE__, __FILE__);
		exit(-1);
	}
	
	if((tsync.listenfd = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
		fprintf(stderr, "socket() error: errno %d, current line %d, file %s\n", errno, __LINE__, __FILE__);
		exit(-1);
	}
	memset(&servaddr, 0, sizeof(servaddr));
	servaddr.sin_family = AF_INET;
	/* servaddr.sin_addr.s_addr = htonl(INADDR_ANY); */
	inet_pton(AF_INET, "10.0.0.16", &servaddr.sin_addr); 
	servaddr.sin_port = htons(SERV_PORT);
	
	if(bind(tsync.listenfd, (struct sockaddr *)&servaddr, sizeof(servaddr)) < 0) {
		fprintf(stderr, "bind() error: errno %d, current line %d, file %s\n", errno, __LINE__, __FILE__);
		exit(-1);
	}
	
	if(listen(tsync.listenfd, BACKLOGNUM) < 0) {
		fprintf(stderr, "listen() error: errno %d, current line %d, file %s\n", errno, __LINE__, __FILE__);
		exit(-1);
	}
	
	printf("...Creating %d threads\n", NUM_THREADS);
	for(i=0; i<NUM_THREADS; i++) {
		tnum[i] = i;
		pthread_retval = pthread_create(&tid[i], NULL, serv_req, &tnum[i]);
		if(pthread_retval != 0) {
			fprintf(stderr, "pthread_create() error: errno %d, current line %d, file %s\n", pthread_retval, __LINE__, __FILE__);
			exit(-1);
		}
	}

	printf("...Done\n");
	for(i=0; i<NUM_THREADS; i++)
		pthread_join(tid[i], NULL);
	pthread_exit(NULL);
}

/*
** Function to process client requests
*/
void *serv_req(void *arg) {
	ssize_t nread;
	char buf[64];
	char peer_addrstr[INET6_ADDRSTRLEN];
	struct sockaddr_in cliaddr;
	int connfd, clilen;
	int pthread_retval;

	while(1) {
		clilen = sizeof(cliaddr);
		
		/* Lock the mutex, so that only one thread calls accept() at a time */
		pthread_retval = pthread_mutex_lock(&(tsync.mutex));
		if(pthread_retval != 0) {
			fprintf(stderr, "pthread_mutex_lock() error: errno %d, current line %d, file %s\n", pthread_retval, __LINE__, __FILE__);
			exit(-1);
		}
		
		if((connfd = accept(tsync.listenfd, (struct sockaddr *)&cliaddr, &clilen)) < 0) {
			fprintf(stderr, "accept() error: errno %d, current line %d, file %s\n", errno, __LINE__, __FILE__);
			exit(-1);
		}

		pthread_retval = pthread_mutex_unlock(&(tsync.mutex));
		if(pthread_retval != 0) {
			fprintf(stderr, "pthread_mutex_unlock() error: errno %d, current line %d, file %s\n", pthread_retval, __LINE__, __FILE__);
			exit(-1);
		}

		inet_ntop(AF_INET, &cliaddr.sin_addr, peer_addrstr, sizeof(peer_addrstr));
		printf("Thread %d accepted connection on fd %d from %s, port %d\n", *((int *)arg), connfd, peer_addrstr, ntohs(cliaddr.sin_port)); 
		/* Read what client sent us into buffer and then print it out */
		while((nread = read(connfd, buf, sizeof(buf))) > 0)
			printf("Thread %d received: %s", *((int *)arg), buf);
		if(nread < 0) {
			fprintf(stderr, "Thread %d read() error: errno %d, current line %d, file %s\n", *((int *)arg), errno, __LINE__, __FILE__);
			exit(-1);
		}
		
		close(connfd);
		printf("Thread %d closed socket descriptor\n", *((int *)arg));
	}
}


--------------000702070409040908010405--