[httperf] httperf + libevent

Theodore Bullock tbullock at canada.com
Sun Jul 13 22:46:38 PDT 2008


On Wed, 2008-07-02 at 23:39 +0800, Adrian Chadd wrote:
> I decided to give libevent'ifying httperf a go.
> 
> http://code.google.com/p/httperf-adrian/
> 
> Testing please!


Alright here is my feedback after playing with this for a couple of
hours. Below is a patch which merges cleanly with the current libevent
CVS sources.

Sure enough the performance gains are phenomenal.

Check out the following CPU times for 100 connections with 10 calls per
connection.

Before:
CPU time [s]: user 8.58 system 35.29 (user 19.2% system 78.9% total
98.1%)

After:
CPU time [s]: user 0.03 system 0.08 (user 0.1% system 0.2% total 0.2%)

      * I want to see return results checked from functions that provide
        them if at all reasonable/possible.  This includes things like
        event_add and the like.  Now is the time to do this rather than
        later.
      * Unused variables like the ones in the revised core_loop should
        be removed.
      * The sys/types.h header should be brought in through the
        generic_types.h header
      * Where possible, lets migrate to the libevent timers so that we
        can eventually remove our own timer code.  Same goes for DNS
        resolution.
      * Eventually, remove the current core_loop function in lieu of a
        persistent libevent loop.

That's it for now.

See your changes below:

-Ted

Index: src/conn.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/conn.c,v
retrieving revision 1.8
diff -u -r1.8 conn.c
--- src/conn.c	16 Jul 2007 04:44:47 -0000	1.8
+++ src/conn.c	14 Jul 2008 04:55:47 -0000
@@ -39,6 +39,9 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include <sys/types.h>
+#include <event.h>
+
 #include <generic_types.h>
 #include <object.h>
 #include <timer.h>
Index: src/conn.h
===================================================================
RCS file: /cvsroot/httperf/httperf/src/conn.h,v
retrieving revision 1.7
diff -u -r1.7 conn.h
--- src/conn.h	12 Sep 2007 04:47:27 -0000	1.7
+++ src/conn.h	14 Jul 2008 04:55:47 -0000
@@ -105,6 +105,7 @@
 #ifdef HAVE_SSL
     SSL *ssl;			/* SSL connection info */
 #endif
+    struct event ev_read, ev_write;
   }
 Conn;
 
Index: src/core.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/core.c,v
retrieving revision 1.13
diff -u -r1.13 core.c
--- src/core.c	18 Mar 2008 00:31:33 -0000	1.13
+++ src/core.c	14 Jul 2008 04:55:47 -0000
@@ -49,6 +49,7 @@
 #ifdef HAVE_SYS_SELECT_H
 #include <sys/select.h>
 #endif
+#include <event.h>
 
 #include <netinet/in.h>
 #include <netinet/tcp.h>
@@ -72,7 +73,6 @@
 static int      running = 1;
 static int      iteration;
 static u_long   max_burst_len;
-static fd_set   rdfds, wrfds;
 static int      min_sd = 0x7fffffff, max_sd = 0, alloced_sd_to_conn = 0;
 static struct timeval select_timeout;
 static struct sockaddr_in myaddr;
@@ -134,6 +134,8 @@
     while (errno == EINTR);			\
   }
 #endif
+static void conn_handle_read_event(int sd, short ev, void *a);
+static void conn_handle_write_event(int sd, short ev, void *a);
 
 struct hash_entry {
 	const char     *hostname;
@@ -273,6 +275,34 @@
 }
 
 static void
+conn_read_set(Conn *s)
+{
+	if (s->sd > -1)
+		event_add(&s->ev_read, NULL);
+}
+
+static void
+conn_read_clear(Conn *s)
+{
+	if (s->sd > -1)
+		event_del(&s->ev_read);
+}
+
+static void
+conn_write_set(Conn *s)
+{
+	if (s->sd > -1)
+		event_add(&s->ev_write, NULL);
+}
+
+static void
+conn_write_clear(Conn *s)
+{
+	if (s->sd > -1)
+		event_del(&s->ev_write);
+}
+
+static void
 conn_failure(Conn * s, int err)
 {
 	Any_Type        arg;
@@ -297,11 +327,9 @@
 		c = 0;
 		if (s->sd >= 0) {
 			now = timer_now();
-			if (FD_ISSET(s->sd, &rdfds)
-			    && s->recvq && now >= s->recvq->timeout)
+			if (s->recvq && now >= s->recvq->timeout)
 				c = s->recvq;
-			else if (FD_ISSET(s->sd, &wrfds)
-				 && s->sendq && now >= s->sendq->timeout)
+			else if (s->sendq && now >= s->sendq->timeout)
 				c = s->sendq;
 		}
 		if (DBG > 0) {
@@ -319,18 +347,12 @@
 }
 
 static void
-set_active(Conn * s, fd_set * fdset)
+set_active_shared(Conn * s)
 {
 	int             sd = s->sd;
 	Any_Type        arg;
 	Time            timeout;
 
-	FD_SET(sd, fdset);
-	if (sd < min_sd)
-		min_sd = sd;
-	if (sd >= max_sd)
-		max_sd = sd;
-
 	if (s->watchdog)
 		return;
 
@@ -348,6 +370,32 @@
 }
 
 static void
+set_active_read(Conn *s)
+{
+	int sd = s->sd;
+
+	conn_read_set(s);
+	if (sd < min_sd)
+		min_sd = sd;
+	if (sd >= max_sd)
+		max_sd = sd;
+	set_active_shared(s);
+}
+
+static void
+set_active_write(Conn *s)
+{
+	int sd = s->sd;
+
+	conn_write_set(s);
+	if (sd < min_sd)
+		min_sd = sd;
+	if (sd >= max_sd)
+		max_sd = sd;
+	set_active_shared(s);
+}
+
+static void
 do_send(Conn * conn)
 {
 	int             async_errno;
@@ -434,7 +482,7 @@
 			 */
 			call->timeout =
 			    param.timeout ? timer_now() + param.timeout : 0.0;
-			set_active(conn, &wrfds);
+			set_active_write(conn);
 			return;
 		}
 
@@ -444,7 +492,7 @@
 		conn->sendq = call->sendq_next;
 		if (!conn->sendq) {
 			conn->sendq_tail = 0;
-			FD_CLR(sd, &wrfds);
+			conn_write_clear(conn);
 		}
 		arg.l = 0;
 		event_signal(EV_CALL_SEND_STOP, (Object *) call, arg);
@@ -468,7 +516,7 @@
 		call->timeout = param.timeout + param.think_timeout;
 		if (call->timeout > 0.0)
 			call->timeout += timer_now();
-		set_active(conn, &rdfds);
+		set_active_read(conn);
 		if (conn->state < S_REPLY_STATUS)
 			conn->state = S_REPLY_STATUS;	/* expecting reply
 							 * status */
@@ -491,7 +539,7 @@
 
 	conn->recvq = call->recvq_next;
 	if (!conn->recvq) {
-		FD_CLR(conn->sd, &rdfds);
+		conn_read_clear(conn);
 		conn->recvq_tail = 0;
 	}
 	/*
@@ -602,7 +650,7 @@
 	while (buf_len > 0);
 
 	if (s->recvq)
-		set_active(c->conn, &rdfds);
+		set_active_read(c->conn);
 }
 
 struct sockaddr_in *
@@ -651,8 +699,6 @@
 	struct rlimit   rlimit;
 
 	memset(&hash_table, 0, sizeof(hash_table));
-	memset(&rdfds, 0, sizeof(rdfds));
-	memset(&wrfds, 0, sizeof(wrfds));
 	memset(&myaddr, 0, sizeof(myaddr));
 	memset(&port_free_map, 0xff, sizeof(port_free_map));
 
@@ -689,12 +735,15 @@
 		exit(1);
 	}
 
+	/* Disable the FD_SETSIZE check for now - perhaps only enforce on libevent select()? */
+#if 0
 	if (rlimit.rlim_max > FD_SETSIZE) {
 		fprintf(stderr, "%s: warning: open file limit > FD_SETSIZE; "
 			"limiting max. # of open files to FD_SETSIZE\n",
 			prog_name);
 		rlimit.rlim_max = FD_SETSIZE;
 	}
+#endif
 
 	rlimit.rlim_cur = rlimit.rlim_max;
 	if (setrlimit(RLIMIT_NOFILE, &rlimit) < 0) {
@@ -741,14 +790,12 @@
 					(reason ==
 					 SSL_ERROR_WANT_READ) ? "read" :
 					"write");
-			if (reason == SSL_ERROR_WANT_READ
-			    && !FD_ISSET(s->sd, &rdfds)) {
-				FD_CLR(s->sd, &wrfds);
-				set_active(s, &rdfds);
-			} else if (reason == SSL_ERROR_WANT_WRITE
-				   && !FD_ISSET(s->sd, &wrfds)) {
-				FD_CLR(s->sd, &rdfds);
-				set_active(s, &wrfds);
+			if (reason == SSL_ERROR_WANT_READ) {
+				conn_write_clear(s);
+				set_active_read(s);
+			} else if (reason == SSL_ERROR_WANT_WRITE) {
+				conn_read_clear(s);
+				set_active_write(s);
 			}
 			return;
 		}
@@ -859,6 +906,8 @@
 	}
 
 	s->sd = sd;
+	event_set(&s->ev_read, sd, EV_READ | EV_PERSIST, conn_handle_read_event, s);
+	event_set(&s->ev_write, sd, EV_WRITE | EV_PERSIST, conn_handle_write_event, s);
 	if (sd >= alloced_sd_to_conn) {
 		size_t          size, old_size;
 
@@ -939,7 +988,7 @@
 		 * connection establishment.  
 		 */
 		s->state = S_CONNECTING;
-		set_active(s, &wrfds);
+		set_active_write(s);
 		if (param.timeout > 0.0) {
 			arg.vp = s;
 			assert(!s->watchdog);
@@ -1040,7 +1089,7 @@
 			return -1;
 		call->timeout =
 		    param.timeout ? timer_now() + param.timeout : 0.0;
-		set_active(conn, &wrfds);
+		set_active_write(conn);
 	} else {
 		conn->sendq_tail->sendq_next = call;
 		conn->sendq_tail = call;
@@ -1097,8 +1146,8 @@
 	if (sd >= 0) {
 		close(sd);
 		sd_to_conn[sd] = 0;
-		FD_CLR(sd, &wrfds);
-		FD_CLR(sd, &rdfds);
+		conn_read_clear(conn);
+		conn_write_clear(conn);
 	}
 	if (conn->myport > 0)
 		port_put(conn->myport);
@@ -1111,124 +1160,66 @@
 	conn_dec_ref(conn);
 }
 
-void
-core_loop(void)
+static void
+conn_handle_read_event(int sd, short ev, void *a)
 {
-	int             is_readable, is_writable, n, sd, bit, min_i, max_i, i =
-	    0;
-	fd_set          readable, writable;
-	fd_mask         mask;
-	Any_Type        arg;
-	Conn           *conn;
+	Conn *conn = sd_to_conn[sd];
+	Any_Type arg;
 
-	while (running) {
-		struct timeval  tv = select_timeout;
+	conn_inc_ref(conn);
 
-		timer_tick();
+	if (conn->watchdog) {
+		timer_cancel(conn->watchdog);
+		conn->watchdog = 0;
+	}
 
-		readable = rdfds;
-		writable = wrfds;
-		min_i = min_sd / NFDBITS;
-		max_i = max_sd / NFDBITS;
+	if (conn->recvq)
+		do_recv(conn);
 
-		SYSCALL(SELECT,
-			n = select(max_sd + 1, &readable, &writable, 0, &tv));
+	conn_dec_ref(conn);
+}
 
-		++iteration;
+static void
+conn_handle_write_event(int sd, short ev, void *a)
+{
+	Conn *conn = sd_to_conn[sd];
+	Any_Type arg;
 
-		if (n <= 0) {
-			if (n < 0) {
-				fprintf(stderr,
-					"%s.core_loop: select failed: %s\n",
-					prog_name, strerror(errno));
-				exit(1);
-			}
-			continue;
-		}
+	conn_inc_ref(conn);
 
-		while (n > 0) {
-			/*
-			 * find the index of the fdmask that has something
-			 * going on: 
-			 */
-			do {
-				++i;
-				if (i > max_i)
-					i = min_i;
-
-				assert(i <= max_i);
-				mask =
-				    readable.fds_bits[i] | writable.
-				    fds_bits[i];
-			}
-			while (!mask);
-			bit = 0;
-			sd = i * NFDBITS + bit;
-			do {
-				if (mask & 1) {
-					--n;
-
-					is_readable =
-					    (FD_ISSET(sd, &readable)
-					     && FD_ISSET(sd, &rdfds));
-					is_writable = (FD_ISSET(sd, &writable)
-						       && FD_ISSET(sd,
-								   &wrfds));
-
-					if (is_readable || is_writable) {
-						/*
-						 * only handle sockets that
-						 * haven't timed out yet 
-						 */
-						conn = sd_to_conn[sd];
-
-						conn_inc_ref(conn);
-
-						if (conn->watchdog) {
-							timer_cancel(conn->
-								     watchdog);
-							conn->watchdog = 0;
-						}
-						if (conn->state ==
-						    S_CONNECTING) {
+	if (conn->watchdog) {
+		timer_cancel(conn->watchdog);
+		conn->watchdog = 0;
+	}
+
+	if (conn->state == S_CONNECTING) {
 #ifdef HAVE_SSL
-							if (param.use_ssl)
-								core_ssl_connect
-								    (conn);
-							else
+		if (param.use_ssl)
+			core_ssl_connect (conn);
+		else {
 #endif
-							if (is_writable) {
-								FD_CLR(sd,
-								       &wrfds);
-								conn->state =
-								    S_CONNECTED;
-								arg.l = 0;
-								event_signal
-								    (EV_CONN_CONNECTED,
-								     (Object *)
-								     conn,
-								     arg);
-							}
-						} else {
-							if (is_writable
-							    && conn->sendq)
-								do_send(conn);
-							if (is_readable
-							    && conn->recvq)
-								do_recv(conn);
-						}
-
-						conn_dec_ref(conn);
-
-						if (n > 0)
-							timer_tick();
-					}
-				}
-				mask = ((u_long) mask) >> 1;
-				++sd;
-			}
-			while (mask);
+			conn_write_clear(conn);
+			conn->state = S_CONNECTED;
+			arg.l = 0;
+			event_signal(EV_CONN_CONNECTED, (Object *) conn, arg);
 		}
+	} else if (conn->sendq)
+		do_send(conn);
+
+	conn_dec_ref(conn);
+}
+
+void
+core_loop(void)
+{
+	int             is_readable, is_writable, n, sd, bit, min_i, max_i, i =
+	    0;
+	fd_set          readable, writable;
+	fd_mask         mask;
+
+	while (running) {
+		timer_tick();
+		n = event_loop(EVLOOP_ONCE);
 	}
 }
 
Index: src/http.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/http.c,v
retrieving revision 1.8
diff -u -r1.8 http.c
--- src/http.c	16 Jul 2007 00:59:53 -0000	1.8
+++ src/http.c	14 Jul 2008 04:55:47 -0000
@@ -41,6 +41,9 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include <sys/types.h>
+#include <event.h>
+
 #include <generic_types.h>
 #include <object.h>
 #include <timer.h>
Index: src/httperf.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/httperf.c,v
retrieving revision 1.21
diff -u -r1.21 httperf.c
--- src/httperf.c	12 Apr 2008 03:07:05 -0000	1.21
+++ src/httperf.c	14 Jul 2008 04:55:47 -0000
@@ -66,6 +66,10 @@
 
 #include <sys/time.h>
 #include <sys/resource.h>
+#include <sys/types.h>
+
+/* LIBEVENT */
+#include <event.h>
 
 #include <generic_types.h>
 #include <object.h>
@@ -267,6 +271,9 @@
 	param.ssl_reuse = 1;
 #endif
 
+	/* setup event stuff early */
+	event_init();
+
 	/*
 	 * get program name: 
 	 */
Index: src/idleconn.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/idleconn.c,v
retrieving revision 1.13
diff -u -r1.13 idleconn.c
--- src/idleconn.c	5 Feb 2008 05:55:38 -0000	1.13
+++ src/idleconn.c	14 Jul 2008 04:55:47 -0000
@@ -42,6 +42,7 @@
 #include <inttypes.h>
 
 #include <sys/time.h>
+#include <sys/types.h>
 #include <event.h>
 #include <evdns.h>
 
Index: src/object.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/object.c,v
retrieving revision 1.9
diff -u -r1.9 object.c
--- src/object.c	16 Nov 2007 04:31:38 -0000	1.9
+++ src/object.c	14 Jul 2008 04:55:47 -0000
@@ -40,6 +40,9 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include <sys/types.h>
+#include <event.h>
+
 #include <generic_types.h>
 #include <object.h>
 #include <timer.h>
Index: src/gen/call_seq.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/gen/call_seq.c,v
retrieving revision 1.8
diff -u -r1.8 call_seq.c
--- src/gen/call_seq.c	16 Jul 2007 00:59:56 -0000	1.8
+++ src/gen/call_seq.c	14 Jul 2008 04:55:47 -0000
@@ -37,6 +37,8 @@
 #include "config.h"
 
 #include <assert.h>
+#include <sys/types.h>
+#include <event.h>
 
 #include <generic_types.h>
 #include <object.h>
Index: src/gen/conn_rate.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/gen/conn_rate.c,v
retrieving revision 1.7
diff -u -r1.7 conn_rate.c
--- src/gen/conn_rate.c	16 Jul 2007 00:59:57 -0000	1.7
+++ src/gen/conn_rate.c	14 Jul 2008 04:55:47 -0000
@@ -39,6 +39,8 @@
 
 #include <assert.h>
 #include <stdio.h>
+#include <sys/types.h>
+#include <event.h>
 
 #include <generic_types.h>
 #include <object.h>
Index: src/gen/sess_cookie.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/gen/sess_cookie.c,v
retrieving revision 1.8
diff -u -r1.8 sess_cookie.c
--- src/gen/sess_cookie.c	16 Jul 2007 00:59:57 -0000	1.8
+++ src/gen/sess_cookie.c	14 Jul 2008 04:55:47 -0000
@@ -49,6 +49,9 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include <sys/types.h>
+#include <event.h>
+
 #include <generic_types.h>
 #include <object.h>
 #include <timer.h>
Index: src/gen/session.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/gen/session.c,v
retrieving revision 1.7
diff -u -r1.7 session.c
--- src/gen/session.c	16 Jul 2007 00:59:57 -0000	1.7
+++ src/gen/session.c	14 Jul 2008 04:55:48 -0000
@@ -66,6 +66,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <event.h>
 
 #include <generic_types.h>
 #include <object.h>
Index: src/gen/uri_wlog.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/gen/uri_wlog.c,v
retrieving revision 1.9
diff -u -r1.9 uri_wlog.c
--- src/gen/uri_wlog.c	16 Nov 2007 03:52:29 -0000	1.9
+++ src/gen/uri_wlog.c	14 Jul 2008 04:55:48 -0000
@@ -75,7 +75,9 @@
 #include <string.h>
 #include <sys/mman.h>
 #include <sys/stat.h>
+#include <sys/types.h>
 #include <unistd.h>
+#include <event.h>
 
 #include <generic_types.h>
 #include <object.h>
Index: src/gen/wsess.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/gen/wsess.c,v
retrieving revision 1.8
diff -u -r1.8 wsess.c
--- src/gen/wsess.c	12 Sep 2007 04:47:27 -0000	1.8
+++ src/gen/wsess.c	14 Jul 2008 04:55:48 -0000
@@ -42,6 +42,8 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/types.h>
+#include <event.h>
 
 #include <generic_types.h>
 #include <object.h>
Index: src/gen/wsesslog.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/gen/wsesslog.c,v
retrieving revision 1.10
diff -u -r1.10 wsesslog.c
--- src/gen/wsesslog.c	12 Sep 2007 04:47:27 -0000	1.10
+++ src/gen/wsesslog.c	14 Jul 2008 04:55:48 -0000
@@ -81,6 +81,8 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/types.h>
+#include <event.h>
 
 #include <generic_types.h>
 #include <object.h>
Index: src/gen/wsesspage.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/gen/wsesspage.c,v
retrieving revision 1.8
diff -u -r1.8 wsesspage.c
--- src/gen/wsesspage.c	12 Sep 2007 04:47:27 -0000	1.8
+++ src/gen/wsesspage.c	14 Jul 2008 04:55:48 -0000
@@ -47,6 +47,8 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/types.h>
+#include <event.h>
 
 #include <generic_types.h>
 #include <object.h>
Index: src/stat/basic.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/stat/basic.c,v
retrieving revision 1.8
diff -u -r1.8 basic.c
--- src/stat/basic.c	16 Jul 2007 00:59:58 -0000	1.8
+++ src/stat/basic.c	14 Jul 2008 04:55:48 -0000
@@ -41,6 +41,8 @@
 #include <float.h>
 #include <stdio.h>
 #include <sys/resource.h>
+#include <sys/types.h>
+#include <event.h>
 
 #include <generic_types.h>
 #include <object.h>
Index: src/stat/print_reply.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/stat/print_reply.c,v
retrieving revision 1.8
diff -u -r1.8 print_reply.c
--- src/stat/print_reply.c	16 Jul 2007 00:59:58 -0000	1.8
+++ src/stat/print_reply.c	14 Jul 2008 04:55:48 -0000
@@ -42,6 +42,9 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include <sys/types.h>
+#include <event.h>
+
 #include <generic_types.h>
 #include <object.h>
 #include <timer.h>
Index: src/stat/sess_stat.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/stat/sess_stat.c,v
retrieving revision 1.8
diff -u -r1.8 sess_stat.c
--- src/stat/sess_stat.c	16 Jul 2007 00:59:58 -0000	1.8
+++ src/stat/sess_stat.c	14 Jul 2008 04:55:48 -0000
@@ -42,6 +42,9 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include <sys/types.h>
+#include <event.h>
+
 #include <generic_types.h>
 #include <object.h>
 #include <timer.h>



More information about the httperf mailing list