[httperf] httperf + libevent
Theodore Bullock
tbullock at canada.com
Sun Jul 13 22:46:38 PDT 2008
On Wed, 2008-07-02 at 23:39 +0800, Adrian Chadd wrote:
> I decided to give libevent'ifying httperf a go.
>
> http://code.google.com/p/httperf-adrian/
>
> Testing please!
Alright here is my feedback after playing with this for a couple of
hours. Below is a patch which merges cleanly with the current libevent
CVS sources.
Sure enough the performance gains are phenomenal.
Check out the following CPU times for 100 connections with 10 calls per
connection.
Before:
CPU time [s]: user 8.58 system 35.29 (user 19.2% system 78.9% total
98.1%)
After:
CPU time [s]: user 0.03 system 0.08 (user 0.1% system 0.2% total 0.2%)
* I want to see return results checked from functions that provide
them if at all reasonable/possible. This includes things like
event_add and the like. Now is the time to do this rather than
later.
* Unused variables like the ones in the revised core_loop should
be removed.
* The sys/types.h header should be brought in through the
generic_types.h header
* Where possible, lets migrate to the libevent timers so that we
can eventually remove our own timer code. Same goes for DNS
resolution.
* Eventually, remove the current core_loop function in lieu of a
persistent libevent loop.
That's it for now.
See your changes below:
-Ted
Index: src/conn.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/conn.c,v
retrieving revision 1.8
diff -u -r1.8 conn.c
--- src/conn.c 16 Jul 2007 04:44:47 -0000 1.8
+++ src/conn.c 14 Jul 2008 04:55:47 -0000
@@ -39,6 +39,9 @@
#include <stdlib.h>
#include <string.h>
+#include <sys/types.h>
+#include <event.h>
+
#include <generic_types.h>
#include <object.h>
#include <timer.h>
Index: src/conn.h
===================================================================
RCS file: /cvsroot/httperf/httperf/src/conn.h,v
retrieving revision 1.7
diff -u -r1.7 conn.h
--- src/conn.h 12 Sep 2007 04:47:27 -0000 1.7
+++ src/conn.h 14 Jul 2008 04:55:47 -0000
@@ -105,6 +105,7 @@
#ifdef HAVE_SSL
SSL *ssl; /* SSL connection info */
#endif
+ struct event ev_read, ev_write;
}
Conn;
Index: src/core.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/core.c,v
retrieving revision 1.13
diff -u -r1.13 core.c
--- src/core.c 18 Mar 2008 00:31:33 -0000 1.13
+++ src/core.c 14 Jul 2008 04:55:47 -0000
@@ -49,6 +49,7 @@
#ifdef HAVE_SYS_SELECT_H
#include <sys/select.h>
#endif
+#include <event.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
@@ -72,7 +73,6 @@
static int running = 1;
static int iteration;
static u_long max_burst_len;
-static fd_set rdfds, wrfds;
static int min_sd = 0x7fffffff, max_sd = 0, alloced_sd_to_conn = 0;
static struct timeval select_timeout;
static struct sockaddr_in myaddr;
@@ -134,6 +134,8 @@
while (errno == EINTR); \
}
#endif
+static void conn_handle_read_event(int sd, short ev, void *a);
+static void conn_handle_write_event(int sd, short ev, void *a);
struct hash_entry {
const char *hostname;
@@ -273,6 +275,34 @@
}
static void
+conn_read_set(Conn *s)
+{
+ if (s->sd > -1)
+ event_add(&s->ev_read, NULL);
+}
+
+static void
+conn_read_clear(Conn *s)
+{
+ if (s->sd > -1)
+ event_del(&s->ev_read);
+}
+
+static void
+conn_write_set(Conn *s)
+{
+ if (s->sd > -1)
+ event_add(&s->ev_write, NULL);
+}
+
+static void
+conn_write_clear(Conn *s)
+{
+ if (s->sd > -1)
+ event_del(&s->ev_write);
+}
+
+static void
conn_failure(Conn * s, int err)
{
Any_Type arg;
@@ -297,11 +327,9 @@
c = 0;
if (s->sd >= 0) {
now = timer_now();
- if (FD_ISSET(s->sd, &rdfds)
- && s->recvq && now >= s->recvq->timeout)
+ if (s->recvq && now >= s->recvq->timeout)
c = s->recvq;
- else if (FD_ISSET(s->sd, &wrfds)
- && s->sendq && now >= s->sendq->timeout)
+ else if (s->sendq && now >= s->sendq->timeout)
c = s->sendq;
}
if (DBG > 0) {
@@ -319,18 +347,12 @@
}
static void
-set_active(Conn * s, fd_set * fdset)
+set_active_shared(Conn * s)
{
int sd = s->sd;
Any_Type arg;
Time timeout;
- FD_SET(sd, fdset);
- if (sd < min_sd)
- min_sd = sd;
- if (sd >= max_sd)
- max_sd = sd;
-
if (s->watchdog)
return;
@@ -348,6 +370,32 @@
}
static void
+set_active_read(Conn *s)
+{
+ int sd = s->sd;
+
+ conn_read_set(s);
+ if (sd < min_sd)
+ min_sd = sd;
+ if (sd >= max_sd)
+ max_sd = sd;
+ set_active_shared(s);
+}
+
+static void
+set_active_write(Conn *s)
+{
+ int sd = s->sd;
+
+ conn_write_set(s);
+ if (sd < min_sd)
+ min_sd = sd;
+ if (sd >= max_sd)
+ max_sd = sd;
+ set_active_shared(s);
+}
+
+static void
do_send(Conn * conn)
{
int async_errno;
@@ -434,7 +482,7 @@
*/
call->timeout =
param.timeout ? timer_now() + param.timeout : 0.0;
- set_active(conn, &wrfds);
+ set_active_write(conn);
return;
}
@@ -444,7 +492,7 @@
conn->sendq = call->sendq_next;
if (!conn->sendq) {
conn->sendq_tail = 0;
- FD_CLR(sd, &wrfds);
+ conn_write_clear(conn);
}
arg.l = 0;
event_signal(EV_CALL_SEND_STOP, (Object *) call, arg);
@@ -468,7 +516,7 @@
call->timeout = param.timeout + param.think_timeout;
if (call->timeout > 0.0)
call->timeout += timer_now();
- set_active(conn, &rdfds);
+ set_active_read(conn);
if (conn->state < S_REPLY_STATUS)
conn->state = S_REPLY_STATUS; /* expecting reply
* status */
@@ -491,7 +539,7 @@
conn->recvq = call->recvq_next;
if (!conn->recvq) {
- FD_CLR(conn->sd, &rdfds);
+ conn_read_clear(conn);
conn->recvq_tail = 0;
}
/*
@@ -602,7 +650,7 @@
while (buf_len > 0);
if (s->recvq)
- set_active(c->conn, &rdfds);
+ set_active_read(c->conn);
}
struct sockaddr_in *
@@ -651,8 +699,6 @@
struct rlimit rlimit;
memset(&hash_table, 0, sizeof(hash_table));
- memset(&rdfds, 0, sizeof(rdfds));
- memset(&wrfds, 0, sizeof(wrfds));
memset(&myaddr, 0, sizeof(myaddr));
memset(&port_free_map, 0xff, sizeof(port_free_map));
@@ -689,12 +735,15 @@
exit(1);
}
+ /* Disable the FD_SETSIZE check for now - perhaps only enforce on libevent select()? */
+#if 0
if (rlimit.rlim_max > FD_SETSIZE) {
fprintf(stderr, "%s: warning: open file limit > FD_SETSIZE; "
"limiting max. # of open files to FD_SETSIZE\n",
prog_name);
rlimit.rlim_max = FD_SETSIZE;
}
+#endif
rlimit.rlim_cur = rlimit.rlim_max;
if (setrlimit(RLIMIT_NOFILE, &rlimit) < 0) {
@@ -741,14 +790,12 @@
(reason ==
SSL_ERROR_WANT_READ) ? "read" :
"write");
- if (reason == SSL_ERROR_WANT_READ
- && !FD_ISSET(s->sd, &rdfds)) {
- FD_CLR(s->sd, &wrfds);
- set_active(s, &rdfds);
- } else if (reason == SSL_ERROR_WANT_WRITE
- && !FD_ISSET(s->sd, &wrfds)) {
- FD_CLR(s->sd, &rdfds);
- set_active(s, &wrfds);
+ if (reason == SSL_ERROR_WANT_READ) {
+ conn_write_clear(s);
+ set_active_read(s);
+ } else if (reason == SSL_ERROR_WANT_WRITE) {
+ conn_read_clear(s);
+ set_active_write(s);
}
return;
}
@@ -859,6 +906,8 @@
}
s->sd = sd;
+ event_set(&s->ev_read, sd, EV_READ | EV_PERSIST, conn_handle_read_event, s);
+ event_set(&s->ev_write, sd, EV_WRITE | EV_PERSIST, conn_handle_write_event, s);
if (sd >= alloced_sd_to_conn) {
size_t size, old_size;
@@ -939,7 +988,7 @@
* connection establishment.
*/
s->state = S_CONNECTING;
- set_active(s, &wrfds);
+ set_active_write(s);
if (param.timeout > 0.0) {
arg.vp = s;
assert(!s->watchdog);
@@ -1040,7 +1089,7 @@
return -1;
call->timeout =
param.timeout ? timer_now() + param.timeout : 0.0;
- set_active(conn, &wrfds);
+ set_active_write(conn);
} else {
conn->sendq_tail->sendq_next = call;
conn->sendq_tail = call;
@@ -1097,8 +1146,8 @@
if (sd >= 0) {
close(sd);
sd_to_conn[sd] = 0;
- FD_CLR(sd, &wrfds);
- FD_CLR(sd, &rdfds);
+ conn_read_clear(conn);
+ conn_write_clear(conn);
}
if (conn->myport > 0)
port_put(conn->myport);
@@ -1111,124 +1160,66 @@
conn_dec_ref(conn);
}
-void
-core_loop(void)
+static void
+conn_handle_read_event(int sd, short ev, void *a)
{
- int is_readable, is_writable, n, sd, bit, min_i, max_i, i =
- 0;
- fd_set readable, writable;
- fd_mask mask;
- Any_Type arg;
- Conn *conn;
+ Conn *conn = sd_to_conn[sd];
+ Any_Type arg;
- while (running) {
- struct timeval tv = select_timeout;
+ conn_inc_ref(conn);
- timer_tick();
+ if (conn->watchdog) {
+ timer_cancel(conn->watchdog);
+ conn->watchdog = 0;
+ }
- readable = rdfds;
- writable = wrfds;
- min_i = min_sd / NFDBITS;
- max_i = max_sd / NFDBITS;
+ if (conn->recvq)
+ do_recv(conn);
- SYSCALL(SELECT,
- n = select(max_sd + 1, &readable, &writable, 0, &tv));
+ conn_dec_ref(conn);
+}
- ++iteration;
+static void
+conn_handle_write_event(int sd, short ev, void *a)
+{
+ Conn *conn = sd_to_conn[sd];
+ Any_Type arg;
- if (n <= 0) {
- if (n < 0) {
- fprintf(stderr,
- "%s.core_loop: select failed: %s\n",
- prog_name, strerror(errno));
- exit(1);
- }
- continue;
- }
+ conn_inc_ref(conn);
- while (n > 0) {
- /*
- * find the index of the fdmask that has something
- * going on:
- */
- do {
- ++i;
- if (i > max_i)
- i = min_i;
-
- assert(i <= max_i);
- mask =
- readable.fds_bits[i] | writable.
- fds_bits[i];
- }
- while (!mask);
- bit = 0;
- sd = i * NFDBITS + bit;
- do {
- if (mask & 1) {
- --n;
-
- is_readable =
- (FD_ISSET(sd, &readable)
- && FD_ISSET(sd, &rdfds));
- is_writable = (FD_ISSET(sd, &writable)
- && FD_ISSET(sd,
- &wrfds));
-
- if (is_readable || is_writable) {
- /*
- * only handle sockets that
- * haven't timed out yet
- */
- conn = sd_to_conn[sd];
-
- conn_inc_ref(conn);
-
- if (conn->watchdog) {
- timer_cancel(conn->
- watchdog);
- conn->watchdog = 0;
- }
- if (conn->state ==
- S_CONNECTING) {
+ if (conn->watchdog) {
+ timer_cancel(conn->watchdog);
+ conn->watchdog = 0;
+ }
+
+ if (conn->state == S_CONNECTING) {
#ifdef HAVE_SSL
- if (param.use_ssl)
- core_ssl_connect
- (conn);
- else
+ if (param.use_ssl)
+ core_ssl_connect (conn);
+ else {
#endif
- if (is_writable) {
- FD_CLR(sd,
- &wrfds);
- conn->state =
- S_CONNECTED;
- arg.l = 0;
- event_signal
- (EV_CONN_CONNECTED,
- (Object *)
- conn,
- arg);
- }
- } else {
- if (is_writable
- && conn->sendq)
- do_send(conn);
- if (is_readable
- && conn->recvq)
- do_recv(conn);
- }
-
- conn_dec_ref(conn);
-
- if (n > 0)
- timer_tick();
- }
- }
- mask = ((u_long) mask) >> 1;
- ++sd;
- }
- while (mask);
+ conn_write_clear(conn);
+ conn->state = S_CONNECTED;
+ arg.l = 0;
+ event_signal(EV_CONN_CONNECTED, (Object *) conn, arg);
}
+ } else if (conn->sendq)
+ do_send(conn);
+
+ conn_dec_ref(conn);
+}
+
+void
+core_loop(void)
+{
+ int is_readable, is_writable, n, sd, bit, min_i, max_i, i =
+ 0;
+ fd_set readable, writable;
+ fd_mask mask;
+
+ while (running) {
+ timer_tick();
+ n = event_loop(EVLOOP_ONCE);
}
}
Index: src/http.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/http.c,v
retrieving revision 1.8
diff -u -r1.8 http.c
--- src/http.c 16 Jul 2007 00:59:53 -0000 1.8
+++ src/http.c 14 Jul 2008 04:55:47 -0000
@@ -41,6 +41,9 @@
#include <stdlib.h>
#include <string.h>
+#include <sys/types.h>
+#include <event.h>
+
#include <generic_types.h>
#include <object.h>
#include <timer.h>
Index: src/httperf.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/httperf.c,v
retrieving revision 1.21
diff -u -r1.21 httperf.c
--- src/httperf.c 12 Apr 2008 03:07:05 -0000 1.21
+++ src/httperf.c 14 Jul 2008 04:55:47 -0000
@@ -66,6 +66,10 @@
#include <sys/time.h>
#include <sys/resource.h>
+#include <sys/types.h>
+
+/* LIBEVENT */
+#include <event.h>
#include <generic_types.h>
#include <object.h>
@@ -267,6 +271,9 @@
param.ssl_reuse = 1;
#endif
+ /* setup event stuff early */
+ event_init();
+
/*
* get program name:
*/
Index: src/idleconn.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/idleconn.c,v
retrieving revision 1.13
diff -u -r1.13 idleconn.c
--- src/idleconn.c 5 Feb 2008 05:55:38 -0000 1.13
+++ src/idleconn.c 14 Jul 2008 04:55:47 -0000
@@ -42,6 +42,7 @@
#include <inttypes.h>
#include <sys/time.h>
+#include <sys/types.h>
#include <event.h>
#include <evdns.h>
Index: src/object.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/object.c,v
retrieving revision 1.9
diff -u -r1.9 object.c
--- src/object.c 16 Nov 2007 04:31:38 -0000 1.9
+++ src/object.c 14 Jul 2008 04:55:47 -0000
@@ -40,6 +40,9 @@
#include <stdlib.h>
#include <string.h>
+#include <sys/types.h>
+#include <event.h>
+
#include <generic_types.h>
#include <object.h>
#include <timer.h>
Index: src/gen/call_seq.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/gen/call_seq.c,v
retrieving revision 1.8
diff -u -r1.8 call_seq.c
--- src/gen/call_seq.c 16 Jul 2007 00:59:56 -0000 1.8
+++ src/gen/call_seq.c 14 Jul 2008 04:55:47 -0000
@@ -37,6 +37,8 @@
#include "config.h"
#include <assert.h>
+#include <sys/types.h>
+#include <event.h>
#include <generic_types.h>
#include <object.h>
Index: src/gen/conn_rate.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/gen/conn_rate.c,v
retrieving revision 1.7
diff -u -r1.7 conn_rate.c
--- src/gen/conn_rate.c 16 Jul 2007 00:59:57 -0000 1.7
+++ src/gen/conn_rate.c 14 Jul 2008 04:55:47 -0000
@@ -39,6 +39,8 @@
#include <assert.h>
#include <stdio.h>
+#include <sys/types.h>
+#include <event.h>
#include <generic_types.h>
#include <object.h>
Index: src/gen/sess_cookie.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/gen/sess_cookie.c,v
retrieving revision 1.8
diff -u -r1.8 sess_cookie.c
--- src/gen/sess_cookie.c 16 Jul 2007 00:59:57 -0000 1.8
+++ src/gen/sess_cookie.c 14 Jul 2008 04:55:47 -0000
@@ -49,6 +49,9 @@
#include <stdlib.h>
#include <string.h>
+#include <sys/types.h>
+#include <event.h>
+
#include <generic_types.h>
#include <object.h>
#include <timer.h>
Index: src/gen/session.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/gen/session.c,v
retrieving revision 1.7
diff -u -r1.7 session.c
--- src/gen/session.c 16 Jul 2007 00:59:57 -0000 1.7
+++ src/gen/session.c 14 Jul 2008 04:55:48 -0000
@@ -66,6 +66,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <event.h>
#include <generic_types.h>
#include <object.h>
Index: src/gen/uri_wlog.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/gen/uri_wlog.c,v
retrieving revision 1.9
diff -u -r1.9 uri_wlog.c
--- src/gen/uri_wlog.c 16 Nov 2007 03:52:29 -0000 1.9
+++ src/gen/uri_wlog.c 14 Jul 2008 04:55:48 -0000
@@ -75,7 +75,9 @@
#include <string.h>
#include <sys/mman.h>
#include <sys/stat.h>
+#include <sys/types.h>
#include <unistd.h>
+#include <event.h>
#include <generic_types.h>
#include <object.h>
Index: src/gen/wsess.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/gen/wsess.c,v
retrieving revision 1.8
diff -u -r1.8 wsess.c
--- src/gen/wsess.c 12 Sep 2007 04:47:27 -0000 1.8
+++ src/gen/wsess.c 14 Jul 2008 04:55:48 -0000
@@ -42,6 +42,8 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <sys/types.h>
+#include <event.h>
#include <generic_types.h>
#include <object.h>
Index: src/gen/wsesslog.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/gen/wsesslog.c,v
retrieving revision 1.10
diff -u -r1.10 wsesslog.c
--- src/gen/wsesslog.c 12 Sep 2007 04:47:27 -0000 1.10
+++ src/gen/wsesslog.c 14 Jul 2008 04:55:48 -0000
@@ -81,6 +81,8 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <sys/types.h>
+#include <event.h>
#include <generic_types.h>
#include <object.h>
Index: src/gen/wsesspage.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/gen/wsesspage.c,v
retrieving revision 1.8
diff -u -r1.8 wsesspage.c
--- src/gen/wsesspage.c 12 Sep 2007 04:47:27 -0000 1.8
+++ src/gen/wsesspage.c 14 Jul 2008 04:55:48 -0000
@@ -47,6 +47,8 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <sys/types.h>
+#include <event.h>
#include <generic_types.h>
#include <object.h>
Index: src/stat/basic.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/stat/basic.c,v
retrieving revision 1.8
diff -u -r1.8 basic.c
--- src/stat/basic.c 16 Jul 2007 00:59:58 -0000 1.8
+++ src/stat/basic.c 14 Jul 2008 04:55:48 -0000
@@ -41,6 +41,8 @@
#include <float.h>
#include <stdio.h>
#include <sys/resource.h>
+#include <sys/types.h>
+#include <event.h>
#include <generic_types.h>
#include <object.h>
Index: src/stat/print_reply.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/stat/print_reply.c,v
retrieving revision 1.8
diff -u -r1.8 print_reply.c
--- src/stat/print_reply.c 16 Jul 2007 00:59:58 -0000 1.8
+++ src/stat/print_reply.c 14 Jul 2008 04:55:48 -0000
@@ -42,6 +42,9 @@
#include <stdlib.h>
#include <string.h>
+#include <sys/types.h>
+#include <event.h>
+
#include <generic_types.h>
#include <object.h>
#include <timer.h>
Index: src/stat/sess_stat.c
===================================================================
RCS file: /cvsroot/httperf/httperf/src/stat/sess_stat.c,v
retrieving revision 1.8
diff -u -r1.8 sess_stat.c
--- src/stat/sess_stat.c 16 Jul 2007 00:59:58 -0000 1.8
+++ src/stat/sess_stat.c 14 Jul 2008 04:55:48 -0000
@@ -42,6 +42,9 @@
#include <stdlib.h>
#include <string.h>
+#include <sys/types.h>
+#include <event.h>
+
#include <generic_types.h>
#include <object.h>
#include <timer.h>
More information about the httperf
mailing list