[httperf] reading request body and headers from a file

Steffen Kiefer steffen.kiefer at webde.de
Wed Jan 23 02:51:21 PST 2008


Hi,

i would like to contribute another feature for httperf.
Following patch allows to read request header and body from a given 
"request-file".

This works for command line as well as wsesslog mode.
The requestfile may contain only body or headers and body of a request.
In that case these are delimited by a blank line.

The requestfile can be passed via the command line option "--request-file".

In wsesslog mode you can specify the request-file(s) as follows:

/my/uri method=POST request-file='myrequestfile'
/my/2nduri method=PUT request-file='myrequestfile2'

I would be pleased if you added this patch to the source tree.

Please inform me, if there are further questions.
Thanks in advance

Steffen

-- 
Steffen Kiefer
Softwareentwickler

1 & 1 Internet AG
Brauerstraße 48  -   D-76135 Karlsruhe
Tel. +49-721-91374-4884  Fax +49-721-91374-2740
steffen.kiefer at webde.de  http://www.web.de/

Amtsgericht Montabaur HRB 6484
Vorstand: Henning Ahlert, Ralph Dommermuth, Matthias Ehrlich, Andreas 
Gauger, Thomas Gottschlich, Matthias Greve, Robert Hoffmann, Norbert 
Lang, Achim Weiss
Aufsichtsratsvorsitzender: Michael Scheeren
-------------- next part --------------
--- httperf-0.9.0_orig/src/gen/misc.c	2007-04-07 09:01:56.000000000 +0200
+++ httperf-0.9.0_patched/src/gen/misc.c	2008-01-22 13:15:05.000000000 +0100
@@ -52,11 +52,23 @@
 #include <call.h>
 #include <event.h>
 
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+#include <fcntl.h>
+#include <unistd.h>
+
 static const char *extra;
 static size_t extra_len;
 
 static size_t method_len;
 
+static size_t body_len;
+static const char *body;
+
+static const char *request_file_header;
+static size_t request_file_header_len;
+
 /* A simple module that collects cookies from the server responses and
    includes them in future calls to the server.  */
 
@@ -120,30 +132,110 @@
   if (method_len > 0)
     call_set_method (c, param.method, method_len);
 
+  if (body_len > 0) 
+  {
+    call_set_contents (c, body, body_len);
+    char content_length_header[50];
+    snprintf(content_length_header, 50, "Content-length: %d\r\n", body_len);
+    call_append_request_header (c, strdup(content_length_header), strlen(content_length_header));
+  }
+
   if (extra_len > 0)
     call_append_request_header (c, extra, extra_len);
+
+  if (request_file_header_len > 0)
+    call_append_request_header (c, request_file_header, request_file_header_len);
 }
 
+static void
+parse_request_header_and_body(const char* request_file)
+{
+  // read headers & body from request_file
+  int fd = open(request_file, O_RDONLY);
+  if (fd < 0)
+    panic ("%s: can't open given request file '%s'.\n", prog_name, request_file);
+
+  struct stat statbuf;
+  if (fstat(fd, &statbuf) < 0)
+    panic ("%s: can't determine filesize of file %s.\n", prog_name, request_file);
+
+  off_t filesize = statbuf.st_size;
+  body = (char*) malloc(filesize);
+  if (filesize > 0) 
+    body = mmap(0, filesize, PROT_READ, MAP_SHARED, fd, 0);
+
+  // parse & set request headers
+  int headerlen = 0;
+  int newblankline = 1;
+  char* headersbuf = malloc(filesize);
+  for (;;)
+  {
+    headerlen++;
+    if (*body == 0)
+    {
+      // no headers given
+      body = headersbuf;
+      request_file_header_len = 0;
+      body_len = filesize;
+      break;
+    }
+    else if (*body == '\n')
+    {
+      if (newblankline == 1) 
+      {
+        // finished parsing headers
+        body++;
+        headersbuf[headerlen] = '\0';
+        request_file_header_len = headerlen;
+        request_file_header = unescape (headersbuf, &request_file_header_len);
+        body_len = filesize - headerlen;
+        break;
+      }
+      else
+      {
+        newblankline = 1;
+      }
+    }
+    else if (isspace(*body) == 0)
+    {
+      newblankline = 0;
+    }
+    headersbuf[headerlen - 1] = *body;
+    body++;
+  }
+}
 
 static void
 init (void)
 {
   Any_Type arg;
 
+  if (param.request_file) 
+    parse_request_header_and_body(param.request_file);
+
   if (param.additional_header)
     extra = unescape (param.additional_header, &extra_len);
 
   if (param.method)
     method_len = strlen (param.method);
-
+ 
   arg.l = 0;
   event_register_handler (EV_CALL_NEW, call_created, arg);
 }
 
+static void
+stop (void)
+{
+  if (body_len > 0 && body != 0) 
+  {
+    munmap(body, body_len);
+  }
+}
+
 Load_Generator misc =
   {
     "Miscellaneous command line options",
     init,
     no_op,
-    no_op
+    stop 
   };
--- httperf-0.9.0_orig/src/gen/wsesslog.c	2007-04-07 09:01:56.000000000 +0200
+++ httperf-0.9.0_patched/src/gen/wsesslog.c	2008-01-22 11:14:04.000000000 +0100
@@ -87,6 +87,10 @@
 #include <rate.h>
 #include <session.h>
 #include <timer.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+#include <unistd.h>
 
 /* Maximum number of sessions that can be defined in the configuration
    file.  */
@@ -111,7 +115,9 @@
     int uri_len;
     char *contents;
     int contents_len;
-    char extra_hdrs[50];	/* plenty for "Content-length: 1234567890" */
+    char *request_file;
+    int request_filelen;
+    char extra_hdrs[50000];	
     int extra_hdrs_len;
   };
 
@@ -175,6 +181,7 @@
   sess = (Sess *) obj;
 
   priv = SESS_PRIVATE_DATA (sess);
+
   if (priv->timer)
     {
       timer_cancel (priv->timer);
@@ -388,6 +395,8 @@
   char method_str[1000];
   char this_arg[10000];
   char contents[10000];
+  char request_file[10000];
+
   double think_time;
   int bytes_read;
   REQ *reqptr;
@@ -561,7 +570,162 @@
 		    strlen (sptr->current_req->extra_hdrs);
 		}
 	    }
-	  else
+      else if (sscanf (this_arg, "request-file=%s", request_file) == 1)
+	    {
+	      /* this is tricky since request-file might be a quoted
+		 string with embedded spaces or escaped quotes.  We
+		 should parse this carefully from parsed_so_far */
+	      from = strchr (parsed_so_far, '=') + 1;
+	      to = request_file;
+	      single_quoted = FALSE;
+	      double_quoted = FALSE;
+	      escaped = FALSE;
+	      done = FALSE;
+	      while ((ch = *from++) != '\0' && !done)
+		{
+		  if (escaped == TRUE)
+		    {
+		      switch (ch)
+			{
+			case 'n':
+			  *to++ = '\n';
+			  break;
+			case 'r':
+			  *to++ = '\r';
+			  break;
+			case 't':
+			  *to++ = '\t';
+			  break;
+			case '\n':
+			  *to++ = '\n';
+			  /* this allows an escaped newline to
+			     continue the parsing to the next line. */
+			  if (fgets(line,sizeof(line),fp) == NULL)
+			    {
+			      lineno++;
+			      panic ("%s: premature EOF seen in '%s'\n",
+				     prog_name, param.wsesslog.file);  
+			    }
+			  parsed_so_far = from = line;
+			  break;
+			default:
+			  *to++ = ch;
+			  break;
+			}
+		      escaped = FALSE;
+		    }
+		  else if (ch == '"' && double_quoted)
+		    {
+		      double_quoted = FALSE;
+		    }
+		  else if (ch == '\'' && single_quoted)
+		    {
+		      single_quoted = FALSE;
+		    }
+		  else
+		    {
+		      switch (ch)
+			{
+			case '\t':
+			case '\n':
+			case ' ':
+			  if (single_quoted == FALSE &&
+			      double_quoted == FALSE)
+			    done = TRUE;	/* we are done */
+			  else
+			    *to++ = ch;
+			  break;
+			case '\\':		/* backslash */
+			  escaped = TRUE;
+			  break;
+			case '"':		/* double quote */
+			  if (single_quoted)
+			    *to++ = ch;
+			  else
+			    double_quoted = TRUE;
+			  break;
+			case '\'':		/* single quote */
+			  if (double_quoted)
+			    *to++ = ch;
+			  else
+			    single_quoted = TRUE;
+			  break;
+			default:
+			  *to++ = ch;
+			  break;
+			}
+		    }
+		}
+	      *to = '\0';
+	      from--;		/* back up 'from' to '\0' or white-space */
+	      bytes_read = from - parsed_so_far;
+  	      if (strlen (request_file) > 0)
+		  {
+            sptr->current_req->request_file = strdup (request_file);
+
+            // read headers & body from request_file
+            int fd = open(sptr->current_req->request_file, O_RDONLY);
+            if (fd < 0)
+              panic ("%s: can't open given request file '%s' from wsesslog.\n", prog_name, sptr->current_req->request_file);
+
+            struct stat statbuf;
+            if (fstat(fd, &statbuf) < 0)
+              panic ("%s: can't determine filesize of file %s.\n", prog_name, sptr->current_req->request_file);
+
+            off_t filesize = statbuf.st_size;
+            sptr->current_req->request_filelen = filesize;
+            sptr->current_req->contents = (char*) malloc(filesize);
+            if (filesize > 0) 
+              sptr->current_req->contents = mmap(0, filesize, PROT_READ, MAP_SHARED, fd, 0);
+
+            // parse & set request headers
+            int headerlen = 0;
+            int newblankline = 1;
+            char* headersbuf = malloc(filesize);
+            for (;;)
+            {
+              headerlen++;
+              if (*sptr->current_req->contents == 0) 
+              {
+                // no headers given
+                sptr->current_req->contents = headersbuf;
+                sptr->current_req->contents_len = filesize;
+                break;
+              }
+              else if (*sptr->current_req->contents == '\n')
+              {
+                if (newblankline == 1) 
+                {
+                  // finished parsing headers
+                  sptr->current_req->contents++;
+                  headersbuf[headerlen] = '\0';
+                  strncat(sptr->current_req->extra_hdrs, strdup(headersbuf), strlen(headersbuf));
+                  sptr->current_req->extra_hdrs_len = strlen(sptr->current_req->extra_hdrs);
+                  sptr->current_req->contents_len = filesize - headerlen;
+                  break;
+                }
+                else
+                {
+                  newblankline = 1;
+                }
+              }
+              else if (isspace(*sptr->current_req->contents) == 0)
+              {
+                newblankline = 0;
+              }
+              headersbuf[headerlen - 1] = *sptr->current_req->contents;
+              sptr->current_req->contents++;
+            }
+			    
+            char content_length_header[50];
+            snprintf(content_length_header, 50, "Content-length: %d\r\n", sptr->current_req->contents_len);
+            strncat(sptr->current_req->extra_hdrs,
+			    content_length_header, strlen(content_length_header));
+            sptr->current_req->extra_hdrs_len =
+		         strlen (sptr->current_req->extra_hdrs);
+          }
+        }
+      else
 	    {
 	      /* do not recognize this arg */
 	      panic ("%s: did not recognize arg '%s' in %s\n",
@@ -639,10 +803,36 @@
   rate_generator_start (&rg_sess, EV_SESS_DESTROYED);
 }
 
+static void
+stop (void)
+{
+  Sess_Private_Data *priv;
+  int i = 0;
+  for ( ; i < num_templates; i++) 
+  {
+    priv = &session_templates[i];
+    if (priv != 0) 
+    {
+      BURST* firstb = priv->current_burst;
+      BURST* currb; 
+      for (currb = firstb; currb != 0; currb = currb->next) 
+      {
+        REQ* firstreq = currb->req_list;
+        REQ* curreq;
+        for (curreq = firstreq; curreq != 0; curreq = curreq->next ) 
+        {
+          if (curreq->request_file != 0) 
+              munmap(curreq->contents, curreq->request_filelen);
+        }
+      }
+    }
+  }
+}
+
 Load_Generator wsesslog =
   {
     "creates log-based session workload",
     init,
     start,
-    no_op
+    stop 
   };
--- httperf-0.9.0_orig/src/httperf.c	2007-04-07 09:01:56.000000000 +0200
+++ httperf-0.9.0_patched/src/httperf.c	2008-01-21 14:59:54.000000000 +0100
@@ -126,6 +126,7 @@
     {"print-request",optional_argument, &param.print_request, 		0},
     {"rate",	     required_argument,	(int *) &param.rate,		0},
     {"recv-buffer",  required_argument, (int *) &param.recv_buffer_size, 0},
+    {"request-file",  required_argument, (int *) &param.request_file, 0},
     {"retry-on-failure", no_argument,	&param.retry_on_failure,	1},
     {"send-buffer",  required_argument, (int *) &param.send_buffer_size, 0},
     {"server",	     required_argument, (int *) &param.server,		0},
@@ -160,7 +161,7 @@
 	  "\t[--num-calls N] [--num-conns N] [--period [d|u|e]T1[,T2]]\n"
 	  "\t[--port N] "
 	  "[--print-reply [header|body]] [--print-request [header|body]]\n"
-	  "\t[--rate X] [--recv-buffer N] [--retry-on-failure] "
+	  "\t[--rate X] [--recv-buffer N] [--request-file file] [--retry-on-failure] "
 	  "[--send-buffer N]\n"
 	  "\t[--server S] [--server-name S] [--session-cookies]\n"
 #ifdef HAVE_SSL
@@ -533,6 +534,8 @@
 #endif
 	  else if (flag == &param.uri)
 	    param.uri = optarg;
+	  else if (flag == &param.request_file) 
+	    param.request_file = optarg;
 	  else if (flag == &param.think_timeout)
 	    {
 	      errno = 0;
@@ -682,7 +685,6 @@
 	      if (*end != ',')
 		goto bad_wsesslog_param;
 	      optarg = end + 1;
-
 	      /* simulate parsing of string */
 	      param.wsesslog.file = optarg;
 	      if ((end = strchr (optarg, ',')) == NULL)
@@ -836,7 +838,7 @@
       gen[num_gen++] = &sess_cookie;
     }
 
-  if (param.additional_header || param.method)
+  if (param.additional_header || param.method || param.request_file)
     gen[num_gen++] = &misc;
 
   /* echo command invocation for logging purposes: */
--- httperf-0.9.0_orig/src/httperf.h	2007-04-07 09:01:56.000000000 +0200
+++ httperf-0.9.0_patched/src/httperf.h	2008-01-22 11:21:53.000000000 +0100
@@ -136,6 +136,7 @@
 #endif
     const char *additional_header;	/* additional request header(s) */
     const char *method;	/* default call method */
+    const char *request_file;	/* name of the file where entries are */
     struct
       {
 	u_int id;


More information about the httperf mailing list