From: Andriy Senkovych Date: Wed, 9 Nov 2016 23:36:21 +0000 (+0000) Subject: Import webdis_0.1.2+dfsg.orig.tar.gz X-Git-Tag: archive/raspbian/0.1.4+dfsg-2+rpi1~1^2^2~7 X-Git-Url: https://dgit.raspbian.org/?a=commitdiff_plain;h=ab3d1401289f138d105f6b47ae8e2ec3f0d1187f;p=webdis.git Import webdis_0.1.2+dfsg.orig.tar.gz [dgit import orig webdis_0.1.2+dfsg.orig.tar.gz] --- ab3d1401289f138d105f6b47ae8e2ec3f0d1187f diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..67cccf6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +*.log +*.swp +*.o +webdis +websocket +*.png +pubsub diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..1d124ec --- /dev/null +++ b/.travis.yml @@ -0,0 +1,9 @@ +script: "make clean all" +language: c +compiler: + - gcc + - clang +before_install: + - sudo apt-get update + - sudo apt-get install libevent-dev +install: "sudo make install" diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..a9ae619 --- /dev/null +++ b/COPYING @@ -0,0 +1,24 @@ +Copyright (c) 2010-2011, Nicolas Favre-Felix + +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..432f4ef --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM tianon/debian:wheezy +MAINTAINER Nicolas Favre-Felix + +RUN apt-get -y --force-yes install wget make gcc libevent-dev +RUN apt-get -y --force-yes install redis-server +RUN wget --no-check-certificate https://github.com/nicolasff/webdis/archive/0.1.1.tar.gz -O webdis-0.1.1.tar.gz +RUN tar -xvzf webdis-0.1.1.tar.gz +RUN cd webdis-0.1.1 && make && make install && cd .. +RUN rm -rf webdis-0.1.1 webdis-0.1.1.tag.gz +RUN apt-get remove -y wget make gcc + +CMD /etc/init.d/redis-server start && /usr/local/bin/webdis /etc/webdis.prod.json && bash + +EXPOSE 7379 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..e29ad4b --- /dev/null +++ b/Makefile @@ -0,0 +1,66 @@ +OUT=webdis +HIREDIS_OBJ?=hiredis/hiredis.o hiredis/sds.o hiredis/net.o hiredis/async.o hiredis/read.o hiredis/dict.o +JANSSON_OBJ?=jansson/src/dump.o jansson/src/error.o jansson/src/hashtable.o jansson/src/load.o jansson/src/strbuffer.o jansson/src/utf.o jansson/src/value.o jansson/src/variadic.o +B64_OBJS?=b64/cencode.o +FORMAT_OBJS?=formats/json.o formats/raw.o formats/common.o formats/custom-type.o +HTTP_PARSER_OBJS?=http-parser/http_parser.o + +CFLAGS ?= -O0 -ggdb -Wall -Wextra -I. -Ijansson/src -Ihttp-parser +LDFLAGS ?= -levent -pthread + +# check for MessagePack +MSGPACK_LIB=$(shell ls /usr/lib/libmsgpack.so 2>/dev/null) +ifneq ($(strip $(MSGPACK_LIB)),) + FORMAT_OBJS += formats/msgpack.o + CFLAGS += -DMSGPACK=1 + LDFLAGS += -lmsgpack +endif + + +DEPS=$(FORMAT_OBJS) $(HIREDIS_OBJ) $(JANSSON_OBJ) $(HTTP_PARSER_OBJS) $(B64_OBJS) +OBJS=webdis.o cmd.o worker.o slog.o server.o acl.o md5/md5.o sha1/sha1.o http.o client.o websocket.o pool.o conf.o $(DEPS) + + + +PREFIX ?= /usr/local +CONFDIR ?= $(DESTDIR)/etc + +INSTALL_DIRS = $(DESTDIR) \ + $(DESTDIR)/$(PREFIX) \ + $(DESTDIR)/$(PREFIX)/bin \ + $(CONFDIR) + +all: $(OUT) Makefile + +$(OUT): $(OBJS) Makefile + $(CC) -o $(OUT) $(OBJS) $(LDFLAGS) + +%.o: %.c %.h Makefile + $(CC) -c $(CFLAGS) -o $@ $< + +%.o: %.c Makefile + $(CC) -c $(CFLAGS) -o $@ $< + +$(INSTALL_DIRS): + mkdir -p $@ + +clean: + rm -f $(OBJS) $(OUT) + +install: $(OUT) $(INSTALL_DIRS) + cp $(OUT) $(DESTDIR)/$(PREFIX)/bin + cp webdis.prod.json $(CONFDIR) + + +WEBDIS_PORT ?= 7379 + +test_all: test perftest + +test: + python tests/basic.py + python tests/limits.py + ./tests/pubsub -p $(WEBDIS_PORT) + +perftest: + # This is a performance test that requires apache2-utils and curl + ./tests/bench.sh diff --git a/README.markdown b/README.markdown new file mode 100644 index 0000000..54ad875 --- /dev/null +++ b/README.markdown @@ -0,0 +1,321 @@ +# About + +A very simple web server providing an HTTP interface to Redis. It uses [hiredis](https://github.com/antirez/hiredis), [jansson](https://github.com/akheron/jansson), [libevent](http://monkey.org/~provos/libevent/), and [http-parser](https://github.com/ry/http-parser/). + +Webdis depends on libevent-dev. You can install it on Ubuntu by typing `sudo apt-get install libevent-dev` or on OS X by typing `brew install libevent`. +
+make clean all
+
+./webdis &
+
+curl http://127.0.0.1:7379/SET/hello/world
+→ {"SET":[true,"OK"]}
+
+curl http://127.0.0.1:7379/GET/hello
+→ {"GET":"world"}
+
+curl -d "GET/hello" http://127.0.0.1:7379/
+→ {"GET":"world"}
+
+
+ +# Features +* `GET` and `POST` are supported, as well as `PUT` for file uploads. +* JSON output by default, optional JSONP parameter (`?jsonp=myFunction` or `?callback=myFunction`). +* Raw Redis 2.0 protocol output with `.raw` suffix +* MessagePack output with `.msg` suffix +* HTTP 1.1 pipelining (70,000 http requests per second on a desktop Linux machine.) +* Multi-threaded server, configurable number of worker threads. +* WebSocket support (Currently using the “hixie-76” specification). +* Connects to Redis using a TCP or UNIX socket. +* Restricted commands by IP range (CIDR subnet + mask) or HTTP Basic Auth, returning 403 errors. +* Possible Redis authentication in the config file. +* Pub/Sub using `Transfer-Encoding: chunked`, works with JSONP as well. Webdis can be used as a Comet server. +* Drop privileges on startup. +* Custom Content-Type using a pre-defined file extension, or with `?type=some/thing`. +* URL-encoded parameters for binary data or slashes and question marks. For instance, `%2f` is decoded as `/` but not used as a command separator. +* Logs, with a configurable verbosity. +* Cross-origin requests, usable with XMLHttpRequest2 (Cross-Origin Resource Sharing - CORS). +* File upload with PUT. +* With the JSON output, the return value of INFO is parsed and transformed into an object. +* Optional daemonize: set `"daemonize": true` and `"pidfile": "/var/run/webdis.pid"` in webdis.json. +* Default root object: Add `"default_root": "/GET/index.html"` in webdis.json to substitute the request to `/` with a Redis request. +* HTTP request limit with `http_max_request_size` (in bytes, set to 128MB by default). +* Database selection in the URL, using e.g. `/7/GET/key` to run the command on DB 7. + +# Ideas, TODO... +* Add better support for PUT, DELETE, HEAD, OPTIONS? How? For which commands? + * This could be done using a “strict mode” with a table of commands and the verbs that can/must be used with each command. Strict mode would be optional, configurable. How would webdis know of new commands remains to be determined. +* MULTI/EXEC/DISCARD/WATCH are disabled at the moment; find a way to use them. +* Support POST of raw Redis protocol data, and execute the whole thing. This could be useful for MULTI/EXEC transactions. +* Enrich config file: + * Provide timeout (maybe for some commands only?). What should the response be? 504 Gateway Timeout? 503 Service Unavailable? +* Multi-server support, using consistent hashing. +* SSL? + * Not sure if this is such a good idea. +* SPDY? + * SPDY is mostly useful for parallel fetches. Not sure if it would make sense for Webdis. +* Send your ideas using the github tracker, on twitter [@yowgi](http://twitter.com/yowgi) or by mail to n.favrefelix@gmail.com. + +# HTTP error codes +* Unknown HTTP verb: 405 Method Not Allowed. +* Redis is unreachable: 503 Service Unavailable. +* Matching ETag sent using `If-None-Match`: 304 Not Modified. +* Could also be used: + * Timeout on the redis side: 503 Service Unavailable. + * Missing key: 404 Not Found. + * Unauthorized command (disabled in config file): 403 Forbidden. + +# Command format +The URI `/COMMAND/arg0/arg1/.../argN.ext` executes the command on Redis and returns the response to the client. GET, POST, and PUT are supported: + +* `GET /COMMAND/arg0/.../argN.ext` +* `POST /` with `COMMAND/arg0/.../argN` in the HTTP body. +* `PUT /COMMAND/arg0.../argN-1` with `argN` in the HTTP body (see section on [file uploads](#file-upload).) + +`.ext` is an optional extension; it is not read as part of the last argument but only represents the output format. Several formats are available (see below). + +Special characters: `/` and `.` have special meanings, `/` separates arguments and `.` changes the Content-Type. They can be replaced by `%2f` and `%2e`, respectively. + +# ACL +Access control is configured in `webdis.json`. Each configuration tries to match a client profile according to two criterias: + +* [CIDR](http://en.wikipedia.org/wiki/CIDR) subnet + mask +* [HTTP Basic Auth](http://en.wikipedia.org/wiki/Basic_access_authentication) in the format of "user:password". + +Each ACL contains two lists of commands, `enabled` and `disabled`. All commands being enabled by default, it is up to the administrator to disable or re-enable them on a per-profile basis. +Examples: +
+{
+	"disabled":	["DEBUG", "FLUSHDB", "FLUSHALL"],
+},
+
+{
+	"http_basic_auth": "user:password",
+	"disabled":	["DEBUG", "FLUSHDB", "FLUSHALL"],
+	"enabled":	["SET"]
+},
+
+{
+	"ip": 		"192.168.10.0/24",
+	"enabled":	["SET"]
+},
+
+{
+	"http_basic_auth": "user:password",
+	"ip": 		"192.168.10.0/24",
+	"enabled":	["SET", "DEL"]
+}
+
+ACLs are interpreted in order, later authorizations superseding earlier ones if a client matches several. The special value "*" matches all commands. + +# JSON output +JSON is the default output format. Each command returns a JSON object with the command as a key and the result as a value. + +**Examples:** +
+// string
+$ curl http://127.0.0.1:7379/GET/y
+{"GET":"41"}
+
+// number
+$ curl http://127.0.0.1:7379/INCR/y
+{"INCR":42}
+
+// list
+$ curl http://127.0.0.1:7379/LRANGE/x/0/1
+{"LRANGE":["abc","def"]}
+
+// status
+$ curl http://127.0.0.1:7379/TYPE/y
+{"TYPE":[true,"string"]}
+
+// error, which is basically a status
+$ curl http://127.0.0.1:7379/MAKE-ME-COFFEE
+{"MAKE-ME-COFFEE":[false,"ERR unknown command 'MAKE-ME-COFFEE'"]}
+
+// JSONP callback:
+$ curl  "http://127.0.0.1:7379/TYPE/y?jsonp=myCustomFunction"
+myCustomFunction({"TYPE":[true,"string"]})
+
+ +# RAW output +This is the raw output of Redis; enable it with the `.raw` suffix. +
+
+// string
+$ curl http://127.0.0.1:7379/GET/z.raw
+$5
+hello
+
+// number
+curl http://127.0.0.1:7379/INCR/a.raw
+:2
+
+// list
+$ curl http://127.0.0.1:7379/LRANGE/x/0/-1.raw
+*2
+$3
+abc
+$3
+def
+
+// status
+$ curl http://127.0.0.1:7379/TYPE/y.raw
++zset
+
+// error, which is basically a status
+$ curl http://127.0.0.1:7379/MAKE-ME-COFFEE.raw
+-ERR unknown command 'MAKE-ME-COFFEE'
+
+ +# Custom content-type +Several content-types are available: + +* `.json` for `application/json` (this is the default Content-Type). +* `.msg` for `application/x-msgpack`. See [http://msgpack.org/](http://msgpack.org/) for the specs. +* `.txt` for `text/plain` +* `.html` for `text/html` +* `xhtml` for `application/xhtml+xml` +* `xml` for `text/xml` +* `.png` for `image/png` +* `jpg` or `jpeg` for `image/jpeg` +* Any other with the `?type=anything/youwant` query string. +* Add a custom separator for list responses with `?sep=,` query string. + +
+curl -v "http://127.0.0.1:7379/GET/hello.html"
+[...]
+< HTTP/1.1 200 OK
+< Content-Type: text/html
+< Date: Mon, 03 Jan 2011 20:43:36 GMT
+< Content-Length: 137
+<
+<!DOCTYPE html>
+<html>
+[...]
+</html>
+
+curl -v "http://127.0.0.1:7379/GET/hello.txt"
+[...]
+< HTTP/1.1 200 OK
+< Content-Type: text/plain
+< Date: Mon, 03 Jan 2011 20:43:36 GMT
+< Content-Length: 137
+[...]
+
+curl -v "http://127.0.0.1:7379/GET/big-file?type=application/pdf"
+[...]
+< HTTP/1.1 200 OK
+< Content-Type: application/pdf
+< Date: Mon, 03 Jan 2011 20:45:12 GMT
+[...]
+
+ +# File upload +Webdis supports file upload using HTTP PUT. The command URI is slightly different, as the last argument is taken from the HTTP body. +For example: instead of `/SET/key/value`, the URI becomes `/SET/key` and the value is the entirety of the body. This works for other commands such as LPUSH, etc. + +**Uploading a binary file to webdis**: +
+$ file redis-logo.png
+redis-logo.png: PNG image, 513 x 197, 8-bit/color RGBA, non-interlaced
+
+$ wc -c redis-logo.png
+16744 redis-logo.png
+
+$ curl -v --upload-file redis-logo.png http://127.0.0.1:7379/SET/logo
+[...]
+> PUT /SET/logo HTTP/1.1
+> User-Agent: curl/7.19.7 (x86_64-pc-linux-gnu) libcurl/7.19.7 OpenSSL/0.9.8k zlib/1.2.3.3 libidn/1.15
+> Host: 127.0.0.1:7379
+> Accept: */*
+> Content-Length: 16744
+> Expect: 100-continue
+>
+< HTTP/1.1 100 Continue
+< HTTP/1.1 200 OK
+< Content-Type: application/json
+< ETag: "0db1124cf79ffeb80aff6d199d5822f8"
+< Date: Sun, 09 Jan 2011 16:48:19 GMT
+< Content-Length: 19
+<
+{"SET":[true,"OK"]}
+
+$ curl -vs http://127.0.0.1:7379/GET/logo.png -o out.png
+> GET /GET/logo.png HTTP/1.1
+> User-Agent: curl/7.19.7 (x86_64-pc-linux-gnu) libcurl/7.19.7 OpenSSL/0.9.8k zlib/1.2.3.3 libidn/1.15
+> Host: 127.0.0.1:7379
+> Accept: */*
+>
+< HTTP/1.1 200 OK
+< Content-Type: image/png
+< ETag: "1991df597267d70bf9066a7d11969da0"
+< Date: Sun, 09 Jan 2011 16:50:51 GMT
+< Content-Length: 16744
+
+$ md5sum redis-logo.png out.png
+1991df597267d70bf9066a7d11969da0  redis-logo.png
+1991df597267d70bf9066a7d11969da0  out.png
+
+ +The file was uploaded and re-downloaded properly: it has the same hash and the content-type was set properly thanks to the `.png` extension. + +# WebSockets +Webdis supports WebSocket clients implementing [dixie-76](http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-76). +Web Sockets are supported with the following formats, selected by the connection URL: + +* JSON (on `/` or `/.json`) +* Raw Redis wire protocol (on `/.raw`) + +**Example**: +
+function testJSON() {
+	var jsonSocket = new WebSocket("ws://127.0.0.1:7379/.json");
+	jsonSocket.onopen = function() {
+
+		console.log("JSON socket connected!");
+		jsonSocket.send(JSON.stringify(["SET", "hello", "world"]));
+		jsonSocket.send(JSON.stringify(["GET", "hello"]));
+	};
+	jsonSocket.onmessage = function(messageEvent) {
+		console.log("JSON received:", messageEvent.data);
+	};
+}
+testJSON();
+
+ +This produces the following output: +
+JSON socket connected!
+JSON received: {"SET":[true,"OK"]}
+JSON received: {"GET":"world"}
+
+ +# Pub/Sub with chunked transfer encoding +Webdis exposes Redis PUB/SUB channels to HTTP clients, forwarding messages in the channel as they are published by Redis. This is done using chunked transfer encoding. + +**Example using XMLHttpRequest**: +
+var previous_response_length = 0
+xhr = new XMLHttpRequest()
+xhr.open("GET", "http://127.0.0.1:7379/SUBSCRIBE/hello", true);
+xhr.onreadystatechange = checkData;
+xhr.send(null);
+
+function checkData() {
+	if(xhr.readyState == 3)  {
+    	response = xhr.responseText;
+    	chunk = response.slice(previous_response_length);
+    	previous_response_length = response.length;
+    	console.log(chunk);
+    }
+};
+
+ +Publish messages to redis to see output similar to the following: +
+{"SUBSCRIBE":["subscribe","hello",1]}
+{"SUBSCRIBE":["message","hello","some message"]}
+{"SUBSCRIBE":["message","hello","some other message"]} 
+
diff --git a/acl.c b/acl.c new file mode 100644 index 0000000..160e6e5 --- /dev/null +++ b/acl.c @@ -0,0 +1,98 @@ +#include "acl.h" +#include "cmd.h" +#include "conf.h" +#include "http.h" +#include "client.h" + +#include +#include +#include +#include + +int +acl_match_client(struct acl *a, struct http_client *client, in_addr_t *ip) { + + /* check HTTP Basic Auth */ + const char *auth; + auth = client_get_header(client, "Authorization"); + if(a->http_basic_auth) { + if(auth && strncasecmp(auth, "Basic ", 6) == 0) { /* sent auth */ + if(strcmp(auth + 6, a->http_basic_auth) != 0) { /* bad password */ + return 0; + } + } else { /* no auth sent, required to match this ACL */ + return 0; + } + } + + /* CIDR check. */ + if(a->cidr.enabled == 0) { /* none given, all match */ + return 1; + } + if(((*ip) & a->cidr.mask) == (a->cidr.subnet & a->cidr.mask)) { + return 1; + } + + return 0; +} + +int +acl_allow_command(struct cmd *cmd, struct conf *cfg, struct http_client *client) { + + char *always_off[] = {"MULTI", "EXEC", "WATCH", "DISCARD", "SELECT"}; + + unsigned int i; + int authorized = 1; + struct acl *a; + + in_addr_t client_addr; + + const char *cmd_name; + size_t cmd_len; + + if(cmd->count == 0) { + return 0; + } + + cmd_name = cmd->argv[0]; + cmd_len = cmd->argv_len[0]; + + /* some commands are always disabled, regardless of the config file. */ + for(i = 0; i < sizeof(always_off) / sizeof(always_off[0]); ++i) { + if(strncasecmp(always_off[i], cmd_name, cmd_len) == 0) { + return 0; + } + } + + /* find client's address */ + client_addr = ntohl(client->addr); + + /* go through permissions */ + for(a = cfg->perms; a; a = a->next) { + + if(!acl_match_client(a, client, &client_addr)) continue; /* match client */ + + /* go through authorized commands */ + for(i = 0; i < a->enabled.count; ++i) { + if(strncasecmp(a->enabled.commands[i], cmd_name, cmd_len) == 0) { + authorized = 1; + } + if(strncasecmp(a->enabled.commands[i], "*", 1) == 0) { + authorized = 1; + } + } + + /* go through unauthorized commands */ + for(i = 0; i < a->disabled.count; ++i) { + if(strncasecmp(a->disabled.commands[i], cmd_name, cmd_len) == 0) { + authorized = 0; + } + if(strncasecmp(a->disabled.commands[i], "*", 1) == 0) { + authorized = 0; + } + } + } + + return authorized; +} + diff --git a/acl.h b/acl.h new file mode 100644 index 0000000..2b68b93 --- /dev/null +++ b/acl.h @@ -0,0 +1,39 @@ +#ifndef ACL_H +#define ACL_H + +#include + +struct http_client; +struct cmd; +struct conf; + +struct acl_commands { + unsigned int count; + char **commands; +}; + +struct acl { + + /* CIDR subnet + mask */ + struct { + int enabled; + in_addr_t subnet; + in_addr_t mask; + } cidr; + + char *http_basic_auth; + + /* commands that have been enabled or disabled */ + struct acl_commands enabled; + struct acl_commands disabled; + + struct acl *next; +}; + +int +acl_match_client(struct acl *a, struct http_client *client, in_addr_t *ip); + +int +acl_allow_command(struct cmd *cmd, struct conf *cfg, struct http_client *client); + +#endif diff --git a/client.c b/client.c new file mode 100644 index 0000000..4543cff --- /dev/null +++ b/client.c @@ -0,0 +1,372 @@ +#include "client.h" +#include "http_parser.h" +#include "http.h" +#include "server.h" +#include "worker.h" +#include "websocket.h" +#include "cmd.h" +#include "conf.h" + +#include +#include +#include +#include +#include +#include + +#define CHECK_ALLOC(c, ptr) if(!(ptr)) { c->failed_alloc = 1; return -1;} + +static int +http_client_on_url(struct http_parser *p, const char *at, size_t sz) { + + struct http_client *c = p->data; + + CHECK_ALLOC(c, c->path = realloc(c->path, c->path_sz + sz + 1)); + memcpy(c->path + c->path_sz, at, sz); + c->path_sz += sz; + c->path[c->path_sz] = 0; + + return 0; +} + +/* + * Called when the body is parsed. + */ +static int +http_client_on_body(struct http_parser *p, const char *at, size_t sz) { + + struct http_client *c = p->data; + return http_client_add_to_body(c, at, sz); +} + +int +http_client_add_to_body(struct http_client *c, const char *at, size_t sz) { + + CHECK_ALLOC(c, c->body = realloc(c->body, c->body_sz + sz + 1)); + memcpy(c->body + c->body_sz, at, sz); + c->body_sz += sz; + c->body[c->body_sz] = 0; + + return 0; +} + +static int +http_client_on_header_name(struct http_parser *p, const char *at, size_t sz) { + + struct http_client *c = p->data; + size_t n = c->header_count; + + /* if we're not adding to the same header name as last time, realloc to add one field. */ + if(c->last_cb != LAST_CB_KEY) { + n = ++c->header_count; + CHECK_ALLOC(c, c->headers = realloc(c->headers, n * sizeof(struct http_header))); + memset(&c->headers[n-1], 0, sizeof(struct http_header)); + } + + /* Add data to the current header name. */ + CHECK_ALLOC(c, c->headers[n-1].key = realloc(c->headers[n-1].key, + c->headers[n-1].key_sz + sz + 1)); + memcpy(c->headers[n-1].key + c->headers[n-1].key_sz, at, sz); + c->headers[n-1].key_sz += sz; + c->headers[n-1].key[c->headers[n-1].key_sz] = 0; + + c->last_cb = LAST_CB_KEY; + + return 0; +} + +static char * +wrap_filename(const char *val, size_t val_len) { + + char format[] = "attachment; filename=\""; + size_t sz = sizeof(format) - 1 + val_len + 1; + char *p = calloc(sz + 1, 1); + + memcpy(p, format, sizeof(format)-1); /* copy format */ + memcpy(p + sizeof(format)-1, val, val_len); /* copy filename */ + p[sz-1] = '"'; + + return p; +} + +/* + * Split query string into key/value pairs, process some of them. + */ +static int +http_client_on_query_string(struct http_parser *parser, const char *at, size_t sz) { + + struct http_client *c = parser->data; + const char *p = at; + + while(p < at + sz) { + + const char *key = p, *val; + int key_len, val_len; + char *eq = memchr(key, '=', sz - (p-at)); + if(!eq || eq > at + sz) { /* last argument */ + break; + } else { /* found an '=' */ + char *amp; + val = eq + 1; + key_len = eq - key; + p = eq + 1; + + amp = memchr(p, '&', sz - (p-at)); + if(!amp || amp > at + sz) { + val_len = at + sz - p; /* last arg */ + } else { + val_len = amp - val; /* cur arg */ + p = amp + 1; + } + + if(key_len == 4 && strncmp(key, "type", 4) == 0) { + c->type = calloc(1 + val_len, 1); + memcpy(c->type, val, val_len); + } else if((key_len == 5 && strncmp(key, "jsonp", 5) == 0) + || (key_len == 8 && strncmp(key, "callback", 8) == 0)) { + c->jsonp = calloc(1 + val_len, 1); + memcpy(c->jsonp, val, val_len); + } else if(key_len == 3 && strncmp(key, "sep", 3) == 0) { + c->separator = calloc(1 + val_len, 1); + memcpy(c->separator, val, val_len); + } else if(key_len == 8 && strncmp(key, "filename", 8) == 0) { + c->filename = wrap_filename(val, val_len); + } + + if(!amp) { + break; + } + } + } + return 0; +} + +static int +http_client_on_header_value(struct http_parser *p, const char *at, size_t sz) { + + struct http_client *c = p->data; + size_t n = c->header_count; + + /* Add data to the current header value. */ + CHECK_ALLOC(c, c->headers[n-1].val = realloc(c->headers[n-1].val, + c->headers[n-1].val_sz + sz + 1)); + memcpy(c->headers[n-1].val + c->headers[n-1].val_sz, at, sz); + c->headers[n-1].val_sz += sz; + c->headers[n-1].val[c->headers[n-1].val_sz] = 0; + + c->last_cb = LAST_CB_VAL; + + + /* react to some values. */ + if(strncmp("Expect", c->headers[n-1].key, c->headers[n-1].key_sz) == 0) { + if(sz == 12 && strncasecmp(at, "100-continue", sz) == 0) { + /* support HTTP file upload */ + char http100[] = "HTTP/1.1 100 Continue\r\n\r\n"; + int ret = write(c->fd, http100, sizeof(http100)-1); + (void)ret; + } + } else if(strncasecmp("Connection", c->headers[n-1].key, c->headers[n-1].key_sz) == 0) { + if(sz == 10 && strncasecmp(at, "Keep-Alive", sz) == 0) { + c->keep_alive = 1; + } + } + + return 0; +} + +static int +http_client_on_message_complete(struct http_parser *p) { + + struct http_client *c = p->data; + + /* keep-alive detection */ + if (c->parser.flags & F_CONNECTION_CLOSE) { + c->keep_alive = 0; + } else if(c->parser.http_major == 1 && c->parser.http_minor == 1) { /* 1.1 */ + c->keep_alive = 1; + } + c->http_version = c->parser.http_minor; + + if(p->upgrade && c->w->s->cfg->websockets) { /* WebSocket, don't execute just yet */ + c->is_websocket = 1; + return 0; + } + + /* handle default root object */ + if(c->path_sz == 1 && *c->path == '/' && c->w->s->cfg->default_root) { /* replace */ + free(c->path); + c->path = strdup(c->w->s->cfg->default_root); + c->path_sz = strlen(c->path); + } + + + worker_process_client(c); + http_client_reset(c); + + return 0; +} + +struct http_client * +http_client_new(struct worker *w, int fd, in_addr_t addr) { + + struct http_client *c = calloc(1, sizeof(struct http_client)); + + c->fd = fd; + c->w = w; + c->addr = addr; + c->s = w->s; + + /* parser */ + http_parser_init(&c->parser, HTTP_REQUEST); + c->parser.data = c; + + /* callbacks */ + c->settings.on_url = http_client_on_url; + c->settings.on_query_string = http_client_on_query_string; + c->settings.on_body = http_client_on_body; + c->settings.on_message_complete = http_client_on_message_complete; + c->settings.on_header_field = http_client_on_header_name; + c->settings.on_header_value = http_client_on_header_value; + + c->last_cb = LAST_CB_NONE; + + return c; +} + + +void +http_client_reset(struct http_client *c) { + + int i; + + /* headers */ + for(i = 0; i < c->header_count; ++i) { + free(c->headers[i].key); + free(c->headers[i].val); + } + free(c->headers); + c->headers = NULL; + c->header_count = 0; + + /* other data */ + free(c->body); c->body = NULL; + c->body_sz = 0; + free(c->path); c->path = NULL; + c->path_sz = 0; + free(c->type); c->type = NULL; + free(c->jsonp); c->jsonp = NULL; + free(c->filename); c->filename = NULL; + c->request_sz = 0; + + /* no last known header callback */ + c->last_cb = LAST_CB_NONE; + + /* mark as broken if client doesn't support Keep-Alive. */ + if(c->keep_alive == 0) { + c->broken = 1; + } +} + +void +http_client_free(struct http_client *c) { + + http_client_reset(c); + free(c->buffer); + free(c); +} + +int +http_client_read(struct http_client *c) { + + char buffer[4096]; + int ret; + + ret = read(c->fd, buffer, sizeof(buffer)); + if(ret <= 0) { + /* broken link, free buffer and client object */ + + /* disconnect pub/sub client if there is one. */ + if(c->pub_sub && c->pub_sub->ac) { + struct cmd *cmd = c->pub_sub; + + /* disconnect from all channels */ + redisAsyncDisconnect(c->pub_sub->ac); + if(c->pub_sub) c->pub_sub->ac = NULL; + c->pub_sub = NULL; + + /* delete command object */ + cmd_free(cmd); + } + + close(c->fd); + + http_client_free(c); + return (int)CLIENT_DISCONNECTED; + } + + /* save what we've just read */ + c->buffer = realloc(c->buffer, c->sz + ret); + if(!c->buffer) { + return (int)CLIENT_OOM; + } + memcpy(c->buffer + c->sz, buffer, ret); + c->sz += ret; + + /* keep track of total sent */ + c->request_sz += ret; + + return ret; +} + +int +http_client_remove_data(struct http_client *c, size_t sz) { + + char *buffer; + if(c->sz < sz) + return -1; + + /* replace buffer */ + CHECK_ALLOC(c, buffer = malloc(c->sz - sz)); + memcpy(buffer, c->buffer + sz, c->sz - sz); + free(c->buffer); + c->buffer = buffer; + c->sz -= sz; + + return 0; +} + +int +http_client_execute(struct http_client *c) { + + int nparsed = http_parser_execute(&c->parser, &c->settings, c->buffer, c->sz); + + if(!c->is_websocket) { + /* removed consumed data, all has been copied already. */ + free(c->buffer); + c->buffer = NULL; + c->sz = 0; + } + return nparsed; +} + +/* + * Find header value, returns NULL if not found. + */ +const char * +client_get_header(struct http_client *c, const char *key) { + + int i; + size_t sz = strlen(key); + + for(i = 0; i < c->header_count; ++i) { + + if(sz == c->headers[i].key_sz && + strncasecmp(key, c->headers[i].key, sz) == 0) { + return c->headers[i].val; + } + + } + + return NULL; +} + diff --git a/client.h b/client.h new file mode 100644 index 0000000..72594ef --- /dev/null +++ b/client.h @@ -0,0 +1,93 @@ +#ifndef CLIENT_H +#define CLIENT_H + +#include +#include +#include "http_parser.h" +#include "websocket.h" + +struct http_header; +struct server; +struct cmd; + +typedef enum { + LAST_CB_NONE = 0, + LAST_CB_KEY = 1, + LAST_CB_VAL = 2} last_cb_t; + +typedef enum { + CLIENT_DISCONNECTED = -1, + CLIENT_OOM = -2} client_error_t; + +struct http_client { + + int fd; + in_addr_t addr; + struct event ev; + + struct worker *w; + struct server *s; + + + /* HTTP parsing */ + struct http_parser parser; + struct http_parser_settings settings; + char *buffer; + size_t sz; + size_t request_sz; /* accumulated so far. */ + last_cb_t last_cb; + + /* various flags. */ + char keep_alive; + char broken; + char is_websocket; + char http_version; + char failed_alloc; + + /* HTTP data */ + char *path; + size_t path_sz; + + /* headers */ + struct http_header *headers; + int header_count; + + char *body; + size_t body_sz; + + char *type; /* forced output content-type */ + char *jsonp; /* jsonp wrapper */ + char *separator; /* list separator for raw lists */ + char *filename; /* content-disposition */ + + struct cmd *pub_sub; + + struct ws_msg *frame; /* websocket frame */ +}; + +struct http_client * +http_client_new(struct worker *w, int fd, in_addr_t addr); + +void +http_client_reset(struct http_client *c); + +void +http_client_free(struct http_client *c); + +int +http_client_read(struct http_client *c); + +int +http_client_remove_data(struct http_client *c, size_t sz); + +int +http_client_execute(struct http_client *c); + +int +http_client_add_to_body(struct http_client *c, const char *at, size_t sz); + +const char * +client_get_header(struct http_client *c, const char *key); + + +#endif diff --git a/cmd.c b/cmd.c new file mode 100644 index 0000000..1c661ca --- /dev/null +++ b/cmd.c @@ -0,0 +1,379 @@ +#include "cmd.h" +#include "conf.h" +#include "acl.h" +#include "client.h" +#include "pool.h" +#include "worker.h" +#include "http.h" +#include "server.h" +#include "slog.h" + +#include "formats/json.h" +#include "formats/raw.h" +#ifdef MSGPACK +#include "formats/msgpack.h" +#endif +#include "formats/custom-type.h" + +#include +#include +#include +#include +#include + +struct cmd * +cmd_new(int count) { + + struct cmd *c = calloc(1, sizeof(struct cmd)); + + c->count = count; + + c->argv = calloc(count, sizeof(char*)); + c->argv_len = calloc(count, sizeof(size_t)); + + return c; +} + + +void +cmd_free(struct cmd *c) { + + int i; + if(!c) return; + + for(i = 0; i < c->count; ++i) { + free((char*)c->argv[i]); + } + + free(c->jsonp); + free(c->separator); + free(c->if_none_match); + if(c->mime_free) free(c->mime); + + if (c->ac && /* we have a connection */ + (c->database != c->w->s->cfg->database /* custom DB */ + || cmd_is_subscribe(c))) { + pool_free_context(c->ac); + } + free(c->argv); + free(c->argv_len); + + free(c); +} + +/* taken from libevent */ +static char * +decode_uri(const char *uri, size_t length, size_t *out_len, int always_decode_plus) { + char c; + size_t i, j; + int in_query = always_decode_plus; + + char *ret = malloc(length); + + for (i = j = 0; i < length; i++) { + c = uri[i]; + if (c == '?') { + in_query = 1; + } else if (c == '+' && in_query) { + c = ' '; + } else if (c == '%' && isxdigit((unsigned char)uri[i+1]) && + isxdigit((unsigned char)uri[i+2])) { + char tmp[] = { uri[i+1], uri[i+2], '\0' }; + c = (char)strtol(tmp, NULL, 16); + i += 2; + } + ret[j++] = c; + } + *out_len = (size_t)j; + + return ret; +} + +/* setup headers */ +void +cmd_setup(struct cmd *cmd, struct http_client *client) { + + int i; + cmd->keep_alive = client->keep_alive; + cmd->w = client->w; /* keep track of the worker */ + + for(i = 0; i < client->header_count; ++i) { + if(strcasecmp(client->headers[i].key, "If-None-Match") == 0) { + cmd->if_none_match = calloc(1+client->headers[i].val_sz, 1); + memcpy(cmd->if_none_match, client->headers[i].val, + client->headers[i].val_sz); + } else if(strcasecmp(client->headers[i].key, "Connection") == 0 && + strcasecmp(client->headers[i].val, "Keep-Alive") == 0) { + cmd->keep_alive = 1; + } + } + + if(client->type) { /* transfer pointer ownership */ + cmd->mime = client->type; + cmd->mime_free = 1; + client->type = NULL; + } + + if(client->jsonp) { /* transfer pointer ownership */ + cmd->jsonp = client->jsonp; + client->jsonp = NULL; + } + + if(client->separator) { /* transfer pointer ownership */ + cmd->separator = client->separator; + client->separator = NULL; + } + + if(client->filename) { /* transfer pointer ownership */ + cmd->filename = client->filename; + client->filename = NULL; + } + + cmd->fd = client->fd; + cmd->http_version = client->http_version; +} + + +cmd_response_t +cmd_run(struct worker *w, struct http_client *client, + const char *uri, size_t uri_len, + const char *body, size_t body_len) { + + char *qmark = memchr(uri, '?', uri_len); + char *slash; + const char *p, *cmd_name = uri; + int cmd_len; + int param_count = 0, cur_param = 1; + + struct cmd *cmd; + formatting_fun f_format; + + /* count arguments */ + if(qmark) { + uri_len = qmark - uri; + } + for(p = uri; p && p < uri + uri_len; param_count++) { + p = memchr(p+1, '/', uri_len - (p+1-uri)); + } + + if(body && body_len) { /* PUT request */ + param_count++; + } + if(param_count == 0) { + return CMD_PARAM_ERROR; + } + + cmd = cmd_new(param_count); + cmd->fd = client->fd; + cmd->database = w->s->cfg->database; + + /* get output formatting function */ + uri_len = cmd_select_format(client, cmd, uri, uri_len, &f_format); + + /* add HTTP info */ + cmd_setup(cmd, client); + + /* check if we only have one command or more. */ + slash = memchr(uri, '/', uri_len); + if(slash) { + + /* detect DB number by checking if first arg is only numbers */ + int has_db = 1; + int db_num = 0; + for(p = uri; p < slash; ++p) { + if(*p < '0' || *p > '9') { + has_db = 0; + break; + } + db_num = db_num * 10 + (*p - '0'); + } + + /* shift to next arg if a db was set up */ + if(has_db) { + char *next; + cmd->database = db_num; + cmd->count--; /* overcounted earlier */ + cmd_name = slash + 1; + + if((next = memchr(cmd_name, '/', uri_len - (slash - uri)))) { + cmd_len = next - cmd_name; + } else { + cmd_len = uri_len - (slash - uri + 1); + } + } else { + cmd_len = slash - uri; + } + } else { + cmd_len = uri_len; + } + + /* there is always a first parameter, it's the command name */ + cmd->argv[0] = malloc(cmd_len); + memcpy(cmd->argv[0], cmd_name, cmd_len); + cmd->argv_len[0] = cmd_len; + + /* check that the client is able to run this command */ + if(!acl_allow_command(cmd, w->s->cfg, client)) { + cmd_free(cmd); + return CMD_ACL_FAIL; + } + + if(cmd_is_subscribe(cmd)) { + /* create a new connection to Redis */ + cmd->ac = (redisAsyncContext*)pool_connect(w->pool, cmd->database, 0); + + /* register with the client, used upon disconnection */ + client->pub_sub = cmd; + cmd->pub_sub_client = client; + } else if(cmd->database != w->s->cfg->database) { + /* create a new connection to Redis for custom DBs */ + cmd->ac = (redisAsyncContext*)pool_connect(w->pool, cmd->database, 0); + } else { + /* get a connection from the pool */ + cmd->ac = (redisAsyncContext*)pool_get_context(w->pool); + } + + /* no args (e.g. INFO command) */ + if(!slash) { + if(!cmd->ac) { + cmd_free(cmd); + return CMD_REDIS_UNAVAIL; + } + redisAsyncCommandArgv(cmd->ac, f_format, cmd, 1, + (const char **)cmd->argv, cmd->argv_len); + return CMD_SENT; + } + p = cmd_name + cmd_len + 1; + while(p < uri + uri_len) { + + const char *arg = p; + int arg_len; + char *next = memchr(arg, '/', uri_len - (arg-uri)); + if(!next || next > uri + uri_len) { /* last argument */ + p = uri + uri_len; + arg_len = p - arg; + } else { /* found a slash */ + arg_len = next - arg; + p = next + 1; + } + + /* record argument */ + cmd->argv[cur_param] = decode_uri(arg, arg_len, &cmd->argv_len[cur_param], 1); + cur_param++; + } + + if(body && body_len) { /* PUT request */ + cmd->argv[cur_param] = malloc(body_len); + memcpy(cmd->argv[cur_param], body, body_len); + cmd->argv_len[cur_param] = body_len; + } + + /* send it off! */ + if(cmd->ac) { + cmd_send(cmd, f_format); + return CMD_SENT; + } + /* failed to find a suitable connection to Redis. */ + cmd_free(cmd); + client->pub_sub = NULL; + return CMD_REDIS_UNAVAIL; +} + +void +cmd_send(struct cmd *cmd, formatting_fun f_format) { + redisAsyncCommandArgv(cmd->ac, f_format, cmd, cmd->count, + (const char **)cmd->argv, cmd->argv_len); +} + +/** + * Select Content-Type and processing function. + */ +int +cmd_select_format(struct http_client *client, struct cmd *cmd, + const char *uri, size_t uri_len, formatting_fun *f_format) { + + const char *ext; + int ext_len = -1; + unsigned int i; + int found = 0; /* did we match it to a predefined format? */ + + /* those are the available reply formats */ + struct reply_format { + const char *s; + size_t sz; + formatting_fun f; + const char *ct; + }; + struct reply_format funs[] = { + {.s = "json", .sz = 4, .f = json_reply, .ct = "application/json"}, + {.s = "raw", .sz = 3, .f = raw_reply, .ct = "binary/octet-stream"}, + +#ifdef MSGPACK + {.s = "msg", .sz = 3, .f = msgpack_reply, .ct = "application/x-msgpack"}, +#endif + + {.s = "bin", .sz = 3, .f = custom_type_reply, .ct = "binary/octet-stream"}, + {.s = "txt", .sz = 3, .f = custom_type_reply, .ct = "text/plain"}, + {.s = "html", .sz = 4, .f = custom_type_reply, .ct = "text/html"}, + {.s = "xhtml", .sz = 5, .f = custom_type_reply, .ct = "application/xhtml+xml"}, + {.s = "xml", .sz = 3, .f = custom_type_reply, .ct = "text/xml"}, + + {.s = "png", .sz = 3, .f = custom_type_reply, .ct = "image/png"}, + {.s = "jpg", .sz = 3, .f = custom_type_reply, .ct = "image/jpeg"}, + {.s = "jpeg", .sz = 4, .f = custom_type_reply, .ct = "image/jpeg"}, + + {.s = "js", .sz = 2, .f = json_reply, .ct = "application/javascript"}, + {.s = "css", .sz = 3, .f = custom_type_reply, .ct = "text/css"}, + }; + + /* default */ + *f_format = json_reply; + + /* find extension */ + for(ext = uri + uri_len - 1; ext != uri && *ext != '/'; --ext) { + if(*ext == '.') { + ext++; + ext_len = uri + uri_len - ext; + break; + } + } + if(!ext_len) return uri_len; /* nothing found */ + + /* find function for the given extension */ + for(i = 0; i < sizeof(funs)/sizeof(funs[0]); ++i) { + if(ext_len == (int)funs[i].sz && strncmp(ext, funs[i].s, ext_len) == 0) { + + if(cmd->mime_free) free(cmd->mime); + cmd->mime = (char*)funs[i].ct; + cmd->mime_free = 0; + + *f_format = funs[i].f; + found = 1; + } + } + + /* the user can force it with ?type=some/thing */ + if(client->type) { + *f_format = custom_type_reply; + cmd->mime = strdup(client->type); + cmd->mime_free = 1; + } + + if(found) { + return uri_len - ext_len - 1; + } else { + /* no matching format, use default output with the full argument, extension included. */ + return uri_len; + } +} + +int +cmd_is_subscribe(struct cmd *cmd) { + + if(cmd->count >= 1 && cmd->argv[0] && + (strncasecmp(cmd->argv[0], "SUBSCRIBE", cmd->argv_len[0]) == 0 || + strncasecmp(cmd->argv[0], "PSUBSCRIBE", cmd->argv_len[0]) == 0)) { + return 1; + } + return 0; +} diff --git a/cmd.h b/cmd.h new file mode 100644 index 0000000..e216bde --- /dev/null +++ b/cmd.h @@ -0,0 +1,80 @@ +#ifndef CMD_H +#define CMD_H + +#include +#include +#include +#include +#include + +struct evhttp_request; +struct http_client; +struct server; +struct worker; +struct cmd; + +typedef void (*formatting_fun)(redisAsyncContext *, void *, void *); +typedef enum {CMD_SENT, + CMD_PARAM_ERROR, + CMD_ACL_FAIL, + CMD_REDIS_UNAVAIL} cmd_response_t; + +struct cmd { + int fd; + + int count; + char **argv; + size_t *argv_len; + + /* HTTP data */ + char *mime; /* forced output content-type */ + int mime_free; /* need to free mime buffer */ + + char *filename; /* content-disposition attachment */ + + char *if_none_match; /* used with ETags */ + char *jsonp; /* jsonp wrapper */ + char *separator; /* list separator for raw lists */ + int keep_alive; + + /* various flags */ + int started_responding; + int is_websocket; + int http_version; + int database; + + struct http_client *pub_sub_client; + redisAsyncContext *ac; + struct worker *w; +}; + +struct subscription { + struct server *s; + struct cmd *cmd; +}; + +struct cmd * +cmd_new(int count); + +void +cmd_free(struct cmd *c); + +cmd_response_t +cmd_run(struct worker *w, struct http_client *client, + const char *uri, size_t uri_len, + const char *body, size_t body_len); + +int +cmd_select_format(struct http_client *client, struct cmd *cmd, + const char *uri, size_t uri_len, formatting_fun *f_format); + +int +cmd_is_subscribe(struct cmd *cmd); + +void +cmd_send(struct cmd *cmd, formatting_fun f_format); + +void +cmd_setup(struct cmd *cmd, struct http_client *client); + +#endif diff --git a/conf.c b/conf.c new file mode 100644 index 0000000..4fce300 --- /dev/null +++ b/conf.c @@ -0,0 +1,233 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include "conf.h" +#include "acl.h" + +static struct acl * +conf_parse_acls(json_t *jtab); + +struct conf * +conf_read(const char *filename) { + + json_t *j; + json_error_t error; + struct conf *conf; + void *kv; + + /* defaults */ + conf = calloc(1, sizeof(struct conf)); + conf->redis_host = strdup("127.0.0.1"); + conf->redis_port = 6379; + conf->http_host = strdup("0.0.0.0"); + conf->http_port = 7379; + conf->http_max_request_size = 128*1024*1024; + conf->http_threads = 4; + conf->user = getuid(); + conf->group = getgid(); + conf->logfile = "webdis.log"; + conf->verbosity = WEBDIS_NOTICE; + conf->daemonize = 0; + conf->pidfile = "webdis.pid"; + conf->database = 0; + conf->pool_size_per_thread = 2; + + j = json_load_file(filename, 0, &error); + if(!j) { + fprintf(stderr, "Error: %s (line %d)\n", error.text, error.line); + return conf; + } + + for(kv = json_object_iter(j); kv; kv = json_object_iter_next(j, kv)) { + json_t *jtmp = json_object_iter_value(kv); + + if(strcmp(json_object_iter_key(kv), "redis_host") == 0 && json_typeof(jtmp) == JSON_STRING) { + free(conf->redis_host); + conf->redis_host = strdup(json_string_value(jtmp)); + } else if(strcmp(json_object_iter_key(kv), "redis_port") == 0 && json_typeof(jtmp) == JSON_INTEGER) { + conf->redis_port = (short)json_integer_value(jtmp); + } else if(strcmp(json_object_iter_key(kv), "redis_auth") == 0 && json_typeof(jtmp) == JSON_STRING) { + conf->redis_auth = strdup(json_string_value(jtmp)); + } else if(strcmp(json_object_iter_key(kv), "http_host") == 0 && json_typeof(jtmp) == JSON_STRING) { + free(conf->http_host); + conf->http_host = strdup(json_string_value(jtmp)); + } else if(strcmp(json_object_iter_key(kv), "http_port") == 0 && json_typeof(jtmp) == JSON_INTEGER) { + conf->http_port = (short)json_integer_value(jtmp); + } else if(strcmp(json_object_iter_key(kv), "http_max_request_size") == 0 && json_typeof(jtmp) == JSON_INTEGER) { + conf->http_max_request_size = (size_t)json_integer_value(jtmp); + } else if(strcmp(json_object_iter_key(kv), "threads") == 0 && json_typeof(jtmp) == JSON_INTEGER) { + conf->http_threads = (short)json_integer_value(jtmp); + } else if(strcmp(json_object_iter_key(kv), "acl") == 0 && json_typeof(jtmp) == JSON_ARRAY) { + conf->perms = conf_parse_acls(jtmp); + } else if(strcmp(json_object_iter_key(kv), "user") == 0 && json_typeof(jtmp) == JSON_STRING) { + struct passwd *u; + if((u = getpwnam(json_string_value(jtmp)))) { + conf->user = u->pw_uid; + } + } else if(strcmp(json_object_iter_key(kv), "group") == 0 && json_typeof(jtmp) == JSON_STRING) { + struct group *g; + if((g = getgrnam(json_string_value(jtmp)))) { + conf->group = g->gr_gid; + } + } else if(strcmp(json_object_iter_key(kv),"logfile") == 0 && json_typeof(jtmp) == JSON_STRING){ + conf->logfile = strdup(json_string_value(jtmp)); + } else if(strcmp(json_object_iter_key(kv),"verbosity") == 0 && json_typeof(jtmp) == JSON_INTEGER){ + int tmp = json_integer_value(jtmp); + if(tmp < 0) conf->verbosity = WEBDIS_ERROR; + else if(tmp > (int)WEBDIS_DEBUG) conf->verbosity = WEBDIS_DEBUG; + else conf->verbosity = (log_level)tmp; + } else if(strcmp(json_object_iter_key(kv), "daemonize") == 0 && json_typeof(jtmp) == JSON_TRUE) { + conf->daemonize = 1; + } else if(strcmp(json_object_iter_key(kv),"pidfile") == 0 && json_typeof(jtmp) == JSON_STRING){ + conf->pidfile = strdup(json_string_value(jtmp)); + } else if(strcmp(json_object_iter_key(kv), "websockets") == 0 && json_typeof(jtmp) == JSON_TRUE) { + conf->websockets = 1; + } else if(strcmp(json_object_iter_key(kv), "database") == 0 && json_typeof(jtmp) == JSON_INTEGER) { + conf->database = json_integer_value(jtmp); + } else if(strcmp(json_object_iter_key(kv), "pool_size") == 0 && json_typeof(jtmp) == JSON_INTEGER) { + conf->pool_size_per_thread = json_integer_value(jtmp); + } else if(strcmp(json_object_iter_key(kv), "default_root") == 0 && json_typeof(jtmp) == JSON_STRING) { + conf->default_root = strdup(json_string_value(jtmp)); + } + } + + json_decref(j); + + return conf; +} + +void +acl_read_commands(json_t *jlist, struct acl_commands *ac) { + + unsigned int i, n, cur; + + /* count strings in the array */ + for(i = 0, n = 0; i < json_array_size(jlist); ++i) { + json_t *jelem = json_array_get(jlist, (size_t)i); + if(json_typeof(jelem) == JSON_STRING) { + n++; + } + } + + /* allocate block */ + ac->commands = calloc((size_t)n, sizeof(char*)); + ac->count = n; + + /* add all disabled commands */ + for(i = 0, cur = 0; i < json_array_size(jlist); ++i) { + json_t *jelem = json_array_get(jlist, i); + if(json_typeof(jelem) == JSON_STRING) { + size_t sz; + const char *s = json_string_value(jelem); + sz = strlen(s); + + ac->commands[cur] = calloc(1 + sz, 1); + memcpy(ac->commands[cur], s, sz); + cur++; + } + } +} + +struct acl * +conf_parse_acl(json_t *j) { + + json_t *jcidr, *jbasic, *jlist; + unsigned short mask_bits = 0; + + struct acl *a = calloc(1, sizeof(struct acl)); + + /* parse CIDR */ + if((jcidr = json_object_get(j, "ip")) && json_typeof(jcidr) == JSON_STRING) { + const char *s; + char *p, *ip; + + s = json_string_value(jcidr); + p = strchr(s, '/'); + if(!p) { + ip = strdup(s); + } else { + ip = calloc((size_t)(p - s + 1), 1); + memcpy(ip, s, (size_t)(p - s)); + mask_bits = (unsigned short)atoi(p+1); + } + a->cidr.enabled = 1; + a->cidr.mask = (mask_bits == 0 ? 0xffffffff : (0xffffffff << (32 - mask_bits))); + a->cidr.subnet = ntohl(inet_addr(ip)) & a->cidr.mask; + free(ip); + } + + /* parse basic_auth */ + if((jbasic = json_object_get(j, "http_basic_auth")) && json_typeof(jbasic) == JSON_STRING) { + + /* base64 encode */ + base64_encodestate b64; + int pos; + char *p; + const char *plain = json_string_value(jbasic); + size_t len, plain_len = strlen(plain) + 0; + len = (plain_len + 8) * 8 / 6; + a->http_basic_auth = calloc(len, 1); + + base64_init_encodestate(&b64); + pos = base64_encode_block(plain, (int)plain_len, a->http_basic_auth, &b64); /* FIXME: check return value */ + base64_encode_blockend(a->http_basic_auth + pos, &b64); + + /* end string with \0 rather than \n */ + if((p = strchr(a->http_basic_auth + pos, '\n'))) { + *p = 0; + } + } + + /* parse enabled commands */ + if((jlist = json_object_get(j, "enabled")) && json_typeof(jlist) == JSON_ARRAY) { + acl_read_commands(jlist, &a->enabled); + } + + /* parse disabled commands */ + if((jlist = json_object_get(j, "disabled")) && json_typeof(jlist) == JSON_ARRAY) { + acl_read_commands(jlist, &a->disabled); + } + + return a; +} + +struct acl * +conf_parse_acls(json_t *jtab) { + + struct acl *head = NULL, *tail = NULL, *tmp; + + unsigned int i; + for(i = 0; i < json_array_size(jtab); ++i) { + json_t *val = json_array_get(jtab, i); + + tmp = conf_parse_acl(val); + if(head == NULL && tail == NULL) { + head = tail = tmp; + } else { + tail->next = tmp; + tail = tmp; + } + } + + return head; +} + +void +conf_free(struct conf *conf) { + + free(conf->redis_host); + free(conf->redis_auth); + + free(conf->http_host); + + free(conf); +} diff --git a/conf.h b/conf.h new file mode 100644 index 0000000..28ba115 --- /dev/null +++ b/conf.h @@ -0,0 +1,54 @@ +#ifndef CONF_H +#define CONF_H + +#include +#include "slog.h" + +struct conf { + + /* connection to Redis */ + char *redis_host; + short redis_port; + char *redis_auth; + + /* HTTP server interface */ + char *http_host; + short http_port; + short http_threads; + size_t http_max_request_size; + + /* pool size, one pool per worker thread */ + int pool_size_per_thread; + + /* daemonize process, off by default */ + int daemonize; + char *pidfile; + + /* WebSocket support, off by default */ + int websockets; + + /* database number */ + int database; + + /* ACL */ + struct acl *perms; + + /* user/group */ + uid_t user; + gid_t group; + + /* Logging */ + char *logfile; + log_level verbosity; + + /* Request to serve on “/” */ + char *default_root; +}; + +struct conf * +conf_read(const char *filename); + +void +conf_free(struct conf *conf); + +#endif /* CONF_H */ diff --git a/formats/common.c b/formats/common.c new file mode 100644 index 0000000..ed9f041 --- /dev/null +++ b/formats/common.c @@ -0,0 +1,141 @@ +#include "common.h" +#include "cmd.h" +#include "http.h" +#include "client.h" +#include "websocket.h" + +#include "md5/md5.h" +#include +#include + +/* TODO: replace this with a faster hash function? */ +char *etag_new(const char *p, size_t sz) { + + md5_byte_t buf[16]; + char *etag = calloc(34 + 1, 1); + int i; + + if(!etag) + return NULL; + + md5_state_t pms; + + md5_init(&pms); + md5_append(&pms, (const md5_byte_t *)p, (int)sz); + md5_finish(&pms, buf); + + for(i = 0; i < 16; ++i) { + sprintf(etag + 1 + 2*i, "%.2x", (unsigned char)buf[i]); + } + + etag[0] = '"'; + etag[33] = '"'; + + return etag; +} + +void +format_send_error(struct cmd *cmd, short code, const char *msg) { + + struct http_response *resp; + + if(!cmd->is_websocket && !cmd->pub_sub_client) { + resp = http_response_init(cmd->w, code, msg); + resp->http_version = cmd->http_version; + http_response_set_keep_alive(resp, cmd->keep_alive); + http_response_write(resp, cmd->fd); + } + + /* for pub/sub, remove command from client */ + if(cmd->pub_sub_client) { + cmd->pub_sub_client->pub_sub = NULL; + } else { + cmd_free(cmd); + } +} + +void +format_send_reply(struct cmd *cmd, const char *p, size_t sz, const char *content_type) { + + int free_cmd = 1; + const char *ct = cmd->mime?cmd->mime:content_type; + struct http_response *resp; + + if(cmd->is_websocket) { + ws_reply(cmd, p, sz); + + /* If it's a subscribe command, there'll be more responses */ + if(!cmd_is_subscribe(cmd)) + cmd_free(cmd); + return; + } + + if(cmd_is_subscribe(cmd)) { + free_cmd = 0; + + /* start streaming */ + if(cmd->started_responding == 0) { + cmd->started_responding = 1; + resp = http_response_init(cmd->w, 200, "OK"); + resp->http_version = cmd->http_version; + if(cmd->filename) { + http_response_set_header(resp, "Content-Disposition", cmd->filename); + } + http_response_set_header(resp, "Content-Type", ct); + http_response_set_keep_alive(resp, 1); + http_response_set_header(resp, "Transfer-Encoding", "chunked"); + http_response_set_body(resp, p, sz); + http_response_write(resp, cmd->fd); + } else { + /* Asynchronous chunk write. */ + http_response_write_chunk(cmd->fd, cmd->w, p, sz); + } + + } else { + /* compute ETag */ + char *etag = etag_new(p, sz); + + if(etag) { + /* check If-None-Match */ + if(cmd->if_none_match && strcmp(cmd->if_none_match, etag) == 0) { + /* SAME! send 304. */ + resp = http_response_init(cmd->w, 304, "Not Modified"); + } else { + resp = http_response_init(cmd->w, 200, "OK"); + if(cmd->filename) { + http_response_set_header(resp, "Content-Disposition", cmd->filename); + } + http_response_set_header(resp, "Content-Type", ct); + http_response_set_header(resp, "ETag", etag); + http_response_set_body(resp, p, sz); + } + resp->http_version = cmd->http_version; + http_response_set_keep_alive(resp, cmd->keep_alive); + http_response_write(resp, cmd->fd); + free(etag); + } else { + format_send_error(cmd, 503, "Service Unavailable"); + } + } + + /* cleanup */ + if(free_cmd) { + cmd_free(cmd); + } +} + +int +integer_length(long long int i) { + int sz = 0; + int ci = llabs(i); + while (ci > 0) { + ci = (ci/10); + sz += 1; + } + if(i == 0) { /* log 0 doesn't make sense. */ + sz = 1; + } else if(i < 0) { /* allow for neg sign as well. */ + sz++; + } + return sz; +} diff --git a/formats/common.h b/formats/common.h new file mode 100644 index 0000000..b4bfeb5 --- /dev/null +++ b/formats/common.h @@ -0,0 +1,18 @@ +#ifndef FORMATS_COMMON_H +#define FORMATS_COMMON_H + +#include + +struct cmd; + +void +format_send_reply(struct cmd *cmd, + const char *p, size_t sz, + const char *content_type); + +void +format_send_error(struct cmd *cmd, short code, const char *msg); +int +integer_length(long long int i); + +#endif diff --git a/formats/custom-type.c b/formats/custom-type.c new file mode 100644 index 0000000..3fc6e08 --- /dev/null +++ b/formats/custom-type.c @@ -0,0 +1,118 @@ +#include "custom-type.h" +#include "cmd.h" +#include "common.h" +#include "http.h" + +#include +#include +#include + +static char * +custom_array(struct cmd *cmd, const redisReply *r, size_t *sz); + +void +custom_type_reply(redisAsyncContext *c, void *r, void *privdata) { + + redisReply *reply = r; + struct cmd *cmd = privdata; + (void)c; + char int_buffer[50]; + char *status_buf; + int int_len; + struct http_response *resp; + size_t sz; + char *array_out; + + + if (reply == NULL) { /* broken Redis link */ + format_send_error(cmd, 503, "Service Unavailable"); + return; + } + + if(cmd->mime) { /* use the given content-type, but only for strings */ + switch(reply->type) { + + case REDIS_REPLY_NIL: /* or nil values */ + format_send_error(cmd, 404, "Not found"); + return; + + case REDIS_REPLY_STRING: + format_send_reply(cmd, reply->str, reply->len, cmd->mime); + return; + + case REDIS_REPLY_STATUS: + case REDIS_REPLY_ERROR: + status_buf = calloc(1 + reply->len, 1); + status_buf[0] = (reply->type == REDIS_REPLY_STATUS ? '+' : '-'); + memcpy(status_buf + 1, reply->str, reply->len); + format_send_reply(cmd, status_buf, 1 + reply->len, cmd->mime); + free(status_buf); + return; + + case REDIS_REPLY_INTEGER: + int_len = sprintf(int_buffer, "%lld", reply->integer); + format_send_reply(cmd, int_buffer, int_len, cmd->mime); + return; + case REDIS_REPLY_ARRAY: + array_out = custom_array(cmd, r, &sz); + format_send_reply(cmd, array_out, sz, cmd->mime); + free(array_out); + return; + } + } + + /* couldn't make sense of what the client wanted. */ + resp = http_response_init(cmd->w, 400, "Bad Request"); + http_response_set_header(resp, "Content-Length", "0"); + http_response_set_keep_alive(resp, cmd->keep_alive); + http_response_write(resp, cmd->fd); + + if(!cmd_is_subscribe(cmd)) { + cmd_free(cmd); + } +} + +static char * +custom_array(struct cmd *cmd, const redisReply *r, size_t *sz) { + + unsigned int i; + char *ret, *p; + size_t sep_len = 0; + + if(cmd->separator) + sep_len = strlen(cmd->separator); + + /* compute size */ + *sz = 0; + for(i = 0; i < r->elements; ++i) { + redisReply *e = r->element[i]; + switch(e->type) { + case REDIS_REPLY_STRING: + if(sep_len && i != 0) + *sz += sep_len; + *sz += e->len; + break; + + } + } + + /* allocate */ + p = ret = malloc(*sz); + + /* copy */ + for(i = 0; i < r->elements; ++i) { + redisReply *e = r->element[i]; + switch(e->type) { + case REDIS_REPLY_STRING: + if(sep_len && i != 0) { + memcpy(p, cmd->separator, sep_len); + p += sep_len; + } + memcpy(p, e->str, e->len); + p += e->len; + break; + } + } + + return ret; +} diff --git a/formats/custom-type.h b/formats/custom-type.h new file mode 100644 index 0000000..cb5b7be --- /dev/null +++ b/formats/custom-type.h @@ -0,0 +1,12 @@ +#ifndef CUSTOM_TYPE_H +#define CUSTOM_TYPE_H + +#include +#include + +struct cmd; + +void +custom_type_reply(redisAsyncContext *c, void *r, void *privdata); + +#endif diff --git a/formats/json.c b/formats/json.c new file mode 100644 index 0000000..56c3a95 --- /dev/null +++ b/formats/json.c @@ -0,0 +1,293 @@ +#include "json.h" +#include "common.h" +#include "cmd.h" +#include "http.h" +#include "client.h" + +#include +#include +#include + +static json_t * +json_wrap_redis_reply(const struct cmd *cmd, const redisReply *r); + +void +json_reply(redisAsyncContext *c, void *r, void *privdata) { + + redisReply *reply = r; + struct cmd *cmd = privdata; + json_t *j; + char *jstr; + (void)c; + + if(cmd == NULL) { + /* broken connection */ + return; + } + + if (reply == NULL) { /* broken Redis link */ + format_send_error(cmd, 503, "Service Unavailable"); + return; + } + + /* encode redis reply as JSON */ + j = json_wrap_redis_reply(cmd, r); + + /* get JSON as string, possibly with JSONP wrapper */ + jstr = json_string_output(j, cmd->jsonp); + + /* send reply */ + format_send_reply(cmd, jstr, strlen(jstr), "application/json"); + + /* cleanup */ + json_decref(j); + free(jstr); +} + +/** + * Parse info message and return object. + */ +static json_t * +json_info_reply(const char *s) { + const char *p = s; + size_t sz = strlen(s); + + json_t *jroot = json_object(); + + /* TODO: handle new format */ + + while(p < s + sz) { + char *key, *val, *nl, *colon; + + /* find key */ + colon = strchr(p, ':'); + if(!colon) { + break; + } + key = calloc(colon - p + 1, 1); + memcpy(key, p, colon - p); + p = colon + 1; + + /* find value */ + nl = strchr(p, '\r'); + if(!nl) { + free(key); + break; + } + val = calloc(nl - p + 1, 1); + memcpy(val, p, nl - p); + p = nl + 1; + if(*p == '\n') p++; + + /* add to object */ + json_object_set_new(jroot, key, json_string(val)); + free(key); + free(val); + } + + return jroot; +} + +static json_t * +json_hgetall_reply(const redisReply *r) { + /* zip keys and values together in a json object */ + json_t *jroot; + unsigned int i; + + if(r->elements % 2 != 0) { + return NULL; + } + + jroot = json_object(); + for(i = 0; i < r->elements; i += 2) { + redisReply *k = r->element[i], *v = r->element[i+1]; + + /* keys and values need to be strings */ + if(k->type != REDIS_REPLY_STRING || v->type != REDIS_REPLY_STRING) { + json_decref(jroot); + return NULL; + } + json_object_set_new(jroot, k->str, json_string(v->str)); + } + return jroot; +} + +static json_t * +json_wrap_redis_reply(const struct cmd *cmd, const redisReply *r) { + + unsigned int i; + json_t *jlist, *jroot = json_object(); /* that's what we return */ + + + /* copy verb, as jansson only takes a char* but not its length. */ + char *verb; + if(cmd->count) { + verb = calloc(cmd->argv_len[0]+1, 1); + memcpy(verb, cmd->argv[0], cmd->argv_len[0]); + } else { + verb = strdup(""); + } + + + switch(r->type) { + case REDIS_REPLY_STATUS: + case REDIS_REPLY_ERROR: + jlist = json_array(); + json_array_append_new(jlist, + r->type == REDIS_REPLY_ERROR ? json_false() : json_true()); + json_array_append_new(jlist, json_string(r->str)); + json_object_set_new(jroot, verb, jlist); + break; + + case REDIS_REPLY_STRING: + if(strcasecmp(verb, "INFO") == 0) { + json_object_set_new(jroot, verb, json_info_reply(r->str)); + } else { + json_object_set_new(jroot, verb, json_string(r->str)); + } + break; + + case REDIS_REPLY_INTEGER: + json_object_set_new(jroot, verb, json_integer(r->integer)); + break; + + case REDIS_REPLY_ARRAY: + if(strcasecmp(verb, "HGETALL") == 0) { + json_t *jobj = json_hgetall_reply(r); + if(jobj) { + json_object_set_new(jroot, verb, jobj); + break; + } + } + jlist = json_array(); + for(i = 0; i < r->elements; ++i) { + redisReply *e = r->element[i]; + switch(e->type) { + case REDIS_REPLY_STRING: + json_array_append_new(jlist, json_string(e->str)); + break; + case REDIS_REPLY_INTEGER: + json_array_append_new(jlist, json_integer(e->integer)); + break; + default: + json_array_append_new(jlist, json_null()); + break; + } + } + json_object_set_new(jroot, verb, jlist); + break; + + default: + json_object_set_new(jroot, verb, json_null()); + break; + } + + free(verb); + return jroot; +} + + +char * +json_string_output(json_t *j, const char *jsonp) { + + char *json_reply = json_dumps(j, JSON_COMPACT); + + /* check for JSONP */ + if(jsonp) { + size_t jsonp_len = strlen(jsonp); + size_t json_len = strlen(json_reply); + size_t ret_len = jsonp_len + 1 + json_len + 3; + char *ret = calloc(1 + ret_len, 1); + + memcpy(ret, jsonp, jsonp_len); + ret[jsonp_len]='('; + memcpy(ret + jsonp_len + 1, json_reply, json_len); + memcpy(ret + jsonp_len + 1 + json_len, ");\n", 3); + free(json_reply); + + return ret; + } + + return json_reply; +} + +/* extract JSON from WebSocket frame and fill struct cmd. */ +struct cmd * +json_ws_extract(struct http_client *c, const char *p, size_t sz) { + + struct cmd *cmd = NULL; + json_t *j; + char *jsonz; /* null-terminated */ + + unsigned int i, cur; + int argc = 0; + json_error_t jerror; + + (void)c; + + jsonz = calloc(sz + 1, 1); + memcpy(jsonz, p, sz); + j = json_loads(jsonz, sz, &jerror); + free(jsonz); + + if(!j) { + return NULL; + } + if(json_typeof(j) != JSON_ARRAY) { + json_decref(j); + return NULL; /* invalid JSON */ + } + + /* count elements */ + for(i = 0; i < json_array_size(j); ++i) { + json_t *jelem = json_array_get(j, i); + + switch(json_typeof(jelem)) { + case JSON_STRING: + case JSON_INTEGER: + argc++; + break; + + default: + break; + } + } + + if(!argc) { /* not a single item could be decoded */ + json_decref(j); + return NULL; + } + + /* create command and add args */ + cmd = cmd_new(argc); + for(i = 0, cur = 0; i < json_array_size(j); ++i) { + json_t *jelem = json_array_get(j, i); + char *tmp; + + switch(json_typeof(jelem)) { + case JSON_STRING: + tmp = strdup(json_string_value(jelem)); + + cmd->argv[cur] = tmp; + cmd->argv_len[cur] = strlen(tmp); + cur++; + break; + + case JSON_INTEGER: + tmp = malloc(40); + sprintf(tmp, "%d", (int)json_integer_value(jelem)); + + cmd->argv[cur] = tmp; + cmd->argv_len[cur] = strlen(tmp); + cur++; + break; + + default: + break; + } + } + + json_decref(j); + return cmd; +} + diff --git a/formats/json.h b/formats/json.h new file mode 100644 index 0000000..911f8f5 --- /dev/null +++ b/formats/json.h @@ -0,0 +1,20 @@ +#ifndef JSON_H +#define JSON_H + +#include +#include +#include + +struct cmd; +struct http_client; + +void +json_reply(redisAsyncContext *c, void *r, void *privdata); + +char * +json_string_output(json_t *j, const char *jsonp); + +struct cmd * +json_ws_extract(struct http_client *c, const char *p, size_t sz); + +#endif diff --git a/formats/msgpack.c b/formats/msgpack.c new file mode 100644 index 0000000..dc1ab0c --- /dev/null +++ b/formats/msgpack.c @@ -0,0 +1,236 @@ +#include "msgpack.h" +#include "common.h" +#include "cmd.h" +#include "http.h" +#include "client.h" + +#include +#include +#include + +struct msg_out { + char *p; + size_t sz; +}; + +static void +msgpack_wrap_redis_reply(const struct cmd *cmd, struct msg_out *, const redisReply *r); + +void +msgpack_reply(redisAsyncContext *c, void *r, void *privdata) { + + redisReply *reply = r; + struct cmd *cmd = privdata; + struct msg_out out; + (void)c; + + if(cmd == NULL) { + /* broken connection */ + return; + } + + if (reply == NULL) { /* broken Redis link */ + format_send_error(cmd, 503, "Service Unavailable"); + return; + } + + /* prepare data structure for output */ + out.p = NULL; + out.sz = 0; + + /* encode redis reply */ + msgpack_wrap_redis_reply(cmd, &out, r); + + /* send reply */ + format_send_reply(cmd, out.p, out.sz, "application/x-msgpack"); + + /* cleanup */ + free(out.p); +} + +static int +on_msgpack_write(void *data, const char *s, unsigned int sz) { + + struct msg_out *out = data; + + out->p = realloc(out->p, out->sz + sz); + memcpy(out->p + out->sz, s, sz); + out->sz += sz; + + return sz; +} + +/** + * Parse info message and return object. + */ +void +msg_info_reply(msgpack_packer* pk, const char *s, size_t sz) { + + const char *p = s; + unsigned int count = 0; + + /* TODO: handle new format */ + + /* count number of lines */ + while(p < s + sz) { + p = strchr(p, '\r'); + if(!p) break; + + p++; + count++; + } + + /* create msgpack object */ + msgpack_pack_map(pk, count); + + p = s; + while(p < s + sz) { + char *key, *val, *nl, *colon; + size_t key_sz, val_sz; + + /* find key */ + colon = strchr(p, ':'); + if(!colon) { + break; + } + key_sz = colon - p; + key = calloc(key_sz + 1, 1); + memcpy(key, p, key_sz); + p = colon + 1; + + /* find value */ + nl = strchr(p, '\r'); + if(!nl) { + free(key); + break; + } + val_sz = nl - p; + val = calloc(val_sz + 1, 1); + memcpy(val, p, val_sz); + p = nl + 1; + if(*p == '\n') p++; + + /* add to object */ + msgpack_pack_raw(pk, key_sz); + msgpack_pack_raw_body(pk, key, key_sz); + msgpack_pack_raw(pk, val_sz); + msgpack_pack_raw_body(pk, val, val_sz); + + free(key); + free(val); + } +} + +static void +msg_hgetall_reply(msgpack_packer* pk, const redisReply *r) { + + /* zip keys and values together in a msgpack object */ + + unsigned int i; + + if(r->elements % 2 != 0) { + return; + } + + msgpack_pack_map(pk, r->elements / 2); + for(i = 0; i < r->elements; i += 2) { + redisReply *k = r->element[i], *v = r->element[i+1]; + + /* keys and values need to be strings */ + if(k->type != REDIS_REPLY_STRING || v->type != REDIS_REPLY_STRING) { + return; + } + + /* key */ + msgpack_pack_raw(pk, k->len); + msgpack_pack_raw_body(pk, k->str, k->len); + + /* value */ + msgpack_pack_raw(pk, v->len); + msgpack_pack_raw_body(pk, v->str, v->len); + } +} + +static void +msgpack_wrap_redis_reply(const struct cmd *cmd, struct msg_out *out, const redisReply *r) { + + unsigned int i; + msgpack_packer* pk = msgpack_packer_new(out, on_msgpack_write); + + /* copy verb, as jansson only takes a char* but not its length. */ + char *verb = ""; + size_t verb_sz = 0; + if(cmd->count) { + verb_sz = cmd->argv_len[0]; + verb = cmd->argv[0]; + } + + /* Create map object */ + msgpack_pack_map(pk, 1); + + /* The single element is the verb */ + msgpack_pack_raw(pk, verb_sz); + msgpack_pack_raw_body(pk, verb, verb_sz); + + switch(r->type) { + case REDIS_REPLY_STATUS: + case REDIS_REPLY_ERROR: + msgpack_pack_array(pk, 2); + + /* first element: book */ + if(r->type == REDIS_REPLY_ERROR) + msgpack_pack_false(pk); + else + msgpack_pack_true(pk); + + /* second element: message */ + msgpack_pack_raw(pk, r->len); + msgpack_pack_raw_body(pk, r->str, r->len); + break; + + case REDIS_REPLY_STRING: + if(verb_sz ==4 && strncasecmp(verb, "INFO", 4) == 0) { + msg_info_reply(pk, r->str, r->len); + } else { + msgpack_pack_raw(pk, r->len); + msgpack_pack_raw_body(pk, r->str, r->len); + } + break; + + case REDIS_REPLY_INTEGER: + msgpack_pack_int(pk, r->integer); + break; + + case REDIS_REPLY_ARRAY: + if(verb_sz == 7 && strncasecmp(verb, "HGETALL", 7) == 0) { + msg_hgetall_reply(pk, r); + break; + } + + msgpack_pack_array(pk, r->elements); + + for(i = 0; i < r->elements; ++i) { + redisReply *e = r->element[i]; + switch(e->type) { + case REDIS_REPLY_STRING: + msgpack_pack_raw(pk, e->len); + msgpack_pack_raw_body(pk, e->str, e->len); + break; + case REDIS_REPLY_INTEGER: + msgpack_pack_int(pk, e->integer); + break; + default: + msgpack_pack_nil(pk); + break; + } + } + + break; + + default: + msgpack_pack_nil(pk); + break; + } + + msgpack_packer_free(pk); +} diff --git a/formats/msgpack.h b/formats/msgpack.h new file mode 100644 index 0000000..796caec --- /dev/null +++ b/formats/msgpack.h @@ -0,0 +1,11 @@ +#ifndef MSGPACK_H +#define MSGPACK_H + +#include +#include +#include + +void +msgpack_reply(redisAsyncContext *c, void *r, void *privdata); + +#endif diff --git a/formats/raw.c b/formats/raw.c new file mode 100644 index 0000000..fdc9051 --- /dev/null +++ b/formats/raw.c @@ -0,0 +1,192 @@ +#include "raw.h" +#include "common.h" +#include "http.h" +#include "client.h" +#include "cmd.h" + +#include +#include +#include + +static char * +raw_wrap(const redisReply *r, size_t *sz); + +void +raw_reply(redisAsyncContext *c, void *r, void *privdata) { + + redisReply *reply = r; + struct cmd *cmd = privdata; + char *raw_out; + size_t sz; + (void)c; + + if (reply == NULL) { /* broken Redis link */ + format_send_error(cmd, 503, "Service Unavailable"); + return; + } + + raw_out = raw_wrap(r, &sz); + + /* send reply */ + format_send_reply(cmd, raw_out, sz, "binary/octet-stream"); + + /* cleanup */ + free(raw_out); +} + +/* extract Redis protocol string from WebSocket frame and fill struct cmd. */ +struct cmd * +raw_ws_extract(struct http_client *c, const char *p, size_t sz) { + + struct cmd *cmd = NULL; + void *reader = NULL; + redisReply *reply = NULL; + void **reply_ptr = (void**)&reply; + unsigned int i; + (void)c; + + /* create protocol reader */ + reader = redisReaderCreate(); + + /* add data */ + redisReaderFeed(reader, (char*)p, sz); + + /* parse data into reply object */ + if(redisReaderGetReply(reader, reply_ptr) == REDIS_ERR) { + goto end; + } + + /* add data from reply object to cmd struct */ + if(reply->type != REDIS_REPLY_ARRAY) { + goto end; + } + + /* create cmd object */ + cmd = cmd_new(reply->elements); + + for(i = 0; i < reply->elements; ++i) { + redisReply *ri = reply->element[i]; + + switch(ri->type) { + case REDIS_REPLY_STRING: + cmd->argv_len[i] = ri->len; + cmd->argv[i] = calloc(cmd->argv_len[i] + 1, 1); + memcpy(cmd->argv[i], ri->str, ri->len); + break; + + case REDIS_REPLY_INTEGER: + cmd->argv_len[i] = integer_length(ri->integer); + cmd->argv[i] = calloc(cmd->argv_len[i] + 1, 1); + sprintf(cmd->argv[i], "%lld", ri->integer); + break; + + default: + cmd_free(cmd); + cmd = NULL; + goto end; + } + } + + +end: + /* free reader */ + if(reader) redisReaderFree(reader); + + /* free reply */ + if(reply) freeReplyObject(reply); + + return cmd; +} + + +static char * +raw_array(const redisReply *r, size_t *sz) { + + unsigned int i; + char *ret, *p; + + /* compute size */ + *sz = 0; + *sz += 1 + integer_length(r->elements) + 2; + for(i = 0; i < r->elements; ++i) { + redisReply *e = r->element[i]; + switch(e->type) { + case REDIS_REPLY_STRING: + *sz += 1 + integer_length(e->len) + 2 + + e->len + 2; + break; + case REDIS_REPLY_INTEGER: + *sz += 1 + integer_length(integer_length(e->integer)) + 2 + + integer_length(e->integer) + 2; + break; + + } + } + + /* allocate */ + p = ret = malloc(1+*sz); + p += sprintf(p, "*%zd\r\n", r->elements); + + /* copy */ + for(i = 0; i < r->elements; ++i) { + redisReply *e = r->element[i]; + switch(e->type) { + case REDIS_REPLY_STRING: + p += sprintf(p, "$%d\r\n", e->len); + memcpy(p, e->str, e->len); + p += e->len; + *p = '\r'; + p++; + *p = '\n'; + p++; + break; + case REDIS_REPLY_INTEGER: + p += sprintf(p, "$%d\r\n%lld\r\n", + integer_length(e->integer), e->integer); + break; + } + } + + return ret; +} + +static char * +raw_wrap(const redisReply *r, size_t *sz) { + + char *ret, *p; + + switch(r->type) { + case REDIS_REPLY_STATUS: + case REDIS_REPLY_ERROR: + *sz = 3 + r->len; + ret = malloc(*sz); + ret[0] = (r->type == REDIS_REPLY_STATUS?'+':'-'); + memcpy(ret+1, r->str, *sz-3); + memcpy(ret+*sz - 2, "\r\n", 2); + return ret; + + case REDIS_REPLY_STRING: + *sz = 1 + integer_length(r->len) + 2 + r->len + 2; + p = ret = malloc(*sz); + p += sprintf(p, "$%d\r\n", r->len); + memcpy(p, r->str, *sz - 2 - (p-ret)); + memcpy(ret + *sz - 2, "\r\n", 2); + return ret; + + case REDIS_REPLY_INTEGER: + *sz = 3 + integer_length(r->integer); + ret = malloc(4+*sz); + sprintf(ret, ":%lld\r\n", r->integer); + return ret; + + case REDIS_REPLY_ARRAY: + return raw_array(r, sz); + + default: + *sz = 5; + ret = malloc(*sz); + memcpy(ret, "$-1\r\n", 5); + return ret; + } +} + diff --git a/formats/raw.h b/formats/raw.h new file mode 100644 index 0000000..10321e3 --- /dev/null +++ b/formats/raw.h @@ -0,0 +1,16 @@ +#ifndef RAW_H +#define RAW_H + +#include +#include + +struct cmd; +struct http_client; + +void +raw_reply(redisAsyncContext *c, void *r, void *privdata); + +struct cmd * +raw_ws_extract(struct http_client *c, const char *p, size_t sz); + +#endif diff --git a/http-parser/.gitignore b/http-parser/.gitignore new file mode 100644 index 0000000..73fe6a4 --- /dev/null +++ b/http-parser/.gitignore @@ -0,0 +1,4 @@ +tags +*.o +test +test_g diff --git a/http-parser/CONTRIBUTIONS b/http-parser/CONTRIBUTIONS new file mode 100644 index 0000000..11ba31e --- /dev/null +++ b/http-parser/CONTRIBUTIONS @@ -0,0 +1,4 @@ +Contributors must agree to the Contributor License Agreement before patches +can be accepted. + +http://spreadsheets2.google.com/viewform?hl=en&formkey=dDJXOGUwbzlYaWM4cHN1MERwQS1CSnc6MQ diff --git a/http-parser/LICENSE-MIT b/http-parser/LICENSE-MIT new file mode 100644 index 0000000..f30a31d --- /dev/null +++ b/http-parser/LICENSE-MIT @@ -0,0 +1,19 @@ +Copyright 2009,2010 Ryan Dahl + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. diff --git a/http-parser/README.md b/http-parser/README.md new file mode 100644 index 0000000..72332fb --- /dev/null +++ b/http-parser/README.md @@ -0,0 +1,171 @@ +HTTP Parser +=========== + +This is a parser for HTTP messages written in C. It parses both requests and +responses. The parser is designed to be used in performance HTTP +applications. It does not make any syscalls nor allocations, it does not +buffer data, it can be interrupted at anytime. Depending on your +architecture, it only requires about 40 bytes of data per message +stream (in a web server that is per connection). + +Features: + + * No dependencies + * Handles persistent streams (keep-alive). + * Decodes chunked encoding. + * Upgrade support + * Defends against buffer overflow attacks. + +The parser extracts the following information from HTTP messages: + + * Header fields and values + * Content-Length + * Request method + * Response status code + * Transfer-Encoding + * HTTP version + * Request path, query string, fragment + * Message body + + +Usage +----- + +One `http_parser` object is used per TCP connection. Initialize the struct +using `http_parser_init()` and set the callbacks. That might look something +like this for a request parser: + + http_parser_settings settings; + settings.on_path = my_path_callback; + settings.on_header_field = my_header_field_callback; + /* ... */ + + http_parser *parser = malloc(sizeof(http_parser)); + http_parser_init(parser, HTTP_REQUEST); + parser->data = my_socket; + +When data is received on the socket execute the parser and check for errors. + + size_t len = 80*1024, nparsed; + char buf[len]; + ssize_t recved; + + recved = recv(fd, buf, len, 0); + + if (recved < 0) { + /* Handle error. */ + } + + /* Start up / continue the parser. + * Note we pass recved==0 to signal that EOF has been recieved. + */ + nparsed = http_parser_execute(parser, &settings, buf, recved); + + if (parser->upgrade) { + /* handle new protocol */ + } else if (nparsed != recved) { + /* Handle error. Usually just close the connection. */ + } + +HTTP needs to know where the end of the stream is. For example, sometimes +servers send responses without Content-Length and expect the client to +consume input (for the body) until EOF. To tell http_parser about EOF, give +`0` as the forth parameter to `http_parser_execute()`. Callbacks and errors +can still be encountered during an EOF, so one must still be prepared +to receive them. + +Scalar valued message information such as `status_code`, `method`, and the +HTTP version are stored in the parser structure. This data is only +temporally stored in `http_parser` and gets reset on each new message. If +this information is needed later, copy it out of the structure during the +`headers_complete` callback. + +The parser decodes the transfer-encoding for both requests and responses +transparently. That is, a chunked encoding is decoded before being sent to +the on_body callback. + + +The Special Problem of Upgrade +------------------------------ + +HTTP supports upgrading the connection to a different protocol. An +increasingly common example of this is the Web Socket protocol which sends +a request like + + GET /demo HTTP/1.1 + Upgrade: WebSocket + Connection: Upgrade + Host: example.com + Origin: http://example.com + WebSocket-Protocol: sample + +followed by non-HTTP data. + +(See http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-75 for more +information the Web Socket protocol.) + +To support this, the parser will treat this as a normal HTTP message without a +body. Issuing both on_headers_complete and on_message_complete callbacks. However +http_parser_execute() will stop parsing at the end of the headers and return. + +The user is expected to check if `parser->upgrade` has been set to 1 after +`http_parser_execute()` returns. Non-HTTP data begins at the buffer supplied +offset by the return value of `http_parser_execute()`. + + +Callbacks +--------- + +During the `http_parser_execute()` call, the callbacks set in +`http_parser_settings` will be executed. The parser maintains state and +never looks behind, so buffering the data is not necessary. If you need to +save certain data for later usage, you can do that from the callbacks. + +There are two types of callbacks: + +* notification `typedef int (*http_cb) (http_parser*);` + Callbacks: on_message_begin, on_headers_complete, on_message_complete. +* data `typedef int (*http_data_cb) (http_parser*, const char *at, size_t length);` + Callbacks: (requests only) on_path, on_query_string, on_uri, on_fragment, + (common) on_header_field, on_header_value, on_body; + +Callbacks must return 0 on success. Returning a non-zero value indicates +error to the parser, making it exit immediately. + +In case you parse HTTP message in chunks (i.e. `read()` request line +from socket, parse, read half headers, parse, etc) your data callbacks +may be called more than once. Http-parser guarantees that data pointer is only +valid for the lifetime of callback. You can also `read()` into a heap allocated +buffer to avoid copying memory around if this fits your application. + +Reading headers may be a tricky task if you read/parse headers partially. +Basically, you need to remember whether last header callback was field or value +and apply following logic: + + (on_header_field and on_header_value shortened to on_h_*) + ------------------------ ------------ -------------------------------------------- + | State (prev. callback) | Callback | Description/action | + ------------------------ ------------ -------------------------------------------- + | nothing (first call) | on_h_field | Allocate new buffer and copy callback data | + | | | into it | + ------------------------ ------------ -------------------------------------------- + | value | on_h_field | New header started. | + | | | Copy current name,value buffers to headers | + | | | list and allocate new buffer for new name | + ------------------------ ------------ -------------------------------------------- + | field | on_h_field | Previous name continues. Reallocate name | + | | | buffer and append callback data to it | + ------------------------ ------------ -------------------------------------------- + | field | on_h_value | Value for current header started. Allocate | + | | | new buffer and copy callback data to it | + ------------------------ ------------ -------------------------------------------- + | value | on_h_value | Value continues. Reallocate value buffer | + | | | and append callback data to it | + ------------------------ ------------ -------------------------------------------- + + +See examples of reading in headers: + +* [partial example](http://gist.github.com/155877) in C +* [from http-parser tests](http://github.com/ry/http-parser/blob/37a0ff8928fb0d83cec0d0d8909c5a4abcd221af/test.c#L403) in C +* [from Node library](http://github.com/ry/node/blob/842eaf446d2fdcb33b296c67c911c32a0dabc747/src/http.js#L284) in Javascript diff --git a/http-parser/http_parser.c b/http-parser/http_parser.c new file mode 100644 index 0000000..7823085 --- /dev/null +++ b/http-parser/http_parser.c @@ -0,0 +1,1591 @@ +/* Copyright 2009,2010 Ryan Dahl + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +#include +#include +#include + + +#ifndef MIN +# define MIN(a,b) ((a) < (b) ? (a) : (b)) +#endif + + +#define CALLBACK2(FOR) \ +do { \ + if (settings->on_##FOR) { \ + if (0 != settings->on_##FOR(parser)) return (p - data); \ + } \ +} while (0) + + +#define MARK(FOR) \ +do { \ + FOR##_mark = p; \ +} while (0) + +#define CALLBACK_NOCLEAR(FOR) \ +do { \ + if (FOR##_mark) { \ + if (settings->on_##FOR) { \ + if (0 != settings->on_##FOR(parser, \ + FOR##_mark, \ + p - FOR##_mark)) \ + { \ + return (p - data); \ + } \ + } \ + } \ +} while (0) + + +#define CALLBACK(FOR) \ +do { \ + CALLBACK_NOCLEAR(FOR); \ + FOR##_mark = NULL; \ +} while (0) + + +#define PROXY_CONNECTION "proxy-connection" +#define CONNECTION "connection" +#define CONTENT_LENGTH "content-length" +#define TRANSFER_ENCODING "transfer-encoding" +#define UPGRADE "upgrade" +#define CHUNKED "chunked" +#define KEEP_ALIVE "keep-alive" +#define CLOSE "close" + + +static const char *method_strings[] = + { "DELETE" + , "GET" + , "HEAD" + , "POST" + , "PUT" + , "CONNECT" + , "OPTIONS" + , "TRACE" + , "COPY" + , "LOCK" + , "MKCOL" + , "MOVE" + , "PROPFIND" + , "PROPPATCH" + , "UNLOCK" + , "REPORT" + , "MKACTIVITY" + , "CHECKOUT" + , "MERGE" + , "M-SEARCH" + , "NOTIFY" + , "SUBSCRIBE" + , "UNSUBSCRIBE" + }; + + +/* Tokens as defined by rfc 2616. Also lowercases them. + * token = 1* + * separators = "(" | ")" | "<" | ">" | "@" + * | "," | ";" | ":" | "\" | <"> + * | "/" | "[" | "]" | "?" | "=" + * | "{" | "}" | SP | HT + */ +static const char tokens[256] = { +/* 0 nul 1 soh 2 stx 3 etx 4 eot 5 enq 6 ack 7 bel */ + 0, 0, 0, 0, 0, 0, 0, 0, +/* 8 bs 9 ht 10 nl 11 vt 12 np 13 cr 14 so 15 si */ + 0, 0, 0, 0, 0, 0, 0, 0, +/* 16 dle 17 dc1 18 dc2 19 dc3 20 dc4 21 nak 22 syn 23 etb */ + 0, 0, 0, 0, 0, 0, 0, 0, +/* 24 can 25 em 26 sub 27 esc 28 fs 29 gs 30 rs 31 us */ + 0, 0, 0, 0, 0, 0, 0, 0, +/* 32 sp 33 ! 34 " 35 # 36 $ 37 % 38 & 39 ' */ + ' ', '!', '"', '#', '$', '%', '&', '\'', +/* 40 ( 41 ) 42 * 43 + 44 , 45 - 46 . 47 / */ + 0, 0, '*', '+', 0, '-', '.', '/', +/* 48 0 49 1 50 2 51 3 52 4 53 5 54 6 55 7 */ + '0', '1', '2', '3', '4', '5', '6', '7', +/* 56 8 57 9 58 : 59 ; 60 < 61 = 62 > 63 ? */ + '8', '9', 0, 0, 0, 0, 0, 0, +/* 64 @ 65 A 66 B 67 C 68 D 69 E 70 F 71 G */ + 0, 'a', 'b', 'c', 'd', 'e', 'f', 'g', +/* 72 H 73 I 74 J 75 K 76 L 77 M 78 N 79 O */ + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', +/* 80 P 81 Q 82 R 83 S 84 T 85 U 86 V 87 W */ + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', +/* 88 X 89 Y 90 Z 91 [ 92 \ 93 ] 94 ^ 95 _ */ + 'x', 'y', 'z', 0, 0, 0, '^', '_', +/* 96 ` 97 a 98 b 99 c 100 d 101 e 102 f 103 g */ + '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', +/* 104 h 105 i 106 j 107 k 108 l 109 m 110 n 111 o */ + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', +/* 112 p 113 q 114 r 115 s 116 t 117 u 118 v 119 w */ + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', +/* 120 x 121 y 122 z 123 { 124 | 125 } 126 ~ 127 del */ + 'x', 'y', 'z', 0, '|', '}', '~', 0 }; + + +static const int8_t unhex[256] = + {-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 + ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 + ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 + , 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1,-1,-1,-1,-1 + ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1 + ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 + ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1 + ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 + }; + + +static const uint8_t normal_url_char[256] = { +/* 0 nul 1 soh 2 stx 3 etx 4 eot 5 enq 6 ack 7 bel */ + 0, 0, 0, 0, 0, 0, 0, 0, +/* 8 bs 9 ht 10 nl 11 vt 12 np 13 cr 14 so 15 si */ + 0, 0, 0, 0, 0, 0, 0, 0, +/* 16 dle 17 dc1 18 dc2 19 dc3 20 dc4 21 nak 22 syn 23 etb */ + 0, 0, 0, 0, 0, 0, 0, 0, +/* 24 can 25 em 26 sub 27 esc 28 fs 29 gs 30 rs 31 us */ + 0, 0, 0, 0, 0, 0, 0, 0, +/* 32 sp 33 ! 34 " 35 # 36 $ 37 % 38 & 39 ' */ + 0, 1, 1, 0, 1, 1, 1, 1, +/* 40 ( 41 ) 42 * 43 + 44 , 45 - 46 . 47 / */ + 1, 1, 1, 1, 1, 1, 1, 1, +/* 48 0 49 1 50 2 51 3 52 4 53 5 54 6 55 7 */ + 1, 1, 1, 1, 1, 1, 1, 1, +/* 56 8 57 9 58 : 59 ; 60 < 61 = 62 > 63 ? */ + 1, 1, 1, 1, 1, 1, 1, 0, +/* 64 @ 65 A 66 B 67 C 68 D 69 E 70 F 71 G */ + 1, 1, 1, 1, 1, 1, 1, 1, +/* 72 H 73 I 74 J 75 K 76 L 77 M 78 N 79 O */ + 1, 1, 1, 1, 1, 1, 1, 1, +/* 80 P 81 Q 82 R 83 S 84 T 85 U 86 V 87 W */ + 1, 1, 1, 1, 1, 1, 1, 1, +/* 88 X 89 Y 90 Z 91 [ 92 \ 93 ] 94 ^ 95 _ */ + 1, 1, 1, 1, 1, 1, 1, 1, +/* 96 ` 97 a 98 b 99 c 100 d 101 e 102 f 103 g */ + 1, 1, 1, 1, 1, 1, 1, 1, +/* 104 h 105 i 106 j 107 k 108 l 109 m 110 n 111 o */ + 1, 1, 1, 1, 1, 1, 1, 1, +/* 112 p 113 q 114 r 115 s 116 t 117 u 118 v 119 w */ + 1, 1, 1, 1, 1, 1, 1, 1, +/* 120 x 121 y 122 z 123 { 124 | 125 } 126 ~ 127 del */ + 1, 1, 1, 1, 1, 1, 1, 0 }; + + +enum state + { s_dead = 1 /* important that this is > 0 */ + + , s_start_req_or_res + , s_res_or_resp_H + , s_start_res + , s_res_H + , s_res_HT + , s_res_HTT + , s_res_HTTP + , s_res_first_http_major + , s_res_http_major + , s_res_first_http_minor + , s_res_http_minor + , s_res_first_status_code + , s_res_status_code + , s_res_status + , s_res_line_almost_done + + , s_start_req + + , s_req_method + , s_req_spaces_before_url + , s_req_schema + , s_req_schema_slash + , s_req_schema_slash_slash + , s_req_host + , s_req_port + , s_req_path + , s_req_query_string_start + , s_req_query_string + , s_req_fragment_start + , s_req_fragment + , s_req_http_start + , s_req_http_H + , s_req_http_HT + , s_req_http_HTT + , s_req_http_HTTP + , s_req_first_http_major + , s_req_http_major + , s_req_first_http_minor + , s_req_http_minor + , s_req_line_almost_done + + , s_header_field_start + , s_header_field + , s_header_value_start + , s_header_value + + , s_header_almost_done + + , s_headers_almost_done + /* Important: 's_headers_almost_done' must be the last 'header' state. All + * states beyond this must be 'body' states. It is used for overflow + * checking. See the PARSING_HEADER() macro. + */ + , s_chunk_size_start + , s_chunk_size + , s_chunk_size_almost_done + , s_chunk_parameters + , s_chunk_data + , s_chunk_data_almost_done + , s_chunk_data_done + + , s_body_identity + , s_body_identity_eof + }; + + +#define PARSING_HEADER(state) (state <= s_headers_almost_done && 0 == (parser->flags & F_TRAILING)) + + +enum header_states + { h_general = 0 + , h_C + , h_CO + , h_CON + + , h_matching_connection + , h_matching_proxy_connection + , h_matching_content_length + , h_matching_transfer_encoding + , h_matching_upgrade + + , h_connection + , h_content_length + , h_transfer_encoding + , h_upgrade + + , h_matching_transfer_encoding_chunked + , h_matching_connection_keep_alive + , h_matching_connection_close + + , h_transfer_encoding_chunked + , h_connection_keep_alive + , h_connection_close + }; + + +#define CR '\r' +#define LF '\n' +#define LOWER(c) (unsigned char)(c | 0x20) +#define TOKEN(c) tokens[(unsigned char)c] + + +#define start_state (parser->type == HTTP_REQUEST ? s_start_req : s_start_res) + + +#if HTTP_PARSER_STRICT +# define STRICT_CHECK(cond) if (cond) goto error +# define NEW_MESSAGE() (http_should_keep_alive(parser) ? start_state : s_dead) +#else +# define STRICT_CHECK(cond) +# define NEW_MESSAGE() start_state +#endif + + +size_t http_parser_execute (http_parser *parser, + const http_parser_settings *settings, + const char *data, + size_t len) +{ + char c, ch; + const char *p = data, *pe; + int64_t to_read; + + enum state state = (enum state) parser->state; + enum header_states header_state = (enum header_states) parser->header_state; + uint64_t index = parser->index; + uint64_t nread = parser->nread; + + if (len == 0) { + if (state == s_body_identity_eof) { + CALLBACK2(message_complete); + } + return 0; + } + + /* technically we could combine all of these (except for url_mark) into one + variable, saving stack space, but it seems more clear to have them + separated. */ + const char *header_field_mark = 0; + const char *header_value_mark = 0; + const char *fragment_mark = 0; + const char *query_string_mark = 0; + const char *path_mark = 0; + const char *url_mark = 0; + + if (state == s_header_field) + header_field_mark = data; + if (state == s_header_value) + header_value_mark = data; + if (state == s_req_fragment) + fragment_mark = data; + if (state == s_req_query_string) + query_string_mark = data; + if (state == s_req_path) + path_mark = data; + if (state == s_req_path || state == s_req_schema || state == s_req_schema_slash + || state == s_req_schema_slash_slash || state == s_req_port + || state == s_req_query_string_start || state == s_req_query_string + || state == s_req_host + || state == s_req_fragment_start || state == s_req_fragment) + url_mark = data; + + for (p=data, pe=data+len; p != pe; p++) { + ch = *p; + + if (PARSING_HEADER(state)) { + ++nread; + /* Buffer overflow attack */ + if (nread > HTTP_MAX_HEADER_SIZE) goto error; + } + + switch (state) { + + case s_dead: + /* this state is used after a 'Connection: close' message + * the parser will error out if it reads another message + */ + goto error; + + case s_start_req_or_res: + { + if (ch == CR || ch == LF) + break; + parser->flags = 0; + parser->content_length = -1; + + CALLBACK2(message_begin); + + if (ch == 'H') + state = s_res_or_resp_H; + else { + parser->type = HTTP_REQUEST; + goto start_req_method_assign; + } + break; + } + + case s_res_or_resp_H: + if (ch == 'T') { + parser->type = HTTP_RESPONSE; + state = s_res_HT; + } else { + if (ch != 'E') goto error; + parser->type = HTTP_REQUEST; + parser->method = HTTP_HEAD; + index = 2; + state = s_req_method; + } + break; + + case s_start_res: + { + parser->flags = 0; + parser->content_length = -1; + + CALLBACK2(message_begin); + + switch (ch) { + case 'H': + state = s_res_H; + break; + + case CR: + case LF: + break; + + default: + goto error; + } + break; + } + + case s_res_H: + STRICT_CHECK(ch != 'T'); + state = s_res_HT; + break; + + case s_res_HT: + STRICT_CHECK(ch != 'T'); + state = s_res_HTT; + break; + + case s_res_HTT: + STRICT_CHECK(ch != 'P'); + state = s_res_HTTP; + break; + + case s_res_HTTP: + STRICT_CHECK(ch != '/'); + state = s_res_first_http_major; + break; + + case s_res_first_http_major: + if (ch < '1' || ch > '9') goto error; + parser->http_major = ch - '0'; + state = s_res_http_major; + break; + + /* major HTTP version or dot */ + case s_res_http_major: + { + if (ch == '.') { + state = s_res_first_http_minor; + break; + } + + if (ch < '0' || ch > '9') goto error; + + parser->http_major *= 10; + parser->http_major += ch - '0'; + + if (parser->http_major > 999) goto error; + break; + } + + /* first digit of minor HTTP version */ + case s_res_first_http_minor: + if (ch < '0' || ch > '9') goto error; + parser->http_minor = ch - '0'; + state = s_res_http_minor; + break; + + /* minor HTTP version or end of request line */ + case s_res_http_minor: + { + if (ch == ' ') { + state = s_res_first_status_code; + break; + } + + if (ch < '0' || ch > '9') goto error; + + parser->http_minor *= 10; + parser->http_minor += ch - '0'; + + if (parser->http_minor > 999) goto error; + break; + } + + case s_res_first_status_code: + { + if (ch < '0' || ch > '9') { + if (ch == ' ') { + break; + } + goto error; + } + parser->status_code = ch - '0'; + state = s_res_status_code; + break; + } + + case s_res_status_code: + { + if (ch < '0' || ch > '9') { + switch (ch) { + case ' ': + state = s_res_status; + break; + case CR: + state = s_res_line_almost_done; + break; + case LF: + state = s_header_field_start; + break; + default: + goto error; + } + break; + } + + parser->status_code *= 10; + parser->status_code += ch - '0'; + + if (parser->status_code > 999) goto error; + break; + } + + case s_res_status: + /* the human readable status. e.g. "NOT FOUND" + * we are not humans so just ignore this */ + if (ch == CR) { + state = s_res_line_almost_done; + break; + } + + if (ch == LF) { + state = s_header_field_start; + break; + } + break; + + case s_res_line_almost_done: + STRICT_CHECK(ch != LF); + state = s_header_field_start; + break; + + case s_start_req: + { + if (ch == CR || ch == LF) + break; + parser->flags = 0; + parser->content_length = -1; + + CALLBACK2(message_begin); + + if (ch < 'A' || 'Z' < ch) goto error; + + start_req_method_assign: + parser->method = (enum http_method) 0; + index = 1; + switch (ch) { + case 'C': parser->method = HTTP_CONNECT; /* or COPY, CHECKOUT */ break; + case 'D': parser->method = HTTP_DELETE; break; + case 'G': parser->method = HTTP_GET; break; + case 'H': parser->method = HTTP_HEAD; break; + case 'L': parser->method = HTTP_LOCK; break; + case 'M': parser->method = HTTP_MKCOL; /* or MOVE, MKACTIVITY, MERGE, M-SEARCH */ break; + case 'N': parser->method = HTTP_NOTIFY; break; + case 'O': parser->method = HTTP_OPTIONS; break; + case 'P': parser->method = HTTP_POST; /* or PROPFIND or PROPPATCH or PUT */ break; + case 'R': parser->method = HTTP_REPORT; break; + case 'S': parser->method = HTTP_SUBSCRIBE; break; + case 'T': parser->method = HTTP_TRACE; break; + case 'U': parser->method = HTTP_UNLOCK; /* or UNSUBSCRIBE */ break; + default: goto error; + } + state = s_req_method; + break; + } + + case s_req_method: + { + if (ch == '\0') + goto error; + + const char *matcher = method_strings[parser->method]; + if (ch == ' ' && matcher[index] == '\0') { + state = s_req_spaces_before_url; + } else if (ch == matcher[index]) { + ; /* nada */ + } else if (parser->method == HTTP_CONNECT) { + if (index == 1 && ch == 'H') { + parser->method = HTTP_CHECKOUT; + } else if (index == 2 && ch == 'P') { + parser->method = HTTP_COPY; + } + } else if (parser->method == HTTP_MKCOL) { + if (index == 1 && ch == 'O') { + parser->method = HTTP_MOVE; + } else if (index == 1 && ch == 'E') { + parser->method = HTTP_MERGE; + } else if (index == 1 && ch == '-') { + parser->method = HTTP_MSEARCH; + } else if (index == 2 && ch == 'A') { + parser->method = HTTP_MKACTIVITY; + } + } else if (index == 1 && parser->method == HTTP_POST && ch == 'R') { + parser->method = HTTP_PROPFIND; /* or HTTP_PROPPATCH */ + } else if (index == 1 && parser->method == HTTP_POST && ch == 'U') { + parser->method = HTTP_PUT; + } else if (index == 2 && parser->method == HTTP_UNLOCK && ch == 'S') { + parser->method = HTTP_UNSUBSCRIBE; + } else if (index == 4 && parser->method == HTTP_PROPFIND && ch == 'P') { + parser->method = HTTP_PROPPATCH; + } else { + goto error; + } + + ++index; + break; + } + case s_req_spaces_before_url: + { + if (ch == ' ') break; + + if (ch == '/' || ch == '*') { + MARK(url); + MARK(path); + state = s_req_path; + break; + } + + c = LOWER(ch); + + if (c >= 'a' && c <= 'z') { + MARK(url); + state = s_req_schema; + break; + } + + goto error; + } + + case s_req_schema: + { + c = LOWER(ch); + + if (c >= 'a' && c <= 'z') break; + + if (ch == ':') { + state = s_req_schema_slash; + break; + } else if (ch == '.') { + state = s_req_host; + break; + } else if ('0' <= ch && ch <= '9') { + state = s_req_host; + break; + } + + goto error; + } + + case s_req_schema_slash: + STRICT_CHECK(ch != '/'); + state = s_req_schema_slash_slash; + break; + + case s_req_schema_slash_slash: + STRICT_CHECK(ch != '/'); + state = s_req_host; + break; + + case s_req_host: + { + c = LOWER(ch); + if (c >= 'a' && c <= 'z') break; + if ((ch >= '0' && ch <= '9') || ch == '.' || ch == '-') break; + switch (ch) { + case ':': + state = s_req_port; + break; + case '/': + MARK(path); + state = s_req_path; + break; + case ' ': + /* The request line looks like: + * "GET http://foo.bar.com HTTP/1.1" + * That is, there is no path. + */ + CALLBACK(url); + state = s_req_http_start; + break; + default: + goto error; + } + break; + } + + case s_req_port: + { + if (ch >= '0' && ch <= '9') break; + switch (ch) { + case '/': + MARK(path); + state = s_req_path; + break; + case ' ': + /* The request line looks like: + * "GET http://foo.bar.com:1234 HTTP/1.1" + * That is, there is no path. + */ + CALLBACK(url); + state = s_req_http_start; + break; + default: + goto error; + } + break; + } + + case s_req_path: + { + if (normal_url_char[(unsigned char)ch]) break; + + switch (ch) { + case ' ': + CALLBACK(url); + CALLBACK(path); + state = s_req_http_start; + break; + case CR: + CALLBACK(url); + CALLBACK(path); + parser->http_major = 0; + parser->http_minor = 9; + state = s_req_line_almost_done; + break; + case LF: + CALLBACK(url); + CALLBACK(path); + parser->http_major = 0; + parser->http_minor = 9; + state = s_header_field_start; + break; + case '?': + CALLBACK(path); + state = s_req_query_string_start; + break; + case '#': + CALLBACK(path); + state = s_req_fragment_start; + break; + default: + goto error; + } + break; + } + + case s_req_query_string_start: + { + if (normal_url_char[(unsigned char)ch]) { + MARK(query_string); + state = s_req_query_string; + break; + } + + switch (ch) { + case '?': + break; /* XXX ignore extra '?' ... is this right? */ + case ' ': + CALLBACK(url); + state = s_req_http_start; + break; + case CR: + CALLBACK(url); + parser->http_major = 0; + parser->http_minor = 9; + state = s_req_line_almost_done; + break; + case LF: + CALLBACK(url); + parser->http_major = 0; + parser->http_minor = 9; + state = s_header_field_start; + break; + case '#': + state = s_req_fragment_start; + break; + default: + goto error; + } + break; + } + + case s_req_query_string: + { + if (normal_url_char[(unsigned char)ch]) break; + + switch (ch) { + case '?': + /* allow extra '?' in query string */ + break; + case ' ': + CALLBACK(url); + CALLBACK(query_string); + state = s_req_http_start; + break; + case CR: + CALLBACK(url); + CALLBACK(query_string); + parser->http_major = 0; + parser->http_minor = 9; + state = s_req_line_almost_done; + break; + case LF: + CALLBACK(url); + CALLBACK(query_string); + parser->http_major = 0; + parser->http_minor = 9; + state = s_header_field_start; + break; + case '#': + CALLBACK(query_string); + state = s_req_fragment_start; + break; + default: + goto error; + } + break; + } + + case s_req_fragment_start: + { + if (normal_url_char[(unsigned char)ch]) { + MARK(fragment); + state = s_req_fragment; + break; + } + + switch (ch) { + case ' ': + CALLBACK(url); + state = s_req_http_start; + break; + case CR: + CALLBACK(url); + parser->http_major = 0; + parser->http_minor = 9; + state = s_req_line_almost_done; + break; + case LF: + CALLBACK(url); + parser->http_major = 0; + parser->http_minor = 9; + state = s_header_field_start; + break; + case '?': + MARK(fragment); + state = s_req_fragment; + break; + case '#': + break; + default: + goto error; + } + break; + } + + case s_req_fragment: + { + if (normal_url_char[(unsigned char)ch]) break; + + switch (ch) { + case ' ': + CALLBACK(url); + CALLBACK(fragment); + state = s_req_http_start; + break; + case CR: + CALLBACK(url); + CALLBACK(fragment); + parser->http_major = 0; + parser->http_minor = 9; + state = s_req_line_almost_done; + break; + case LF: + CALLBACK(url); + CALLBACK(fragment); + parser->http_major = 0; + parser->http_minor = 9; + state = s_header_field_start; + break; + case '?': + case '#': + break; + default: + goto error; + } + break; + } + + case s_req_http_start: + switch (ch) { + case 'H': + state = s_req_http_H; + break; + case ' ': + break; + default: + goto error; + } + break; + + case s_req_http_H: + STRICT_CHECK(ch != 'T'); + state = s_req_http_HT; + break; + + case s_req_http_HT: + STRICT_CHECK(ch != 'T'); + state = s_req_http_HTT; + break; + + case s_req_http_HTT: + STRICT_CHECK(ch != 'P'); + state = s_req_http_HTTP; + break; + + case s_req_http_HTTP: + STRICT_CHECK(ch != '/'); + state = s_req_first_http_major; + break; + + /* first digit of major HTTP version */ + case s_req_first_http_major: + if (ch < '1' || ch > '9') goto error; + parser->http_major = ch - '0'; + state = s_req_http_major; + break; + + /* major HTTP version or dot */ + case s_req_http_major: + { + if (ch == '.') { + state = s_req_first_http_minor; + break; + } + + if (ch < '0' || ch > '9') goto error; + + parser->http_major *= 10; + parser->http_major += ch - '0'; + + if (parser->http_major > 999) goto error; + break; + } + + /* first digit of minor HTTP version */ + case s_req_first_http_minor: + if (ch < '0' || ch > '9') goto error; + parser->http_minor = ch - '0'; + state = s_req_http_minor; + break; + + /* minor HTTP version or end of request line */ + case s_req_http_minor: + { + if (ch == CR) { + state = s_req_line_almost_done; + break; + } + + if (ch == LF) { + state = s_header_field_start; + break; + } + + /* XXX allow spaces after digit? */ + + if (ch < '0' || ch > '9') goto error; + + parser->http_minor *= 10; + parser->http_minor += ch - '0'; + + if (parser->http_minor > 999) goto error; + break; + } + + /* end of request line */ + case s_req_line_almost_done: + { + if (ch != LF) goto error; + state = s_header_field_start; + break; + } + + case s_header_field_start: + { + if (ch == CR) { + state = s_headers_almost_done; + break; + } + + if (ch == LF) { + /* they might be just sending \n instead of \r\n so this would be + * the second \n to denote the end of headers*/ + state = s_headers_almost_done; + goto headers_almost_done; + } + + c = TOKEN(ch); + + if (!c) goto error; + + MARK(header_field); + + index = 0; + state = s_header_field; + + switch (c) { + case 'c': + header_state = h_C; + break; + + case 'p': + header_state = h_matching_proxy_connection; + break; + + case 't': + header_state = h_matching_transfer_encoding; + break; + + case 'u': + header_state = h_matching_upgrade; + break; + + default: + header_state = h_general; + break; + } + break; + } + + case s_header_field: + { + c = TOKEN(ch); + + if (c) { + switch (header_state) { + case h_general: + break; + + case h_C: + index++; + header_state = (c == 'o' ? h_CO : h_general); + break; + + case h_CO: + index++; + header_state = (c == 'n' ? h_CON : h_general); + break; + + case h_CON: + index++; + switch (c) { + case 'n': + header_state = h_matching_connection; + break; + case 't': + header_state = h_matching_content_length; + break; + default: + header_state = h_general; + break; + } + break; + + /* connection */ + + case h_matching_connection: + index++; + if (index > sizeof(CONNECTION)-1 + || c != CONNECTION[index]) { + header_state = h_general; + } else if (index == sizeof(CONNECTION)-2) { + header_state = h_connection; + } + break; + + /* proxy-connection */ + + case h_matching_proxy_connection: + index++; + if (index > sizeof(PROXY_CONNECTION)-1 + || c != PROXY_CONNECTION[index]) { + header_state = h_general; + } else if (index == sizeof(PROXY_CONNECTION)-2) { + header_state = h_connection; + } + break; + + /* content-length */ + + case h_matching_content_length: + index++; + if (index > sizeof(CONTENT_LENGTH)-1 + || c != CONTENT_LENGTH[index]) { + header_state = h_general; + } else if (index == sizeof(CONTENT_LENGTH)-2) { + header_state = h_content_length; + } + break; + + /* transfer-encoding */ + + case h_matching_transfer_encoding: + index++; + if (index > sizeof(TRANSFER_ENCODING)-1 + || c != TRANSFER_ENCODING[index]) { + header_state = h_general; + } else if (index == sizeof(TRANSFER_ENCODING)-2) { + header_state = h_transfer_encoding; + } + break; + + /* upgrade */ + + case h_matching_upgrade: + index++; + if (index > sizeof(UPGRADE)-1 + || c != UPGRADE[index]) { + header_state = h_general; + } else if (index == sizeof(UPGRADE)-2) { + header_state = h_upgrade; + } + break; + + case h_connection: + case h_content_length: + case h_transfer_encoding: + case h_upgrade: + if (ch != ' ') header_state = h_general; + break; + + default: + assert(0 && "Unknown header_state"); + break; + } + break; + } + + if (ch == ':') { + CALLBACK(header_field); + state = s_header_value_start; + break; + } + + if (ch == CR) { + state = s_header_almost_done; + CALLBACK(header_field); + break; + } + + if (ch == LF) { + CALLBACK(header_field); + state = s_header_field_start; + break; + } + + goto error; + } + + case s_header_value_start: + { + if (ch == ' ') break; + + MARK(header_value); + + state = s_header_value; + index = 0; + + c = LOWER(ch); + + if (ch == CR) { + CALLBACK(header_value); + header_state = h_general; + state = s_header_almost_done; + break; + } + + if (ch == LF) { + CALLBACK(header_value); + state = s_header_field_start; + break; + } + + switch (header_state) { + case h_upgrade: + parser->flags |= F_UPGRADE; + header_state = h_general; + break; + + case h_transfer_encoding: + /* looking for 'Transfer-Encoding: chunked' */ + if ('c' == c) { + header_state = h_matching_transfer_encoding_chunked; + } else { + header_state = h_general; + } + break; + + case h_content_length: + if (ch < '0' || ch > '9') goto error; + parser->content_length = ch - '0'; + break; + + case h_connection: + /* looking for 'Connection: keep-alive' */ + if (c == 'k') { + header_state = h_matching_connection_keep_alive; + /* looking for 'Connection: close' */ + } else if (c == 'c') { + header_state = h_matching_connection_close; + } else { + header_state = h_general; + } + break; + + default: + header_state = h_general; + break; + } + break; + } + + case s_header_value: + { + c = LOWER(ch); + + if (ch == CR) { + CALLBACK(header_value); + state = s_header_almost_done; + break; + } + + if (ch == LF) { + CALLBACK(header_value); + goto header_almost_done; + } + + switch (header_state) { + case h_general: + break; + + case h_connection: + case h_transfer_encoding: + assert(0 && "Shouldn't get here."); + break; + + case h_content_length: + if (ch == ' ') break; + if (ch < '0' || ch > '9') goto error; + parser->content_length *= 10; + parser->content_length += ch - '0'; + break; + + /* Transfer-Encoding: chunked */ + case h_matching_transfer_encoding_chunked: + index++; + if (index > sizeof(CHUNKED)-1 + || c != CHUNKED[index]) { + header_state = h_general; + } else if (index == sizeof(CHUNKED)-2) { + header_state = h_transfer_encoding_chunked; + } + break; + + /* looking for 'Connection: keep-alive' */ + case h_matching_connection_keep_alive: + index++; + if (index > sizeof(KEEP_ALIVE)-1 + || c != KEEP_ALIVE[index]) { + header_state = h_general; + } else if (index == sizeof(KEEP_ALIVE)-2) { + header_state = h_connection_keep_alive; + } + break; + + /* looking for 'Connection: close' */ + case h_matching_connection_close: + index++; + if (index > sizeof(CLOSE)-1 || c != CLOSE[index]) { + header_state = h_general; + } else if (index == sizeof(CLOSE)-2) { + header_state = h_connection_close; + } + break; + + case h_transfer_encoding_chunked: + case h_connection_keep_alive: + case h_connection_close: + if (ch != ' ') header_state = h_general; + break; + + default: + state = s_header_value; + header_state = h_general; + break; + } + break; + } + + case s_header_almost_done: + header_almost_done: + { + STRICT_CHECK(ch != LF); + + state = s_header_field_start; + + switch (header_state) { + case h_connection_keep_alive: + parser->flags |= F_CONNECTION_KEEP_ALIVE; + break; + case h_connection_close: + parser->flags |= F_CONNECTION_CLOSE; + break; + case h_transfer_encoding_chunked: + parser->flags |= F_CHUNKED; + break; + default: + break; + } + break; + } + + case s_headers_almost_done: + headers_almost_done: + { + STRICT_CHECK(ch != LF); + + if (parser->flags & F_TRAILING) { + /* End of a chunked request */ + CALLBACK2(message_complete); + state = NEW_MESSAGE(); + break; + } + + nread = 0; + + if (parser->flags & F_UPGRADE || parser->method == HTTP_CONNECT) { + parser->upgrade = 1; + } + + /* Here we call the headers_complete callback. This is somewhat + * different than other callbacks because if the user returns 1, we + * will interpret that as saying that this message has no body. This + * is needed for the annoying case of recieving a response to a HEAD + * request. + */ + if (settings->on_headers_complete) { + switch (settings->on_headers_complete(parser)) { + case 0: + break; + + case 1: + parser->flags |= F_SKIPBODY; + break; + + default: + return p - data; /* Error */ + } + } + + /* Exit, the rest of the connect is in a different protocol. */ + if (parser->upgrade) { + CALLBACK2(message_complete); + return (p - data); + } + + if (parser->flags & F_SKIPBODY) { + CALLBACK2(message_complete); + state = NEW_MESSAGE(); + } else if (parser->flags & F_CHUNKED) { + /* chunked encoding - ignore Content-Length header */ + state = s_chunk_size_start; + } else { + if (parser->content_length == 0) { + /* Content-Length header given but zero: Content-Length: 0\r\n */ + CALLBACK2(message_complete); + state = NEW_MESSAGE(); + } else if (parser->content_length > 0) { + /* Content-Length header given and non-zero */ + state = s_body_identity; + } else { + if (parser->type == HTTP_REQUEST || http_should_keep_alive(parser)) { + /* Assume content-length 0 - read the next */ + CALLBACK2(message_complete); + state = NEW_MESSAGE(); + } else { + /* Read body until EOF */ + state = s_body_identity_eof; + } + } + } + + break; + } + + case s_body_identity: + to_read = MIN(pe - p, (int64_t)parser->content_length); + if (to_read > 0) { + if (settings->on_body) settings->on_body(parser, p, to_read); + p += to_read - 1; + parser->content_length -= to_read; + if (parser->content_length == 0) { + CALLBACK2(message_complete); + state = NEW_MESSAGE(); + } + } + break; + + /* read until EOF */ + case s_body_identity_eof: + to_read = pe - p; + if (to_read > 0) { + if (settings->on_body) settings->on_body(parser, p, to_read); + p += to_read - 1; + } + break; + + case s_chunk_size_start: + { + assert(parser->flags & F_CHUNKED); + + c = unhex[(unsigned char)ch]; + if (c == -1) goto error; + parser->content_length = c; + state = s_chunk_size; + break; + } + + case s_chunk_size: + { + assert(parser->flags & F_CHUNKED); + + if (ch == CR) { + state = s_chunk_size_almost_done; + break; + } + + c = unhex[(unsigned char)ch]; + + if (c == -1) { + if (ch == ';' || ch == ' ') { + state = s_chunk_parameters; + break; + } + goto error; + } + + parser->content_length *= 16; + parser->content_length += c; + break; + } + + case s_chunk_parameters: + { + assert(parser->flags & F_CHUNKED); + /* just ignore this shit. TODO check for overflow */ + if (ch == CR) { + state = s_chunk_size_almost_done; + break; + } + break; + } + + case s_chunk_size_almost_done: + { + assert(parser->flags & F_CHUNKED); + STRICT_CHECK(ch != LF); + + if (parser->content_length == 0) { + parser->flags |= F_TRAILING; + state = s_header_field_start; + } else { + state = s_chunk_data; + } + break; + } + + case s_chunk_data: + { + assert(parser->flags & F_CHUNKED); + + to_read = MIN(pe - p, (int64_t)(parser->content_length)); + + if (to_read > 0) { + if (settings->on_body) settings->on_body(parser, p, to_read); + p += to_read - 1; + } + + if (to_read == parser->content_length) { + state = s_chunk_data_almost_done; + } + + parser->content_length -= to_read; + break; + } + + case s_chunk_data_almost_done: + assert(parser->flags & F_CHUNKED); + STRICT_CHECK(ch != CR); + state = s_chunk_data_done; + break; + + case s_chunk_data_done: + assert(parser->flags & F_CHUNKED); + STRICT_CHECK(ch != LF); + state = s_chunk_size_start; + break; + + default: + assert(0 && "unhandled state"); + goto error; + } + } + + CALLBACK_NOCLEAR(header_field); + CALLBACK_NOCLEAR(header_value); + CALLBACK_NOCLEAR(fragment); + CALLBACK_NOCLEAR(query_string); + CALLBACK_NOCLEAR(path); + CALLBACK_NOCLEAR(url); + + parser->state = state; + parser->header_state = header_state; + parser->index = index; + parser->nread = nread; + + return len; + +error: + parser->state = s_dead; + return (p - data); +} + + +int +http_should_keep_alive (http_parser *parser) +{ + if (parser->http_major > 0 && parser->http_minor > 0) { + /* HTTP/1.1 */ + if (parser->flags & F_CONNECTION_CLOSE) { + return 0; + } else { + return 1; + } + } else { + /* HTTP/1.0 or earlier */ + if (parser->flags & F_CONNECTION_KEEP_ALIVE) { + return 1; + } else { + return 0; + } + } +} + + +const char * http_method_str (enum http_method m) +{ + return method_strings[m]; +} + + +void +http_parser_init (http_parser *parser, enum http_parser_type t) +{ + parser->type = t; + parser->state = (t == HTTP_REQUEST ? s_start_req : (t == HTTP_RESPONSE ? s_start_res : s_start_req_or_res)); + parser->nread = 0; + parser->upgrade = 0; + parser->flags = 0; + parser->method = 0; +} diff --git a/http-parser/http_parser.h b/http-parser/http_parser.h new file mode 100644 index 0000000..71beecd --- /dev/null +++ b/http-parser/http_parser.h @@ -0,0 +1,191 @@ +/* Copyright 2009,2010 Ryan Dahl + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +#ifndef http_parser_h +#define http_parser_h +#ifdef __cplusplus +extern "C" { +#endif + + +#include +#if defined(_WIN32) && !defined(__MINGW32__) +typedef __int8 int8_t; +typedef unsigned __int8 uint8_t; +typedef __int16 int16_t; +typedef unsigned __int16 uint16_t; +typedef __int32 int32_t; +typedef unsigned __int32 uint32_t; +typedef __int64 int64_t; +typedef unsigned __int64 uint64_t; + +typedef unsigned int size_t; +typedef int ssize_t; +#else +#include +#endif + +/* Compile with -DHTTP_PARSER_STRICT=0 to make less checks, but run + * faster + */ +#ifndef HTTP_PARSER_STRICT +# define HTTP_PARSER_STRICT 1 +#else +# define HTTP_PARSER_STRICT 0 +#endif + + +/* Maximium header size allowed */ +#define HTTP_MAX_HEADER_SIZE (80*1024) + + +typedef struct http_parser http_parser; +typedef struct http_parser_settings http_parser_settings; + + +/* Callbacks should return non-zero to indicate an error. The parser will + * then halt execution. + * + * The one exception is on_headers_complete. In a HTTP_RESPONSE parser + * returning '1' from on_headers_complete will tell the parser that it + * should not expect a body. This is used when receiving a response to a + * HEAD request which may contain 'Content-Length' or 'Transfer-Encoding: + * chunked' headers that indicate the presence of a body. + * + * http_data_cb does not return data chunks. It will be call arbitrarally + * many times for each string. E.G. you might get 10 callbacks for "on_path" + * each providing just a few characters more data. + */ +typedef int (*http_data_cb) (http_parser*, const char *at, size_t length); +typedef int (*http_cb) (http_parser*); + + +/* Request Methods */ +enum http_method + { HTTP_DELETE = 0 + , HTTP_GET + , HTTP_HEAD + , HTTP_POST + , HTTP_PUT + /* pathological */ + , HTTP_CONNECT + , HTTP_OPTIONS + , HTTP_TRACE + /* webdav */ + , HTTP_COPY + , HTTP_LOCK + , HTTP_MKCOL + , HTTP_MOVE + , HTTP_PROPFIND + , HTTP_PROPPATCH + , HTTP_UNLOCK + /* subversion */ + , HTTP_REPORT + , HTTP_MKACTIVITY + , HTTP_CHECKOUT + , HTTP_MERGE + /* upnp */ + , HTTP_MSEARCH + , HTTP_NOTIFY + , HTTP_SUBSCRIBE + , HTTP_UNSUBSCRIBE + }; + + +enum http_parser_type { HTTP_REQUEST, HTTP_RESPONSE, HTTP_BOTH }; + + +struct http_parser { + /** PRIVATE **/ + unsigned char type : 2; + unsigned char flags : 6; + unsigned char state; + unsigned char header_state; + unsigned char index; + + uint32_t nread; + int64_t content_length; + + /** READ-ONLY **/ + unsigned short http_major; + unsigned short http_minor; + unsigned short status_code; /* responses only */ + unsigned char method; /* requests only */ + + /* 1 = Upgrade header was present and the parser has exited because of that. + * 0 = No upgrade header present. + * Should be checked when http_parser_execute() returns in addition to + * error checking. + */ + char upgrade; + + /** PUBLIC **/ + void *data; /* A pointer to get hook to the "connection" or "socket" object */ +}; + + +struct http_parser_settings { + http_cb on_message_begin; + http_data_cb on_path; + http_data_cb on_query_string; + http_data_cb on_url; + http_data_cb on_fragment; + http_data_cb on_header_field; + http_data_cb on_header_value; + http_cb on_headers_complete; + http_data_cb on_body; + http_cb on_message_complete; +}; + + +enum flags + { F_CHUNKED = 1 << 0 + , F_CONNECTION_KEEP_ALIVE = 1 << 1 + , F_CONNECTION_CLOSE = 1 << 2 + , F_TRAILING = 1 << 3 + , F_UPGRADE = 1 << 4 + , F_SKIPBODY = 1 << 5 + }; + + +void http_parser_init(http_parser *parser, enum http_parser_type type); + + +size_t http_parser_execute(http_parser *parser, + const http_parser_settings *settings, + const char *data, + size_t len); + + +/* If http_should_keep_alive() in the on_headers_complete or + * on_message_complete callback returns true, then this will be should be + * the last message on the connection. + * If you are the server, respond with the "Connection: close" header. + * If you are the client, close the connection. + */ +int http_should_keep_alive(http_parser *parser); + +/* Returns a string version of the HTTP method. */ +const char *http_method_str(enum http_method); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/http.c b/http.c new file mode 100644 index 0000000..09001e2 --- /dev/null +++ b/http.c @@ -0,0 +1,330 @@ +#include "http.h" +#include "server.h" +#include "worker.h" +#include "client.h" + +#include +#include +#include +#include + +/* HTTP Response */ + +struct http_response * +http_response_init(struct worker *w, int code, const char *msg) { + + /* create object */ + struct http_response *r = calloc(1, sizeof(struct http_response)); + + r->code = code; + r->msg = msg; + r->w = w; + r->keep_alive = 0; /* default */ + + http_response_set_header(r, "Server", "Webdis"); + + /* Cross-Origin Resource Sharing, CORS. */ + http_response_set_header(r, "Allow", "GET,POST,PUT,OPTIONS"); + /* + Chrome doesn't support Allow and requires + Access-Control-Allow-Methods + */ + http_response_set_header(r, "Access-Control-Allow-Methods", "GET,POST,PUT,OPTIONS"); + http_response_set_header(r, "Access-Control-Allow-Origin", "*"); + /* + According to + http://www.w3.org/TR/cors/#access-control-allow-headers-response-header + Access-Control-Allow-Headers cannot be a wildcard and must be set + with explicit names + */ + http_response_set_header(r, "Access-Control-Allow-Headers", "X-Requested-With, Content-Type, Authorization"); + + return r; +} + + +void +http_response_set_header(struct http_response *r, const char *k, const char *v) { + + int i, pos = r->header_count; + size_t key_sz = strlen(k); + size_t val_sz = strlen(v); + + for(i = 0; i < r->header_count; ++i) { + if(strncmp(r->headers[i].key, k, key_sz) == 0) { + pos = i; + /* free old value before replacing it. */ + free(r->headers[i].key); + free(r->headers[i].val); + break; + } + } + + /* extend array */ + if(pos == r->header_count) { + r->headers = realloc(r->headers, + sizeof(struct http_header)*(r->header_count + 1)); + r->header_count++; + } + + /* copy key */ + r->headers[pos].key = calloc(key_sz + 1, 1); + memcpy(r->headers[pos].key, k, key_sz); + r->headers[pos].key_sz = key_sz; + + /* copy val */ + r->headers[pos].val = calloc(val_sz + 1, 1); + memcpy(r->headers[pos].val, v, val_sz); + r->headers[pos].val_sz = val_sz; + + if(!r->chunked && !strcmp(k, "Transfer-Encoding") && !strcmp(v, "chunked")) { + r->chunked = 1; + } +} + +void +http_response_set_body(struct http_response *r, const char *body, size_t body_len) { + + r->body = body; + r->body_len = body_len; +} + +static void +http_response_cleanup(struct http_response *r, int fd, int success) { + + int i; + + /* cleanup buffer */ + free(r->out); + if(!r->keep_alive || !success) { + /* Close fd is client doesn't support Keep-Alive. */ + close(fd); + } + + /* cleanup response object */ + for(i = 0; i < r->header_count; ++i) { + free(r->headers[i].key); + free(r->headers[i].val); + } + free(r->headers); + + free(r); +} + +static void +http_can_write(int fd, short event, void *p) { + + int ret; + struct http_response *r = p; + + (void)event; + + ret = write(fd, r->out + r->sent, r->out_sz - r->sent); + + if(ret > 0) + r->sent += ret; + + if(ret <= 0 || r->out_sz - r->sent == 0) { /* error or done */ + http_response_cleanup(r, fd, (int)r->out_sz == r->sent ? 1 : 0); + } else { /* reschedule write */ + http_schedule_write(fd, r); + } +} + +void +http_schedule_write(int fd, struct http_response *r) { + + if(r->w) { /* async */ + event_set(&r->ev, fd, EV_WRITE, http_can_write, r); + event_base_set(r->w->base, &r->ev); + event_add(&r->ev, NULL); + } else { /* blocking */ + http_can_write(fd, 0, r); + } + +} + +static char * +format_chunk(const char *p, size_t sz, size_t *out_sz) { + + char *out, tmp[64]; + int chunk_size; + + /* calculate format size */ + chunk_size = sprintf(tmp, "%x\r\n", (int)sz); + + *out_sz = chunk_size + sz + 2; + out = malloc(*out_sz); + memcpy(out, tmp, chunk_size); + memcpy(out + chunk_size, p, sz); + memcpy(out + chunk_size + sz, "\r\n", 2); + + return out; +} + +void +http_response_write(struct http_response *r, int fd) { + + char *p; + int i, ret; + + /*r->keep_alive = 0;*/ + r->out_sz = sizeof("HTTP/1.x xxx ")-1 + strlen(r->msg) + 2; + r->out = calloc(r->out_sz + 1, 1); + + ret = sprintf(r->out, "HTTP/1.%d %d %s\r\n", (r->http_version?1:0), r->code, r->msg); + (void)ret; + p = r->out; + + if(!r->chunked) { + if(r->code == 200 && r->body) { + char content_length[10]; + sprintf(content_length, "%zd", r->body_len); + http_response_set_header(r, "Content-Length", content_length); + } else { + http_response_set_header(r, "Content-Length", "0"); + } + } + + for(i = 0; i < r->header_count; ++i) { + /* "Key: Value\r\n" */ + size_t header_sz = r->headers[i].key_sz + 2 + r->headers[i].val_sz + 2; + r->out = realloc(r->out, r->out_sz + header_sz); + p = r->out + r->out_sz; + + /* add key */ + memcpy(p, r->headers[i].key, r->headers[i].key_sz); + p += r->headers[i].key_sz; + + /* add ": " */ + *(p++) = ':'; + *(p++) = ' '; + + /* add value */ + memcpy(p, r->headers[i].val, r->headers[i].val_sz); + p += r->headers[i].val_sz; + + /* add "\r\n" */ + *(p++) = '\r'; + *(p++) = '\n'; + + r->out_sz += header_sz; + + if(strncasecmp("Connection", r->headers[i].key, r->headers[i].key_sz) == 0 && + strncasecmp("Keep-Alive", r->headers[i].val, r->headers[i].val_sz) == 0) { + r->keep_alive = 1; + } + } + + /* end of headers */ + r->out = realloc(r->out, r->out_sz + 2); + memcpy(r->out + r->out_sz, "\r\n", 2); + r->out_sz += 2; + + /* append body if there is one. */ + if(r->body && r->body_len) { + + char *tmp = (char*)r->body; + size_t tmp_len = r->body_len; + if(r->chunked) { /* replace body with formatted chunk */ + tmp = format_chunk(r->body, r->body_len, &tmp_len); + } + + r->out = realloc(r->out, r->out_sz + tmp_len); + memcpy(r->out + r->out_sz, tmp, tmp_len); + r->out_sz += tmp_len; + + if(r->chunked) { /* need to free the chunk */ + free(tmp); + } + } + + /* send buffer to client */ + r->sent = 0; + http_schedule_write(fd, r); +} + +static void +http_response_set_connection_header(struct http_client *c, struct http_response *r) { + http_response_set_keep_alive(r, c->keep_alive); +} + + + +/* Adobe flash cross-domain request */ +void +http_crossdomain(struct http_client *c) { + + struct http_response *resp = http_response_init(NULL, 200, "OK"); + char out[] = "\n" +"\n" +"\n" + "\n" +"\n"; + + resp->http_version = c->http_version; + http_response_set_connection_header(c, resp); + http_response_set_header(resp, "Content-Type", "application/xml"); + http_response_set_body(resp, out, sizeof(out)-1); + + http_response_write(resp, c->fd); + http_client_reset(c); +} + +/* Simple error response */ +void +http_send_error(struct http_client *c, short code, const char *msg) { + + struct http_response *resp = http_response_init(NULL, code, msg); + resp->http_version = c->http_version; + http_response_set_connection_header(c, resp); + http_response_set_body(resp, NULL, 0); + + http_response_write(resp, c->fd); + http_client_reset(c); +} + +/** + * Set Connection field, either Keep-Alive or Close. + */ +void +http_response_set_keep_alive(struct http_response *r, int enabled) { + r->keep_alive = enabled; + if(enabled) { + http_response_set_header(r, "Connection", "Keep-Alive"); + } else { + http_response_set_header(r, "Connection", "Close"); + } +} + +/* Response to HTTP OPTIONS */ +void +http_send_options(struct http_client *c) { + + struct http_response *resp = http_response_init(NULL, 200, "OK"); + resp->http_version = c->http_version; + http_response_set_connection_header(c, resp); + + http_response_set_header(resp, "Content-Type", "text/html"); + http_response_set_header(resp, "Content-Length", "0"); + + http_response_write(resp, c->fd); + http_client_reset(c); +} + +/** + * Write HTTP chunk. + */ +void +http_response_write_chunk(int fd, struct worker *w, const char *p, size_t sz) { + + struct http_response *r = http_response_init(w, 0, NULL); + r->keep_alive = 1; /* chunks are always keep-alive */ + + /* format packet */ + r->out = format_chunk(p, sz, &r->out_sz); + + /* send async write */ + http_schedule_write(fd, r); +} + diff --git a/http.h b/http.h new file mode 100644 index 0000000..eac1ffe --- /dev/null +++ b/http.h @@ -0,0 +1,75 @@ +#ifndef HTTP_H +#define HTTP_H + +#include +#include + +struct http_client; +struct worker; + +struct http_header { + char *key; + size_t key_sz; + + char *val; + size_t val_sz; +}; + + +struct http_response { + + struct event ev; + + short code; + const char *msg; + + struct http_header *headers; + int header_count; + + const char *body; + size_t body_len; + + char *out; + size_t out_sz; + + int chunked; + int http_version; + int keep_alive; + int sent; + + struct worker *w; +}; + +/* HTTP response */ + +struct http_response * +http_response_init(struct worker *w, int code, const char *msg); + +void +http_response_set_header(struct http_response *r, const char *k, const char *v); + +void +http_response_set_body(struct http_response *r, const char *body, size_t body_len); + +void +http_response_write(struct http_response *r, int fd); + +void +http_schedule_write(int fd, struct http_response *r); + +void +http_crossdomain(struct http_client *c); + +void +http_send_error(struct http_client *c, short code, const char *msg); + +void +http_send_options(struct http_client *c); + +void +http_response_write_chunk(int fd, struct worker *w, const char *p, size_t sz); + +void +http_response_set_keep_alive(struct http_response *r, int enabled); + +#endif diff --git a/md5/md5.c b/md5/md5.c new file mode 100644 index 0000000..c35d96c --- /dev/null +++ b/md5/md5.c @@ -0,0 +1,381 @@ +/* + Copyright (C) 1999, 2000, 2002 Aladdin Enterprises. All rights reserved. + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + L. Peter Deutsch + ghost@aladdin.com + + */ +/* $Id: md5.c,v 1.6 2002/04/13 19:20:28 lpd Exp $ */ +/* + Independent implementation of MD5 (RFC 1321). + + This code implements the MD5 Algorithm defined in RFC 1321, whose + text is available at + http://www.ietf.org/rfc/rfc1321.txt + The code is derived from the text of the RFC, including the test suite + (section A.5) but excluding the rest of Appendix A. It does not include + any code or documentation that is identified in the RFC as being + copyrighted. + + The original and principal author of md5.c is L. Peter Deutsch + . Other authors are noted in the change history + that follows (in reverse chronological order): + + 2002-04-13 lpd Clarified derivation from RFC 1321; now handles byte order + either statically or dynamically; added missing #include + in library. + 2002-03-11 lpd Corrected argument list for main(), and added int return + type, in test program and T value program. + 2002-02-21 lpd Added missing #include in test program. + 2000-07-03 lpd Patched to eliminate warnings about "constant is + unsigned in ANSI C, signed in traditional"; made test program + self-checking. + 1999-11-04 lpd Edited comments slightly for automatic TOC extraction. + 1999-10-18 lpd Fixed typo in header comment (ansi2knr rather than md5). + 1999-05-03 lpd Original version. + */ + +#include "md5.h" +#include + +#undef BYTE_ORDER /* 1 = big-endian, -1 = little-endian, 0 = unknown */ +#ifdef ARCH_IS_BIG_ENDIAN +# define BYTE_ORDER (ARCH_IS_BIG_ENDIAN ? 1 : -1) +#else +# define BYTE_ORDER 0 +#endif + +#define T_MASK ((md5_word_t)~0) +#define T1 /* 0xd76aa478 */ (T_MASK ^ 0x28955b87) +#define T2 /* 0xe8c7b756 */ (T_MASK ^ 0x173848a9) +#define T3 0x242070db +#define T4 /* 0xc1bdceee */ (T_MASK ^ 0x3e423111) +#define T5 /* 0xf57c0faf */ (T_MASK ^ 0x0a83f050) +#define T6 0x4787c62a +#define T7 /* 0xa8304613 */ (T_MASK ^ 0x57cfb9ec) +#define T8 /* 0xfd469501 */ (T_MASK ^ 0x02b96afe) +#define T9 0x698098d8 +#define T10 /* 0x8b44f7af */ (T_MASK ^ 0x74bb0850) +#define T11 /* 0xffff5bb1 */ (T_MASK ^ 0x0000a44e) +#define T12 /* 0x895cd7be */ (T_MASK ^ 0x76a32841) +#define T13 0x6b901122 +#define T14 /* 0xfd987193 */ (T_MASK ^ 0x02678e6c) +#define T15 /* 0xa679438e */ (T_MASK ^ 0x5986bc71) +#define T16 0x49b40821 +#define T17 /* 0xf61e2562 */ (T_MASK ^ 0x09e1da9d) +#define T18 /* 0xc040b340 */ (T_MASK ^ 0x3fbf4cbf) +#define T19 0x265e5a51 +#define T20 /* 0xe9b6c7aa */ (T_MASK ^ 0x16493855) +#define T21 /* 0xd62f105d */ (T_MASK ^ 0x29d0efa2) +#define T22 0x02441453 +#define T23 /* 0xd8a1e681 */ (T_MASK ^ 0x275e197e) +#define T24 /* 0xe7d3fbc8 */ (T_MASK ^ 0x182c0437) +#define T25 0x21e1cde6 +#define T26 /* 0xc33707d6 */ (T_MASK ^ 0x3cc8f829) +#define T27 /* 0xf4d50d87 */ (T_MASK ^ 0x0b2af278) +#define T28 0x455a14ed +#define T29 /* 0xa9e3e905 */ (T_MASK ^ 0x561c16fa) +#define T30 /* 0xfcefa3f8 */ (T_MASK ^ 0x03105c07) +#define T31 0x676f02d9 +#define T32 /* 0x8d2a4c8a */ (T_MASK ^ 0x72d5b375) +#define T33 /* 0xfffa3942 */ (T_MASK ^ 0x0005c6bd) +#define T34 /* 0x8771f681 */ (T_MASK ^ 0x788e097e) +#define T35 0x6d9d6122 +#define T36 /* 0xfde5380c */ (T_MASK ^ 0x021ac7f3) +#define T37 /* 0xa4beea44 */ (T_MASK ^ 0x5b4115bb) +#define T38 0x4bdecfa9 +#define T39 /* 0xf6bb4b60 */ (T_MASK ^ 0x0944b49f) +#define T40 /* 0xbebfbc70 */ (T_MASK ^ 0x4140438f) +#define T41 0x289b7ec6 +#define T42 /* 0xeaa127fa */ (T_MASK ^ 0x155ed805) +#define T43 /* 0xd4ef3085 */ (T_MASK ^ 0x2b10cf7a) +#define T44 0x04881d05 +#define T45 /* 0xd9d4d039 */ (T_MASK ^ 0x262b2fc6) +#define T46 /* 0xe6db99e5 */ (T_MASK ^ 0x1924661a) +#define T47 0x1fa27cf8 +#define T48 /* 0xc4ac5665 */ (T_MASK ^ 0x3b53a99a) +#define T49 /* 0xf4292244 */ (T_MASK ^ 0x0bd6ddbb) +#define T50 0x432aff97 +#define T51 /* 0xab9423a7 */ (T_MASK ^ 0x546bdc58) +#define T52 /* 0xfc93a039 */ (T_MASK ^ 0x036c5fc6) +#define T53 0x655b59c3 +#define T54 /* 0x8f0ccc92 */ (T_MASK ^ 0x70f3336d) +#define T55 /* 0xffeff47d */ (T_MASK ^ 0x00100b82) +#define T56 /* 0x85845dd1 */ (T_MASK ^ 0x7a7ba22e) +#define T57 0x6fa87e4f +#define T58 /* 0xfe2ce6e0 */ (T_MASK ^ 0x01d3191f) +#define T59 /* 0xa3014314 */ (T_MASK ^ 0x5cfebceb) +#define T60 0x4e0811a1 +#define T61 /* 0xf7537e82 */ (T_MASK ^ 0x08ac817d) +#define T62 /* 0xbd3af235 */ (T_MASK ^ 0x42c50dca) +#define T63 0x2ad7d2bb +#define T64 /* 0xeb86d391 */ (T_MASK ^ 0x14792c6e) + + +static void +md5_process(md5_state_t *pms, const md5_byte_t *data /*[64]*/) +{ + md5_word_t + a = pms->abcd[0], b = pms->abcd[1], + c = pms->abcd[2], d = pms->abcd[3]; + md5_word_t t; +#if BYTE_ORDER > 0 + /* Define storage only for big-endian CPUs. */ + md5_word_t X[16]; +#else + /* Define storage for little-endian or both types of CPUs. */ + md5_word_t xbuf[16]; + const md5_word_t *X; +#endif + + { +#if BYTE_ORDER == 0 + /* + * Determine dynamically whether this is a big-endian or + * little-endian machine, since we can use a more efficient + * algorithm on the latter. + */ + static const int w = 1; + + if (*((const md5_byte_t *)&w)) /* dynamic little-endian */ +#endif +#if BYTE_ORDER <= 0 /* little-endian */ + { + /* + * On little-endian machines, we can process properly aligned + * data without copying it. + */ + if (!((data - (const md5_byte_t *)0) & 3)) { + /* data are properly aligned */ + X = (const md5_word_t *)data; + } else { + /* not aligned */ + memcpy(xbuf, data, 64); + X = xbuf; + } + } +#endif +#if BYTE_ORDER == 0 + else /* dynamic big-endian */ +#endif +#if BYTE_ORDER >= 0 /* big-endian */ + { + /* + * On big-endian machines, we must arrange the bytes in the + * right order. + */ + const md5_byte_t *xp = data; + int i; + +# if BYTE_ORDER == 0 + X = xbuf; /* (dynamic only) */ +# else +# define xbuf X /* (static only) */ +# endif + for (i = 0; i < 16; ++i, xp += 4) + xbuf[i] = xp[0] + (xp[1] << 8) + (xp[2] << 16) + (xp[3] << 24); + } +#endif + } + +#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32 - (n)))) + + /* Round 1. */ + /* Let [abcd k s i] denote the operation + a = b + ((a + F(b,c,d) + X[k] + T[i]) <<< s). */ +#define F(x, y, z) (((x) & (y)) | (~(x) & (z))) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + F(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 0, 7, T1); + SET(d, a, b, c, 1, 12, T2); + SET(c, d, a, b, 2, 17, T3); + SET(b, c, d, a, 3, 22, T4); + SET(a, b, c, d, 4, 7, T5); + SET(d, a, b, c, 5, 12, T6); + SET(c, d, a, b, 6, 17, T7); + SET(b, c, d, a, 7, 22, T8); + SET(a, b, c, d, 8, 7, T9); + SET(d, a, b, c, 9, 12, T10); + SET(c, d, a, b, 10, 17, T11); + SET(b, c, d, a, 11, 22, T12); + SET(a, b, c, d, 12, 7, T13); + SET(d, a, b, c, 13, 12, T14); + SET(c, d, a, b, 14, 17, T15); + SET(b, c, d, a, 15, 22, T16); +#undef SET + + /* Round 2. */ + /* Let [abcd k s i] denote the operation + a = b + ((a + G(b,c,d) + X[k] + T[i]) <<< s). */ +#define G(x, y, z) (((x) & (z)) | ((y) & ~(z))) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + G(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 1, 5, T17); + SET(d, a, b, c, 6, 9, T18); + SET(c, d, a, b, 11, 14, T19); + SET(b, c, d, a, 0, 20, T20); + SET(a, b, c, d, 5, 5, T21); + SET(d, a, b, c, 10, 9, T22); + SET(c, d, a, b, 15, 14, T23); + SET(b, c, d, a, 4, 20, T24); + SET(a, b, c, d, 9, 5, T25); + SET(d, a, b, c, 14, 9, T26); + SET(c, d, a, b, 3, 14, T27); + SET(b, c, d, a, 8, 20, T28); + SET(a, b, c, d, 13, 5, T29); + SET(d, a, b, c, 2, 9, T30); + SET(c, d, a, b, 7, 14, T31); + SET(b, c, d, a, 12, 20, T32); +#undef SET + + /* Round 3. */ + /* Let [abcd k s t] denote the operation + a = b + ((a + H(b,c,d) + X[k] + T[i]) <<< s). */ +#define H(x, y, z) ((x) ^ (y) ^ (z)) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + H(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 5, 4, T33); + SET(d, a, b, c, 8, 11, T34); + SET(c, d, a, b, 11, 16, T35); + SET(b, c, d, a, 14, 23, T36); + SET(a, b, c, d, 1, 4, T37); + SET(d, a, b, c, 4, 11, T38); + SET(c, d, a, b, 7, 16, T39); + SET(b, c, d, a, 10, 23, T40); + SET(a, b, c, d, 13, 4, T41); + SET(d, a, b, c, 0, 11, T42); + SET(c, d, a, b, 3, 16, T43); + SET(b, c, d, a, 6, 23, T44); + SET(a, b, c, d, 9, 4, T45); + SET(d, a, b, c, 12, 11, T46); + SET(c, d, a, b, 15, 16, T47); + SET(b, c, d, a, 2, 23, T48); +#undef SET + + /* Round 4. */ + /* Let [abcd k s t] denote the operation + a = b + ((a + I(b,c,d) + X[k] + T[i]) <<< s). */ +#define I(x, y, z) ((y) ^ ((x) | ~(z))) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + I(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 0, 6, T49); + SET(d, a, b, c, 7, 10, T50); + SET(c, d, a, b, 14, 15, T51); + SET(b, c, d, a, 5, 21, T52); + SET(a, b, c, d, 12, 6, T53); + SET(d, a, b, c, 3, 10, T54); + SET(c, d, a, b, 10, 15, T55); + SET(b, c, d, a, 1, 21, T56); + SET(a, b, c, d, 8, 6, T57); + SET(d, a, b, c, 15, 10, T58); + SET(c, d, a, b, 6, 15, T59); + SET(b, c, d, a, 13, 21, T60); + SET(a, b, c, d, 4, 6, T61); + SET(d, a, b, c, 11, 10, T62); + SET(c, d, a, b, 2, 15, T63); + SET(b, c, d, a, 9, 21, T64); +#undef SET + + /* Then perform the following additions. (That is increment each + of the four registers by the value it had before this block + was started.) */ + pms->abcd[0] += a; + pms->abcd[1] += b; + pms->abcd[2] += c; + pms->abcd[3] += d; +} + +void +md5_init(md5_state_t *pms) +{ + pms->count[0] = pms->count[1] = 0; + pms->abcd[0] = 0x67452301; + pms->abcd[1] = /*0xefcdab89*/ T_MASK ^ 0x10325476; + pms->abcd[2] = /*0x98badcfe*/ T_MASK ^ 0x67452301; + pms->abcd[3] = 0x10325476; +} + +void +md5_append(md5_state_t *pms, const md5_byte_t *data, int nbytes) +{ + const md5_byte_t *p = data; + int left = nbytes; + int offset = (pms->count[0] >> 3) & 63; + md5_word_t nbits = (md5_word_t)(nbytes << 3); + + if (nbytes <= 0) + return; + + /* Update the message length. */ + pms->count[1] += nbytes >> 29; + pms->count[0] += nbits; + if (pms->count[0] < nbits) + pms->count[1]++; + + /* Process an initial partial block. */ + if (offset) { + int copy = (offset + nbytes > 64 ? 64 - offset : nbytes); + + memcpy(pms->buf + offset, p, copy); + if (offset + copy < 64) + return; + p += copy; + left -= copy; + md5_process(pms, pms->buf); + } + + /* Process full blocks. */ + for (; left >= 64; p += 64, left -= 64) + md5_process(pms, p); + + /* Process a final partial block. */ + if (left) + memcpy(pms->buf, p, left); +} + +void +md5_finish(md5_state_t *pms, md5_byte_t digest[16]) +{ + static const md5_byte_t pad[64] = { + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + md5_byte_t data[8]; + int i; + + /* Save the length before padding. */ + for (i = 0; i < 8; ++i) + data[i] = (md5_byte_t)(pms->count[i >> 2] >> ((i & 3) << 3)); + /* Pad to 56 bytes mod 64. */ + md5_append(pms, pad, ((55 - (pms->count[0] >> 3)) & 63) + 1); + /* Append the length. */ + md5_append(pms, data, 8); + for (i = 0; i < 16; ++i) + digest[i] = (md5_byte_t)(pms->abcd[i >> 2] >> ((i & 3) << 3)); +} diff --git a/md5/md5.h b/md5/md5.h new file mode 100644 index 0000000..7143a36 --- /dev/null +++ b/md5/md5.h @@ -0,0 +1,84 @@ +/* + Copyright (C) 1999, 2002 Aladdin Enterprises. All rights reserved. + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + L. Peter Deutsch + ghost@aladdin.com + + */ +/* $Id: md5.h,v 1.4 2002/04/13 19:20:28 lpd Exp $ */ +/* + Independent implementation of MD5 (RFC 1321). + + This code implements the MD5 Algorithm defined in RFC 1321, whose + text is available at + http://www.ietf.org/rfc/rfc1321.txt + The code is derived from the text of the RFC, including the test suite + (section A.5) but excluding the rest of Appendix A. It does not include + any code or documentation that is identified in the RFC as being + copyrighted. + + The original and principal author of md5.h is L. Peter Deutsch + . Other authors are noted in the change history + that follows (in reverse chronological order): + + 2002-04-13 lpd Removed support for non-ANSI compilers; removed + references to Ghostscript; clarified derivation from RFC 1321; + now handles byte order either statically or dynamically. + 1999-11-04 lpd Edited comments slightly for automatic TOC extraction. + 1999-10-18 lpd Fixed typo in header comment (ansi2knr rather than md5); + added conditionalization for C++ compilation from Martin + Purschke . + 1999-05-03 lpd Original version. + */ + +#ifndef md5_INCLUDED +#define md5_INCLUDED + +/* + * This package supports both compile-time and run-time determination of CPU + * byte order. If ARCH_IS_BIG_ENDIAN is defined as 0, the code will be + * compiled to run only on little-endian CPUs; if ARCH_IS_BIG_ENDIAN is + * defined as non-zero, the code will be compiled to run only on big-endian + * CPUs; if ARCH_IS_BIG_ENDIAN is not defined, the code will be compiled to + * run on either big- or little-endian CPUs, but will run slightly less + * efficiently on either one than if ARCH_IS_BIG_ENDIAN is defined. + */ + +typedef unsigned char md5_byte_t; /* 8-bit byte */ +typedef unsigned int md5_word_t; /* 32-bit word */ + +/* Define the state of the MD5 Algorithm. */ +typedef struct md5_state_s { + md5_word_t count[2]; /* message length in bits, lsw first */ + md5_word_t abcd[4]; /* digest buffer */ + md5_byte_t buf[64]; /* accumulate block */ +} md5_state_t; + + +/* Initialize the algorithm. */ +void md5_init(md5_state_t *pms); + +/* Append a string to the message. */ +void md5_append(md5_state_t *pms, const md5_byte_t *data, int nbytes); + +/* Finish the message and return the digest. */ +void md5_finish(md5_state_t *pms, md5_byte_t digest[16]); + + +#endif /* md5_INCLUDED */ diff --git a/pool.c b/pool.c new file mode 100644 index 0000000..e5e29eb --- /dev/null +++ b/pool.c @@ -0,0 +1,181 @@ +#include "pool.h" +#include "worker.h" +#include "conf.h" +#include "server.h" + +#include +#include +#include +#include + +static void +pool_schedule_reconnect(struct pool* p); + +struct pool * +pool_new(struct worker *w, int count) { + + struct pool *p = calloc(1, sizeof(struct pool)); + + p->count = count; + p->ac = calloc(count, sizeof(redisAsyncContext*)); + + p->w = w; + p->cfg = w->s->cfg; + + return p; +} + +void +pool_free_context(redisAsyncContext *ac) { + + if (ac) { + redisAsyncDisconnect(ac); + redisAsyncFree(ac); + } +} + +static void +pool_on_connect(const redisAsyncContext *ac, int status) { + struct pool *p = ac->data; + int i = 0; + + if(!p || status == REDIS_ERR || ac->err) { + if (p) { + pool_schedule_reconnect(p); + } + return; + } + /* connected to redis! */ + + /* add to pool */ + for(i = 0; i < p->count; ++i) { + if(p->ac[i] == NULL) { + p->ac[i] = ac; + return; + } + } +} + +struct pool_reconnect { + struct event ev; + struct pool *p; + + struct timeval tv; +}; + +static void +pool_can_connect(int fd, short event, void *ptr) { + struct pool_reconnect *pr = ptr; + struct pool *p = pr->p; + + (void)fd; + (void)event; + + free(pr); + + pool_connect(p, p->cfg->database, 1); +} +static void +pool_schedule_reconnect(struct pool *p) { + + struct pool_reconnect *pr = malloc(sizeof(struct pool_reconnect)); + pr->p = p; + + pr->tv.tv_sec = 0; + pr->tv.tv_usec = 100*1000; /* 0.1 sec*/ + + evtimer_set(&pr->ev, pool_can_connect, pr); + event_base_set(p->w->base, &pr->ev); + evtimer_add(&pr->ev, &pr->tv); +} + + + +static void +pool_on_disconnect(const redisAsyncContext *ac, int status) { + + struct pool *p = ac->data; + int i = 0; + if (status != REDIS_OK) { + /* fprintf(stderr, "Error: %s\n", ac->errstr); */ + } + + if(p == NULL) { /* no need to clean anything here. */ + return; + } + + /* remove from the pool */ + for(i = 0; i < p->count; ++i) { + if(p->ac[i] == ac) { + p->ac[i] = NULL; + break; + } + } + + /* schedule reconnect */ + pool_schedule_reconnect(p); +} + +/** + * Create new connection. + */ +redisAsyncContext * +pool_connect(struct pool *p, int db_num, int attach) { + + struct redisAsyncContext *ac; + if(p->cfg->redis_host[0] == '/') { /* unix socket */ + ac = redisAsyncConnectUnix(p->cfg->redis_host); + } else { + ac = redisAsyncConnect(p->cfg->redis_host, p->cfg->redis_port); + } + + if(attach) { + ac->data = p; + } else { + ac->data = NULL; + } + + if(ac->err) { + char msg[] = "Connection failed: %s"; + size_t errlen = strlen(ac->errstr); + char *err = malloc(sizeof(msg) + errlen); + if (err) { + size_t sz = sprintf(err, msg, ac->errstr); + slog(p->w->s, WEBDIS_ERROR, err, sz); + free(err); + } + redisAsyncFree(ac); + pool_schedule_reconnect(p); + return NULL; + } + + redisLibeventAttach(ac, p->w->base); + redisAsyncSetConnectCallback(ac, pool_on_connect); + redisAsyncSetDisconnectCallback(ac, pool_on_disconnect); + + if(p->cfg->redis_auth) { /* authenticate. */ + redisAsyncCommand(ac, NULL, NULL, "AUTH %s", p->cfg->redis_auth); + } + if(db_num) { /* change database. */ + redisAsyncCommand(ac, NULL, NULL, "SELECT %d", db_num); + } + return ac; +} + +const redisAsyncContext * +pool_get_context(struct pool *p) { + + int orig = p->cur++; + + do { + p->cur++; + p->cur %= p->count; + if(p->ac[p->cur] != NULL) { + return p->ac[p->cur]; + } + } while(p->cur != orig); + + return NULL; + +} + diff --git a/pool.h b/pool.h new file mode 100644 index 0000000..bd24ee9 --- /dev/null +++ b/pool.h @@ -0,0 +1,33 @@ +#ifndef POOL_H +#define POOL_H + +#include + +struct conf; +struct worker; + +struct pool { + + struct worker *w; + struct conf *cfg; + + const redisAsyncContext **ac; + int count; + int cur; + +}; + + +struct pool * +pool_new(struct worker *w, int count); + +void +pool_free_context(redisAsyncContext *ac); + +redisAsyncContext * +pool_connect(struct pool *p, int db_num, int attach); + +const redisAsyncContext * +pool_get_context(struct pool *p); + +#endif diff --git a/server.c b/server.c new file mode 100644 index 0000000..12ca4a9 --- /dev/null +++ b/server.c @@ -0,0 +1,245 @@ +#include "server.h" +#include "worker.h" +#include "client.h" +#include "conf.h" +#include "version.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * Sets up a non-blocking socket + */ +static int +socket_setup(struct server *s, const char *ip, short port) { + + int reuse = 1; + struct sockaddr_in addr; + int fd, ret; + + memset(&addr, 0, sizeof(addr)); +#if defined __BSD__ + addr.sin_len = sizeof(struct sockaddr_in); +#endif + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + + addr.sin_addr.s_addr = inet_addr(ip); + + /* this sad list of tests could use a Maybe monad... */ + + /* create socket */ + fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (-1 == fd) { + slog(s, WEBDIS_ERROR, strerror(errno), 0); + return -1; + } + + /* reuse address if we've bound to it before. */ + if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &reuse, + sizeof(reuse)) < 0) { + slog(s, WEBDIS_ERROR, strerror(errno), 0); + return -1; + } + + /* set socket as non-blocking. */ + ret = fcntl(fd, F_SETFD, O_NONBLOCK); + if (0 != ret) { + slog(s, WEBDIS_ERROR, strerror(errno), 0); + return -1; + } + + /* bind */ + ret = bind(fd, (struct sockaddr*)&addr, sizeof(addr)); + if (0 != ret) { + slog(s, WEBDIS_ERROR, strerror(errno), 0); + return -1; + } + + /* listen */ + ret = listen(fd, SOMAXCONN); + if (0 != ret) { + slog(s, WEBDIS_ERROR, strerror(errno), 0); + return -1; + } + + /* there you go, ready to accept! */ + return fd; +} + +struct server * +server_new(const char *cfg_file) { + + int i; + struct server *s = calloc(1, sizeof(struct server)); + + s->log.fd = -1; + s->cfg = conf_read(cfg_file); + + /* workers */ + s->w = calloc(s->cfg->http_threads, sizeof(struct worker*)); + for(i = 0; i < s->cfg->http_threads; ++i) { + s->w[i] = worker_new(s); + } + return s; +} + +static void +server_can_accept(int fd, short event, void *ptr) { + + struct server *s = ptr; + struct worker *w; + struct http_client *c; + int client_fd; + struct sockaddr_in addr; + socklen_t addr_sz = sizeof(addr); + char on = 1; + (void)event; + + /* select worker to send the client to */ + w = s->w[s->next_worker]; + + /* accept client */ + client_fd = accept(fd, (struct sockaddr*)&addr, &addr_sz); + + /* make non-blocking */ + ioctl(client_fd, (int)FIONBIO, (char *)&on); + + /* create client and send to worker. */ + if(client_fd > 0) { + c = http_client_new(w, client_fd, addr.sin_addr.s_addr); + worker_add_client(w, c); + + /* loop over ring of workers */ + s->next_worker = (s->next_worker + 1) % s->cfg->http_threads; + } else { /* too many connections */ + slog(s, WEBDIS_NOTICE, "Too many connections", 0); + } +} + +/** + * Daemonize server. + * (taken from Redis) + */ +static void +server_daemonize(const char *pidfile) { + int fd; + + if (fork() != 0) exit(0); /* parent exits */ + setsid(); /* create a new session */ + + /* Every output goes to /dev/null. */ + if ((fd = open("/dev/null", O_RDWR, 0)) != -1) { + dup2(fd, STDIN_FILENO); + dup2(fd, STDOUT_FILENO); + dup2(fd, STDERR_FILENO); + if (fd > STDERR_FILENO) close(fd); + } + + /* write pidfile */ + if (pidfile) { + FILE *f = fopen(pidfile, "w"); + if (f) { + fprintf(f, "%d\n", (int)getpid()); + fclose(f); + } + } +} + +/* global pointer to the server object, used in signal handlers */ +static struct server *__server; + +static void +server_handle_signal(int id) { + + switch(id) { + case SIGHUP: + slog_init(__server); + break; + case SIGTERM: + case SIGINT: + slog(__server, WEBDIS_INFO, "Webdis terminating", 0); + exit(0); + break; + default: + break; + } +} + +static void +server_install_signal_handlers(struct server *s) { + __server = s; + + signal(SIGHUP, server_handle_signal); + signal(SIGTERM, server_handle_signal); + signal(SIGINT, server_handle_signal); +} + +int +server_start(struct server *s) { + + int i, ret; + + /* initialize libevent */ + s->base = event_base_new(); + + if(s->cfg->daemonize) { + server_daemonize(s->cfg->pidfile); + + /* sometimes event mech gets lost on fork */ + if(event_reinit(s->base) != 0) { + fprintf(stderr, "Error: event_reinit failed after fork"); + } + } + + /* ignore sigpipe */ +#ifdef SIGPIPE + signal(SIGPIPE, SIG_IGN); +#endif + + slog_init(s); + + /* install signal handlers */ + server_install_signal_handlers(s); + + /* start worker threads */ + for(i = 0; i < s->cfg->http_threads; ++i) { + worker_start(s->w[i]); + } + + /* create socket */ + s->fd = socket_setup(s, s->cfg->http_host, s->cfg->http_port); + if(s->fd < 0) { + return -1; + } + + /*set keepalive socket option to do with half connection*/ + int keep_alive = 1; + setsockopt(s->fd , SOL_SOCKET, SO_KEEPALIVE, (void*)&keep_alive, sizeof(keep_alive)); + + /* start http server */ + event_set(&s->ev, s->fd, EV_READ | EV_PERSIST, server_can_accept, s); + event_base_set(s->base, &s->ev); + ret = event_add(&s->ev, NULL); + + if(ret < 0) { + slog(s, WEBDIS_ERROR, "Error calling event_add on socket", 0); + return -1; + } + + slog(s, WEBDIS_INFO, "Webdis " WEBDIS_VERSION " up and running", 0); + event_base_dispatch(s->base); + + return 0; +} + diff --git a/server.h b/server.h new file mode 100644 index 0000000..42438b3 --- /dev/null +++ b/server.h @@ -0,0 +1,37 @@ +#ifndef SERVER_H +#define SERVER_H + +#include +#include +#include + +struct worker; +struct conf; + +struct server { + + int fd; + struct event ev; + struct event_base *base; + + struct conf *cfg; + + /* worker threads */ + struct worker **w; + int next_worker; + + /* log lock */ + struct { + pid_t self; + int fd; + } log; +}; + +struct server * +server_new(const char *cfg_file); + +int +server_start(struct server *s); + +#endif + diff --git a/sha1/sha1.c b/sha1/sha1.c new file mode 100644 index 0000000..d87c7f4 --- /dev/null +++ b/sha1/sha1.c @@ -0,0 +1,371 @@ +/* + * sha1.c + * + * Copyright (C) 1998, 2009 + * Paul E. Jones + * All Rights Reserved + * + ***************************************************************************** + * $Id: sha1.c 12 2009-06-22 19:34:25Z paulej $ + ***************************************************************************** + * + * Description: + * This file implements the Secure Hashing Standard as defined + * in FIPS PUB 180-1 published April 17, 1995. + * + * The Secure Hashing Standard, which uses the Secure Hashing + * Algorithm (SHA), produces a 160-bit message digest for a + * given data stream. In theory, it is highly improbable that + * two messages will produce the same message digest. Therefore, + * this algorithm can serve as a means of providing a "fingerprint" + * for a message. + * + * Portability Issues: + * SHA-1 is defined in terms of 32-bit "words". This code was + * written with the expectation that the processor has at least + * a 32-bit machine word size. If the machine word size is larger, + * the code should still function properly. One caveat to that + * is that the input functions taking characters and character + * arrays assume that only 8 bits of information are stored in each + * character. + * + * Caveats: + * SHA-1 is designed to work with messages less than 2^64 bits + * long. Although SHA-1 allows a message digest to be generated for + * messages of any number of bits less than 2^64, this + * implementation only works with messages with a length that is a + * multiple of the size of an 8-bit character. + * + */ + +#include "sha1.h" + +/* + * Define the circular shift macro + */ +#define SHA1CircularShift(bits,word) \ + ((((word) << (bits)) & 0xFFFFFFFF) | \ + ((word) >> (32-(bits)))) + +/* Function prototypes */ +void SHA1ProcessMessageBlock(SHA1Context *); +void SHA1PadMessage(SHA1Context *); + +/* + * SHA1Reset + * + * Description: + * This function will initialize the SHA1Context in preparation + * for computing a new message digest. + * + * Parameters: + * context: [in/out] + * The context to reset. + * + * Returns: + * Nothing. + * + * Comments: + * + */ +void SHA1Reset(SHA1Context *context) +{ + context->Length_Low = 0; + context->Length_High = 0; + context->Message_Block_Index = 0; + + context->Message_Digest[0] = 0x67452301; + context->Message_Digest[1] = 0xEFCDAB89; + context->Message_Digest[2] = 0x98BADCFE; + context->Message_Digest[3] = 0x10325476; + context->Message_Digest[4] = 0xC3D2E1F0; + + context->Computed = 0; + context->Corrupted = 0; +} + +/* + * SHA1Result + * + * Description: + * This function will return the 160-bit message digest into the + * Message_Digest array within the SHA1Context provided + * + * Parameters: + * context: [in/out] + * The context to use to calculate the SHA-1 hash. + * + * Returns: + * 1 if successful, 0 if it failed. + * + * Comments: + * + */ +int SHA1Result(SHA1Context *context) +{ + + if (context->Corrupted) + { + return 0; + } + + if (!context->Computed) + { + SHA1PadMessage(context); + context->Computed = 1; + } + + return 1; +} + +/* + * SHA1Input + * + * Description: + * This function accepts an array of octets as the next portion of + * the message. + * + * Parameters: + * context: [in/out] + * The SHA-1 context to update + * message_array: [in] + * An array of characters representing the next portion of the + * message. + * length: [in] + * The length of the message in message_array + * + * Returns: + * Nothing. + * + * Comments: + * + */ +void SHA1Input( SHA1Context *context, + const unsigned char *message_array, + unsigned length) +{ + if (!length) + { + return; + } + + if (context->Computed || context->Corrupted) + { + context->Corrupted = 1; + return; + } + + while(length-- && !context->Corrupted) + { + context->Message_Block[context->Message_Block_Index++] = + (*message_array & 0xFF); + + context->Length_Low += 8; + /* Force it to 32 bits */ + context->Length_Low &= 0xFFFFFFFF; + if (context->Length_Low == 0) + { + context->Length_High++; + /* Force it to 32 bits */ + context->Length_High &= 0xFFFFFFFF; + if (context->Length_High == 0) + { + /* Message is too long */ + context->Corrupted = 1; + } + } + + if (context->Message_Block_Index == 64) + { + SHA1ProcessMessageBlock(context); + } + + message_array++; + } +} + +/* + * SHA1ProcessMessageBlock + * + * Description: + * This function will process the next 512 bits of the message + * stored in the Message_Block array. + * + * Parameters: + * None. + * + * Returns: + * Nothing. + * + * Comments: + * Many of the variable names in the SHAContext, especially the + * single character names, were used because those were the names + * used in the publication. + * + * + */ +void SHA1ProcessMessageBlock(SHA1Context *context) +{ + const unsigned K[] = /* Constants defined in SHA-1 */ + { + 0x5A827999, + 0x6ED9EBA1, + 0x8F1BBCDC, + 0xCA62C1D6 + }; + int t; /* Loop counter */ + unsigned temp; /* Temporary word value */ + unsigned W[80]; /* Word sequence */ + unsigned A, B, C, D, E; /* Word buffers */ + + /* + * Initialize the first 16 words in the array W + */ + for(t = 0; t < 16; t++) + { + W[t] = ((unsigned) context->Message_Block[t * 4]) << 24; + W[t] |= ((unsigned) context->Message_Block[t * 4 + 1]) << 16; + W[t] |= ((unsigned) context->Message_Block[t * 4 + 2]) << 8; + W[t] |= ((unsigned) context->Message_Block[t * 4 + 3]); + } + + for(t = 16; t < 80; t++) + { + W[t] = SHA1CircularShift(1,W[t-3] ^ W[t-8] ^ W[t-14] ^ W[t-16]); + } + + A = context->Message_Digest[0]; + B = context->Message_Digest[1]; + C = context->Message_Digest[2]; + D = context->Message_Digest[3]; + E = context->Message_Digest[4]; + + for(t = 0; t < 20; t++) + { + temp = SHA1CircularShift(5,A) + + ((B & C) | ((~B) & D)) + E + W[t] + K[0]; + temp &= 0xFFFFFFFF; + E = D; + D = C; + C = SHA1CircularShift(30,B); + B = A; + A = temp; + } + + for(t = 20; t < 40; t++) + { + temp = SHA1CircularShift(5,A) + (B ^ C ^ D) + E + W[t] + K[1]; + temp &= 0xFFFFFFFF; + E = D; + D = C; + C = SHA1CircularShift(30,B); + B = A; + A = temp; + } + + for(t = 40; t < 60; t++) + { + temp = SHA1CircularShift(5,A) + + ((B & C) | (B & D) | (C & D)) + E + W[t] + K[2]; + temp &= 0xFFFFFFFF; + E = D; + D = C; + C = SHA1CircularShift(30,B); + B = A; + A = temp; + } + + for(t = 60; t < 80; t++) + { + temp = SHA1CircularShift(5,A) + (B ^ C ^ D) + E + W[t] + K[3]; + temp &= 0xFFFFFFFF; + E = D; + D = C; + C = SHA1CircularShift(30,B); + B = A; + A = temp; + } + + context->Message_Digest[0] = + (context->Message_Digest[0] + A) & 0xFFFFFFFF; + context->Message_Digest[1] = + (context->Message_Digest[1] + B) & 0xFFFFFFFF; + context->Message_Digest[2] = + (context->Message_Digest[2] + C) & 0xFFFFFFFF; + context->Message_Digest[3] = + (context->Message_Digest[3] + D) & 0xFFFFFFFF; + context->Message_Digest[4] = + (context->Message_Digest[4] + E) & 0xFFFFFFFF; + + context->Message_Block_Index = 0; +} + +/* + * SHA1PadMessage + * + * Description: + * According to the standard, the message must be padded to an even + * 512 bits. The first padding bit must be a '1'. The last 64 + * bits represent the length of the original message. All bits in + * between should be 0. This function will pad the message + * according to those rules by filling the Message_Block array + * accordingly. It will also call SHA1ProcessMessageBlock() + * appropriately. When it returns, it can be assumed that the + * message digest has been computed. + * + * Parameters: + * context: [in/out] + * The context to pad + * + * Returns: + * Nothing. + * + * Comments: + * + */ +void SHA1PadMessage(SHA1Context *context) +{ + /* + * Check to see if the current message block is too small to hold + * the initial padding bits and length. If so, we will pad the + * block, process it, and then continue padding into a second + * block. + */ + if (context->Message_Block_Index > 55) + { + context->Message_Block[context->Message_Block_Index++] = 0x80; + while(context->Message_Block_Index < 64) + { + context->Message_Block[context->Message_Block_Index++] = 0; + } + + SHA1ProcessMessageBlock(context); + + while(context->Message_Block_Index < 56) + { + context->Message_Block[context->Message_Block_Index++] = 0; + } + } + else + { + context->Message_Block[context->Message_Block_Index++] = 0x80; + while(context->Message_Block_Index < 56) + { + context->Message_Block[context->Message_Block_Index++] = 0; + } + } + + /* + * Store the message length as the last 8 octets + */ + context->Message_Block[56] = (context->Length_High >> 24) & 0xFF; + context->Message_Block[57] = (context->Length_High >> 16) & 0xFF; + context->Message_Block[58] = (context->Length_High >> 8) & 0xFF; + context->Message_Block[59] = (context->Length_High) & 0xFF; + context->Message_Block[60] = (context->Length_Low >> 24) & 0xFF; + context->Message_Block[61] = (context->Length_Low >> 16) & 0xFF; + context->Message_Block[62] = (context->Length_Low >> 8) & 0xFF; + context->Message_Block[63] = (context->Length_Low) & 0xFF; + + SHA1ProcessMessageBlock(context); +} diff --git a/sha1/sha1.h b/sha1/sha1.h new file mode 100644 index 0000000..1ca4b10 --- /dev/null +++ b/sha1/sha1.h @@ -0,0 +1,54 @@ +/* + * sha1.h + * + * Copyright (C) 1998, 2009 + * Paul E. Jones + * All Rights Reserved + * + ***************************************************************************** + * $Id: sha1.h 12 2009-06-22 19:34:25Z paulej $ + ***************************************************************************** + * + * Description: + * This class implements the Secure Hashing Standard as defined + * in FIPS PUB 180-1 published April 17, 1995. + * + * Many of the variable names in the SHA1Context, especially the + * single character names, were used because those were the names + * used in the publication. + * + * Please read the file sha1.c for more information. + * + */ + +#ifndef _SHA1_H_ +#define _SHA1_H_ + +/* + * This structure will hold context information for the hashing + * operation + */ +typedef struct SHA1Context +{ + unsigned Message_Digest[5]; /* Message Digest (output) */ + + unsigned Length_Low; /* Message length in bits */ + unsigned Length_High; /* Message length in bits */ + + unsigned char Message_Block[64]; /* 512-bit message blocks */ + int Message_Block_Index; /* Index into message block array */ + + int Computed; /* Is the digest computed? */ + int Corrupted; /* Is the message digest corruped? */ +} SHA1Context; + +/* + * Function Prototypes + */ +void SHA1Reset(SHA1Context *); +int SHA1Result(SHA1Context *); +void SHA1Input( SHA1Context *, + const unsigned char *, + unsigned); + +#endif diff --git a/slog.c b/slog.c new file mode 100644 index 0000000..0d7fd21 --- /dev/null +++ b/slog.c @@ -0,0 +1,80 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "slog.h" +#include "server.h" +#include "conf.h" + +/** + * Initialize log writer. + */ +void +slog_init(struct server *s) { + + s->log.self = getpid(); + + if(s->cfg->logfile) { + + int old_fd = s->log.fd; + + s->log.fd = open(s->cfg->logfile, + O_WRONLY | O_APPEND | O_CREAT, S_IRUSR|S_IWUSR); + + /* close old log */ + if (old_fd != -1) { + close(old_fd); + } + + if (s->log.fd != -1) + return; + + fprintf(stderr, "Could not open %s: %s\n", s->cfg->logfile, + strerror(errno)); + } + s->log.fd = 2; /* stderr */ +} + +/** + * Write log message to disk, or stderr. + */ +void +slog(struct server *s, log_level level, + const char *body, size_t sz) { + + const char *c = ".-*#"; + time_t now; + char time_buf[64]; + char msg[124]; + char line[256]; /* bounds are checked. */ + int line_sz, ret; + + if(level > s->cfg->verbosity) return; /* too verbose */ + + if(!s->log.fd) return; + + /* limit message size */ + sz = sz ? sz:strlen(body); + snprintf(msg, sz + 1 > sizeof(msg) ? sizeof(msg) : sz + 1, "%s", body); + + /* get current time */ + now = time(NULL); + strftime(time_buf, sizeof(time_buf), "%d %b %H:%M:%S", localtime(&now)); + + /* generate output line. */ + line_sz = snprintf(line, sizeof(line), + "[%d] %s %d %s\n", (int)s->log.self, time_buf, c[level], msg); + + /* write to log and flush to disk. */ + ret = write(s->log.fd, line, line_sz); + ret = fsync(s->log.fd); + + (void)ret; +} diff --git a/slog.h b/slog.h new file mode 100644 index 0000000..25d0d89 --- /dev/null +++ b/slog.h @@ -0,0 +1,23 @@ +#ifndef SLOG_H +#define SLOG_H + +typedef enum { + WEBDIS_ERROR = 0, + WEBDIS_WARNING, + WEBDIS_NOTICE, + WEBDIS_INFO, + WEBDIS_DEBUG +} log_level; + +struct server; + +void +slog_reload(); + +void +slog_init(struct server *s); + +void slog(struct server *s, log_level level, + const char *body, size_t sz); + +#endif diff --git a/tests/Makefile b/tests/Makefile new file mode 100644 index 0000000..a0b8a11 --- /dev/null +++ b/tests/Makefile @@ -0,0 +1,18 @@ +OUT=websocket pubsub +CFLAGS=-O3 -Wall -Wextra +LDFLAGS=-levent -lpthread -lrt + +all: $(OUT) Makefile + +websocket: websocket.o + $(CC) -o $@ $< $(LDFLAGS) + +pubsub: pubsub.o + $(CC) -o $@ $< $(LDFLAGS) + +%.o: %.c Makefile + $(CC) -c $(CFLAGS) -o $@ $< + +clean: + rm -f *.o $(OUT) + diff --git a/tests/README.tests b/tests/README.tests new file mode 100644 index 0000000..b47cd80 --- /dev/null +++ b/tests/README.tests @@ -0,0 +1,6 @@ +This directory contains a few test programs for Webdis: + +* basic.py: Unit tests. +* bench.sh: Benchmark of several functions. +* pubsub (run `make' to compile): Tests pub/sub channels; run `./pubsub -h` for options. +* websocket (run `make' to compile): Tests HTML5 WebSockets; run `./websocket -h` for options. diff --git a/tests/basic.py b/tests/basic.py new file mode 100755 index 0000000..89d664e --- /dev/null +++ b/tests/basic.py @@ -0,0 +1,227 @@ +#!/usr/bin/python +import urllib2, unittest, json, hashlib +from functools import wraps +try: + import msgpack +except: + msgpack = None + +import os +host = os.getenv('WEBDIS_HOST', '127.0.0.1') +port = int(os.getenv('WEBDIS_PORT', 7379)) + +class TestWebdis(unittest.TestCase): + + def wrap(self,url): + return 'http://%s:%d/%s' % (host, port, url) + + def query(self, url, data = None, headers={}): + r = urllib2.Request(self.wrap(url), data, headers) + return urllib2.urlopen(r) + +class TestBasics(TestWebdis): + + def test_crossdomain(self): + f = self.query('crossdomain.xml') + self.assertTrue(f.headers.getheader('Content-Type') == 'application/xml') + self.assertTrue("allow-access-from domain" in f.read()) + + def test_options(self): + pass + # not sure if OPTIONS is supported by urllib2... + # f = self.query('') # TODO: call with OPTIONS. + # self.assertTrue(f.headers.getheader('Content-Type') == 'text/html') + # self.assertTrue(f.headers.getheader('Allow') == 'GET,POST,PUT,OPTIONS') + # self.assertTrue(f.headers.getheader('Content-Length') == '0') + # self.assertTrue(f.headers.getheader('Access-Control-Allow-Origin') == '*') + + +class TestJSON(TestWebdis): + + def test_set(self): + "success type (+OK)" + self.query('DEL/hello') + f = self.query('SET/hello/world') + self.assertTrue(f.headers.getheader('Content-Type') == 'application/json') + self.assertTrue(f.headers.getheader('ETag') == '"0db1124cf79ffeb80aff6d199d5822f8"') + self.assertTrue(f.read() == '{"SET":[true,"OK"]}') + + def test_get(self): + "string type" + self.query('SET/hello/world') + f = self.query('GET/hello') + self.assertTrue(f.headers.getheader('Content-Type') == 'application/json') + self.assertTrue(f.headers.getheader('ETag') == '"8cf38afc245b7a6a88696566483d1390"') + self.assertTrue(f.read() == '{"GET":"world"}') + + def test_incr(self): + "integer type" + self.query('DEL/hello') + f = self.query('INCR/hello') + self.assertTrue(f.headers.getheader('Content-Type') == 'application/json') + self.assertTrue(f.headers.getheader('ETag') == '"500e9bcdcbb1e98f25c1fbb880a96c99"') + self.assertTrue(f.read() == '{"INCR":1}') + + def test_list(self): + "list type" + self.query('DEL/hello') + self.query('RPUSH/hello/abc') + self.query('RPUSH/hello/def') + f = self.query('LRANGE/hello/0/-1') + self.assertTrue(f.headers.getheader('Content-Type') == 'application/json') + self.assertTrue(f.headers.getheader('ETag') == '"622e51f547a480bef7cf5452fb7782db"') + self.assertTrue(f.read() == '{"LRANGE":["abc","def"]}') + + def test_error(self): + "error return type" + f = self.query('UNKNOWN/COMMAND') + self.assertTrue(f.headers.getheader('Content-Type') == 'application/json') + try: + obj = json.loads(f.read()) + except: + self.assertTrue(False) + return + + self.assertTrue(len(obj) == 1) + self.assertTrue('UNKNOWN' in obj) + self.assertTrue(isinstance(obj['UNKNOWN'], list)) + self.assertTrue(obj['UNKNOWN'][0] == False) + self.assertTrue(isinstance(obj['UNKNOWN'][1], unicode)) + +class TestCustom(TestWebdis): + def test_list(self): + "List responses with custom format" + self.query('DEL/hello') + self.query('RPUSH/hello/a/b/c') + f = self.query('LRANGE/hello/0/-1.txt') + self.assertTrue(f.headers.getheader('Content-Type') == 'text/plain') + self.assertTrue(f.read() == "abc") + + def test_separator(self): + "Separator in list responses with custom format" + self.query('DEL/hello') + self.query('RPUSH/hello/a/b/c') + f = self.query('LRANGE/hello/0/-1.txt?sep=--') + self.assertTrue(f.headers.getheader('Content-Type') == 'text/plain') + self.assertTrue(f.read() == "a--b--c") + +class TestRaw(TestWebdis): + + def test_set(self): + "success type (+OK)" + self.query('DEL/hello') + f = self.query('SET/hello/world.raw') + self.assertTrue(f.headers.getheader('Content-Type') == 'binary/octet-stream') + self.assertTrue(f.read() == "+OK\r\n") + + def test_get(self): + "string type" + self.query('SET/hello/world') + f = self.query('GET/hello.raw') + self.assertTrue(f.read() == '$5\r\nworld\r\n') + + def test_incr(self): + "integer type" + self.query('DEL/hello') + f = self.query('INCR/hello.raw') + self.assertTrue(f.read() == ':1\r\n') + + def test_list(self): + "list type" + self.query('DEL/hello') + self.query('RPUSH/hello/abc') + self.query('RPUSH/hello/def') + f = self.query('LRANGE/hello/0/-1.raw') + self.assertTrue(f.read() == "*2\r\n$3\r\nabc\r\n$3\r\ndef\r\n") + + def test_error(self): + "error return type" + f = self.query('UNKNOWN/COMMAND.raw') + self.assertTrue(f.read().startswith("-ERR ")) + +def need_msgpack(fn): + def wrapper(self): + if msgpack: + fn(self) + return wrapper + +class TestMsgPack(TestWebdis): + + @need_msgpack + def test_set(self): + "success type (+OK)" + self.query('DEL/hello') + f = self.query('SET/hello/world.msg') + self.assertTrue(f.headers.getheader('Content-Type') == 'application/x-msgpack') + obj = msgpack.loads(f.read()) + self.assertTrue(obj == {'SET': (True, 'OK')}) + + @need_msgpack + def test_get(self): + "string type" + self.query('SET/hello/world') + f = self.query('GET/hello.msg') + obj = msgpack.loads(f.read()) + self.assertTrue(obj == {'GET': 'world'}) + + @need_msgpack + def test_incr(self): + "integer type" + self.query('DEL/hello') + f = self.query('INCR/hello.msg') + obj = msgpack.loads(f.read()) + self.assertTrue(obj == {'INCR': 1}) + + @need_msgpack + def test_list(self): + "list type" + self.query('DEL/hello') + self.query('RPUSH/hello/abc') + self.query('RPUSH/hello/def') + f = self.query('LRANGE/hello/0/-1.msg') + obj = msgpack.loads(f.read()) + self.assertTrue(obj == {'LRANGE': ('abc', 'def')}) + + @need_msgpack + def test_error(self): + "error return type" + f = self.query('UNKNOWN/COMMAND.msg') + obj = msgpack.loads(f.read()) + self.assertTrue('UNKNOWN' in obj) + self.assertTrue(isinstance(obj, dict)) + self.assertTrue(isinstance(obj['UNKNOWN'], tuple)) + self.assertTrue(obj['UNKNOWN'][0] == False) + self.assertTrue(isinstance(obj['UNKNOWN'][1], str)) + +class TestETag(TestWebdis): + + def test_etag_match(self): + self.query('SET/hello/world') + h = hashlib.md5("world").hexdigest() # match Etag + try: + f = self.query('GET/hello.txt', None, {'If-None-Match': '"'+ h +'"'}) + except urllib2.HTTPError as e: + self.assertTrue(e.code == 304) + return + self.assertTrue(False) # we should have received a 304. + + def test_etag_fail(self): + self.query('SET/hello/world') + h = hashlib.md5("nonsense").hexdigest() # non-matching Etag + f = self.query('GET/hello.txt', None, {'If-None-Match': '"'+ h +'"'}) + self.assertTrue(f.read() == 'world') + +class TestDbSwitch(TestWebdis): + def test_db(self): + "Test database change" + self.query('0/SET/key/val0') + self.query('1/SET/key/val1') + f = self.query('0/GET/key.txt') + self.assertTrue(f.read() == "val0") + f = self.query('1/GET/key.txt') + self.assertTrue(f.read() == "val1") + f = self.query('GET/key.txt') + self.assertTrue(f.read() == "val0") + +if __name__ == '__main__': + unittest.main() diff --git a/tests/bench.sh b/tests/bench.sh new file mode 100755 index 0000000..0761696 --- /dev/null +++ b/tests/bench.sh @@ -0,0 +1,71 @@ +#!/bin/bash +CLIENTS=100 +REQUESTS=100000 + +HOST=$WEBDIS_HOST +PORT=$WEBDIS_PORT + +[ -n $HOST ] && HOST=127.0.0.1 +[ -n $PORT ] && PORT=7379 + +info() { + echo "Testing on $HOST:$PORT with $CLIENTS clients in parallel, for a total of $REQUESTS requests per benchmark." +} + +once() { + curl -q http://$HOST:$PORT/$1 1> /dev/null 2> /dev/null +} + +bench() { + NUM=`ab -k -c $CLIENTS -n $REQUESTS http://$HOST:$PORT/$1 2>/dev/null | grep "#/sec" | sed -e "s/[^0-9.]//g"` + echo -ne $NUM +} + +test_ping() { + echo -en "PING: " + bench "PING" + echo " requests/sec." +} + +test_set() { + echo -en "SET(hello,world): " + bench "SET/hello/world" + echo " requests/sec." +} + +test_get() { + echo -en "GET(hello): " + bench "GET/hello" + echo " requests/sec." +} + +test_incr() { + once "DEL/hello" + + echo -en "INCR(hello): " + bench "INCR/hello" + echo " requests/sec." +} + +test_lpush() { + once "DEL/hello" + + echo -en "LPUSH(hello,abc): " + bench "LPUSH/hello/abc" + echo " requests/sec." +} + +test_lrange() { + echo -en "LRANGE(hello,$1,$2): " + bench "LRANGE/hello/$1/$2" + echo " requests/sec." +} + +info +test_ping +test_set +test_get +test_incr +test_lpush +test_lrange 0 10 +test_lrange 0 100 diff --git a/tests/limits.py b/tests/limits.py new file mode 100755 index 0000000..1fdecc8 --- /dev/null +++ b/tests/limits.py @@ -0,0 +1,105 @@ +#!/usr/bin/python +import socket +import unittest + +import os +HOST = os.getenv('WEBDIS_HOST', '127.0.0.1') +PORT = int(os.getenv('WEBDIS_PORT', 7379)) + +class BlockingSocket: + + def __init__(self): + self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.s.setblocking(True) + self.s.connect((HOST, PORT)) + + def recv(self): + out = "" + while True: + try: + ret = self.s.recv(4096) + except: + return out + if len(ret) == 0: + return out + out += ret + + def recv_until(self, limit): + out = "" + while not limit in out: + try: + out += self.s.recv(4096) + except: + return False + return out + + def send(self, buf): + sz = len(buf) + done = 0 + while done < sz: + try: + ret = self.s.send(buf[done:4096]) + except: + return False + done = done + ret + # print "Sent %d/%d so far (%s just now)" % (done, sz, ret) + if ret < 0: + return False + return True + +class LargeString: + + def __init__(self, c, n): + self.char = c + self.len = n + + def __len__(self): + return self.len + + def __getitem__(self, chunk): + if chunk.start > self.len: + return "" + if chunk.start + chunk.stop > self.len: + return self.char * (self.len - chunk.start) + return self.char * chunk.stop + +class TestSocket(unittest.TestCase): + + def __init__(self, *args, **kwargs): + super(TestSocket, self).__init__(*args, **kwargs) + self.s = BlockingSocket() + + +class TestHugeUrl(TestSocket): + + def test_huge_url(self): + n = 1024*1024*1024 # 1GB query-string + + start = "GET /GET/x" + end = " HTTP/1.0\r\n\r\n" + + ok = self.s.send(start) + fail1 = self.s.send(LargeString("A", n)) + fail2 = self.s.send(end) + out = self.s.recv() + + self.assertTrue(ok) + self.assertTrue("400 Bad Request" in out) + + def test_huge_upload(self): + n = 1024*1024*1024 # upload 1GB + + start = "PUT /SET/x HTTP/1.0\r\n"\ + + ("Content-Length: %d\r\n" % (n))\ + + "Expect: 100-continue\r\n\r\n" + + ok = self.s.send(start) + cont = self.s.recv_until("\r\n") + fail = self.s.send(LargeString("A", n)) + + self.assertTrue(ok) + self.assertTrue("HTTP/1.1 100 Continue" in cont) + self.assertFalse(fail) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/pubsub.c b/tests/pubsub.c new file mode 100644 index 0000000..f4a3451 --- /dev/null +++ b/tests/pubsub.c @@ -0,0 +1,339 @@ +#include +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + + +/* + * Connection object. + */ +struct cx { + int fd; + + int *counter; /* shared counter of the number of messages received. */ + int total; /* total number of messages to send */ + + char *http_request; + + void (*read_fun)(int,short,void*); /* called when able to read fd */ + void (*write_fun)(int,short,void*); /* called when able to write fd */ + + /* libevent data structures */ + struct event evr, evw; + struct event_base *base; +}; + +int +webdis_connect(const char *host, short port) { + + int ret; + int fd; + struct sockaddr_in addr; + + /* connect socket */ + fd = socket(AF_INET, SOCK_STREAM, 0); + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + memset(&(addr.sin_addr), 0, sizeof(addr.sin_addr)); + addr.sin_addr.s_addr = inet_addr(host); + + ret = connect(fd, (struct sockaddr*)&addr, sizeof(struct sockaddr)); + if(ret != 0) { + fprintf(stderr, "connect: ret=%d: %s\n", ret, strerror(errno)); + return -1; + } + + return fd; +} + +/** + * Send request and read until the delimiter string is reached. blocking. + */ +void +reader_http_request(struct cx *c, const char* buffer, const char *limit) { + + char resp[2048]; + int pos = 0; + + int r = write(c->fd, buffer, strlen(buffer)); + (void)r; + + memset(resp, 0, sizeof(resp)); + while(1) { + int ret = read(c->fd, resp+pos, sizeof(resp)-pos); + if(ret <= 0) { + return; + } + pos += ret; + + if(strstr(resp, limit) != NULL) { + break; + } + } +} + +/** + * (re)install connection in the event loop. + */ +void +cx_install(struct cx *c) { + + if(c->read_fun) { /* attach callback for read. */ + event_set(&c->evr, c->fd, EV_READ, c->read_fun, c); + event_base_set(c->base, &c->evr); + event_add(&c->evr, NULL); + } + if(c->write_fun) { /* attach callback for write. */ + event_set(&c->evw, c->fd, EV_WRITE, c->write_fun, c); + event_base_set(c->base, &c->evw); + event_add(&c->evw, NULL); + } + +} + +/** + * Called when a reader has received data. + */ +void +reader_can_read(int fd, short event, void *ptr) { + + char buffer[1024]; + struct cx *c = ptr; + const char *p; + + (void)event; + + int ret = read(fd, buffer, sizeof(buffer)); + if(ret > 0) { + + /* count messages, each message starts with '{' */ + p = buffer; + do { + /* look for the start of a message */ + p = memchr(p, '{', buffer + ret - p); + + if(!p) break; /* none left. */ + p++; + + (*c->counter)++; /* increment the global message counter. */ + if(((*c->counter * 100) % c->total) == 0) { /* show progress. */ + printf("\r%d %%", 100 * *c->counter / c->total); + fflush(stdout); + } + if(*c->counter > c->total) { + /* halt event loop. */ + event_base_loopbreak(c->base); + } + } while(1); + } + + cx_install(c); +} + + +/** + * create a new reader object. + */ +void +reader_new(struct event_base *base, const char *host, short port, int total, int *counter, int chan) { + + struct cx *c = calloc(1, sizeof(struct cx)); + c->base = base; + c->counter = counter; + c->total = total; + c->fd = webdis_connect(host, port); + c->read_fun = reader_can_read; + + /* send subscription request. */ + c->http_request = malloc(100); + sprintf(c->http_request, "GET /SUBSCRIBE/chan:%d HTTP/1.1\r\n\r\n", chan); + reader_http_request(c, c->http_request, "{\"SUBSCRIBE\":[\"subscribe\""); + + /* add to the event loop. */ + cx_install(c); +} + +/** + * Called when a writer has received data back. read and ignore. + */ +void +writer_can_read(int fd, short event, void *ptr) { + char buffer[1024]; + struct cx *c = ptr; + int r; + + (void)event; + + r = read(fd, buffer, sizeof(buffer)); /* discard */ + (void)r; + + /* re-install in the event loop. */ + cx_install(c); +} + +/* send request */ +void +writer_can_write(int fd, short event, void *ptr) { + + struct cx *c = ptr; + + (void)fd; + (void)event; + + reader_http_request(c, c->http_request, "{\"PUBLISH\":"); + + cx_install(c); +} + +void +writer_new(struct event_base *base, const char *host, short port, int chan) { + + struct cx *c = malloc(sizeof(struct cx)); + c->base = base; + c->fd = webdis_connect(host, port); + c->read_fun = writer_can_read; + c->write_fun = writer_can_write; + + /* send request. */ + c->http_request = malloc(100); + sprintf(c->http_request, "GET /PUBLISH/chan:%d/hi HTTP/1.1\r\n\r\n", chan); + reader_http_request(c, c->http_request, "{\"PUBLISH\":"); + + cx_install(c); +} + + +void +usage(const char* argv0, char *host_default, short port_default, + int r_default, int w_default, int n_default, int c_default) { + + printf("Usage: %s [options]\n" + "Options are:\n" + "\t-h host\t\t(default = \"%s\")\n" + "\t-p port\t\t(default = %d)\n" + "\t-r readers\t(default = %d)\n" + "\t-w writers\t(default = %d)\n" + "\t-c channels\t(default = %d)\n" + "\t-n messages\t(number of messages to read in total, default = %d)\n", + argv0, host_default, (int)port_default, + r_default, w_default, + c_default, n_default); +} + +static struct event_base *__base; +void on_sigint(int s) { + (void)s; + event_base_loopbreak(__base); +} + + +int +main(int argc, char *argv[]) { + + /* Create R readers and W writers, send N messages in total. */ + + struct timespec t0, t1; + + struct event_base *base = event_base_new(); + int i, count = 0; + + /* getopt vars */ + int opt; + char *colon; + + /* default values */ + short port_default = 7379; + char *host_default = "127.0.0.1"; + int r_default = 450, w_default = 10, n_default = 100000, c_default = 1; + + + /* real values */ + int r = r_default, w = w_default, chans = c_default, n = n_default; + char *host = host_default; + short port = port_default; + + /* getopt */ + while ((opt = getopt(argc, argv, "h:p:r:w:c:n:")) != -1) { + switch (opt) { + case 'h': + colon = strchr(optarg, ':'); + if(!colon) { + size_t sz = strlen(optarg); + host = calloc(1 + sz, 1); + strncpy(host, optarg, sz); + } else { + host = calloc(1+colon-optarg, 1); + strncpy(host, optarg, colon-optarg); + port = (short)atol(colon+1); + } + break; + + case 'p': + port = (short)atol(optarg); + break; + + case 'r': + r = atoi(optarg); + break; + + case 'w': + w = atoi(optarg); + break; + + case 'c': + chans = atoi(optarg); + break; + + case 'n': + n = atoi(optarg); + break; + + default: + usage(argv[0], host_default, port_default, + r_default, w_default, + n_default, c_default); + exit(EXIT_FAILURE); + } + } + + for(i = 0; i < r; ++i) { + reader_new(base, host, port, n, &count, i % chans); + } + + for(i = 0; i < w; ++i) { + writer_new(base, host, port, i % chans); + } + + /* install signal handler */ + __base = base; + signal(SIGINT, on_sigint); + + /* save time now */ + clock_gettime(CLOCK_MONOTONIC, &t0); + + /* run test */ + event_base_dispatch(base); + + /* timing */ + clock_gettime(CLOCK_MONOTONIC, &t1); + float mili0 = t0.tv_sec * 1000 + t0.tv_nsec / 1000000; + float mili1 = t1.tv_sec * 1000 + t1.tv_nsec / 1000000; + + printf("\rReceived %d messages from %d writers to %d readers through %d channels in %0.2f sec: received %0.2f msg/sec\n", + count, w, r, chans, + (mili1-mili0)/1000.0, + 1000*count/(mili1-mili0)); + + return EXIT_SUCCESS; +} + diff --git a/tests/websocket.c b/tests/websocket.c new file mode 100644 index 0000000..1bce69d --- /dev/null +++ b/tests/websocket.c @@ -0,0 +1,326 @@ +/* http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-76 */ + +#include +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +struct host_info { + char *host; + short port; +}; + +/* worker_thread, with counter of remaining messages */ +struct worker_thread { + struct host_info *hi; + struct event_base *base; + + int msg_target; + int msg_received; + int msg_sent; + int byte_count; + pthread_t thread; + + struct evbuffer *buffer; + int got_header; + + int verbose; + struct event ev_w; +}; + +void +process_message(struct worker_thread *wt, size_t sz) { + + // printf("process_message\n"); + if(wt->msg_received % 10000 == 0) { + printf("thread %u: %8d messages left (got %9d bytes so far).\n", + (unsigned int)wt->thread, + wt->msg_target - wt->msg_received, wt->byte_count); + } + wt->byte_count += sz; + + /* decrement read count, and stop receiving when we reach zero. */ + wt->msg_received++; + if(wt->msg_received == wt->msg_target) { + event_base_loopexit(wt->base, NULL); + } +} + +void +websocket_write(int fd, short event, void *ptr) { + int ret; + struct worker_thread *wt = ptr; + + if(event != EV_WRITE) { + return; + } + + char message[] = "\x00[\"SET\",\"key\",\"value\"]\xff\x00[\"GET\",\"key\"]\xff"; + ret = write(fd, message, sizeof(message)-1); + if(ret != sizeof(message)-1) { + fprintf(stderr, "write on %d failed: %s\n", fd, strerror(errno)); + close(fd); + } + + wt->msg_sent += 2; + if(wt->msg_sent < wt->msg_target) { + event_set(&wt->ev_w, fd, EV_WRITE, websocket_write, wt); + event_base_set(wt->base, &wt->ev_w); + ret = event_add(&wt->ev_w, NULL); + } +} + +static void +websocket_read(int fd, short event, void *ptr) { + char packet[2048], *pos; + int ret, success = 1; + + struct worker_thread *wt = ptr; + + if(event != EV_READ) { + return; + } + + /* read message */ + ret = read(fd, packet, sizeof(packet)); + pos = packet; + if(ret > 0) { + char *data, *last; + int sz, msg_sz; + + if(wt->got_header == 0) { /* first response */ + char *frame_start = strstr(packet, "MH"); /* end of the handshake */ + if(frame_start == NULL) { + return; /* not yet */ + } else { /* start monitoring possible writes */ + printf("start monitoring possible writes\n"); + evbuffer_add(wt->buffer, frame_start + 2, ret - (frame_start + 2 - packet)); + + wt->got_header = 1; + event_set(&wt->ev_w, fd, EV_WRITE, + websocket_write, wt); + event_base_set(wt->base, &wt->ev_w); + ret = event_add(&wt->ev_w, NULL); + } + } else { + /* we've had the header already, now bufffer data. */ + evbuffer_add(wt->buffer, packet, ret); + } + + while(1) { + data = (char*)EVBUFFER_DATA(wt->buffer); + sz = EVBUFFER_LENGTH(wt->buffer); + + if(sz == 0) { /* no data */ + break; + } + if(*data != 0) { /* missing frame start */ + success = 0; + break; + } + last = memchr(data, 0xff, sz); /* look for frame end */ + if(!last) { + /* no end of frame in sight. */ + break; + } + msg_sz = last - data - 1; + process_message(ptr, msg_sz); /* record packet */ + + /* drain including frame delimiters (+2 bytes) */ + evbuffer_drain(wt->buffer, msg_sz + 2); + } + } else { + printf("ret=%d\n", ret); + success = 0; + } + if(success == 0) { + shutdown(fd, SHUT_RDWR); + close(fd); + event_base_loopexit(wt->base, NULL); + } +} + +void* +worker_main(void *ptr) { + + char ws_template[] = "GET /.json HTTP/1.1\r\n" + "Host: %s:%d\r\n" + "Connection: Upgrade\r\n" + "Upgrade: WebSocket\r\n" + "Origin: http://%s:%d\r\n" + "Sec-WebSocket-Key1: 18x 6]8vM;54 *(5: { U1]8 z [ 8\r\n" + "Sec-WebSocket-Key2: 1_ tx7X d < nw 334J702) 7]o}` 0\r\n" + "\r\n" + "Tm[K T2u"; + + struct worker_thread *wt = ptr; + + int ret; + int fd; + struct sockaddr_in addr; + char *ws_handshake; + size_t ws_handshake_sz; + + /* connect socket */ + fd = socket(AF_INET, SOCK_STREAM, 0); + addr.sin_family = AF_INET; + addr.sin_port = htons(wt->hi->port); + memset(&(addr.sin_addr), 0, sizeof(addr.sin_addr)); + addr.sin_addr.s_addr = inet_addr(wt->hi->host); + + ret = connect(fd, (struct sockaddr*)&addr, sizeof(struct sockaddr)); + if(ret != 0) { + fprintf(stderr, "connect: ret=%d: %s\n", ret, strerror(errno)); + return NULL; + } + + /* initialize worker thread */ + wt->base = event_base_new(); + wt->buffer = evbuffer_new(); + wt->byte_count = 0; + wt->got_header = 0; + + /* send handshake */ + ws_handshake_sz = sizeof(ws_handshake) + + 2*strlen(wt->hi->host) + 500; + ws_handshake = calloc(ws_handshake_sz, 1); + ws_handshake_sz = (size_t)sprintf(ws_handshake, ws_template, + wt->hi->host, wt->hi->port, + wt->hi->host, wt->hi->port); + ret = write(fd, ws_handshake, ws_handshake_sz); + + struct event ev_r; + event_set(&ev_r, fd, EV_READ | EV_PERSIST, websocket_read, wt); + event_base_set(wt->base, &ev_r); + event_add(&ev_r, NULL); + + /* go! */ + event_base_dispatch(wt->base); + event_base_free(wt->base); + free(ws_handshake); + return NULL; +} + +void +usage(const char* argv0, char *host_default, short port_default, + int thread_count_default, int messages_default) { + + printf("Usage: %s [options]\n" + "Options are:\n" + "\t-h host\t\t(default = \"%s\")\n" + "\t-p port\t\t(default = %d)\n" + "\t-c threads\t(default = %d)\n" + "\t-n count\t(number of messages per thread, default = %d)\n" + "\t-v\t\t(verbose)\n", + argv0, host_default, (int)port_default, + thread_count_default, messages_default); +} + +int +main(int argc, char *argv[]) { + + struct timespec t0, t1; + + int messages_default = 100000; + int thread_count_default = 4; + short port_default = 7379; + char *host_default = "127.0.0.1"; + + int msg_target = messages_default; + int thread_count = thread_count_default; + int i, opt; + char *colon; + double total = 0, total_bytes = 0; + int verbose = 0; + + struct host_info hi = {host_default, port_default}; + + struct worker_thread *workers; + + /* getopt */ + while ((opt = getopt(argc, argv, "h:p:c:n:v")) != -1) { + switch (opt) { + case 'h': + colon = strchr(optarg, ':'); + if(!colon) { + size_t sz = strlen(optarg); + hi.host = calloc(1 + sz, 1); + strncpy(hi.host, optarg, sz); + } else { + hi.host = calloc(1+colon-optarg, 1); + strncpy(hi.host, optarg, colon-optarg); + hi.port = (short)atol(colon+1); + } + break; + + case 'p': + hi.port = (short)atol(optarg); + break; + + case 'c': + thread_count = atoi(optarg); + break; + + case 'n': + msg_target = atoi(optarg); + break; + + case 'v': + verbose = 1; + break; + default: /* '?' */ + usage(argv[0], host_default, port_default, + thread_count_default, + messages_default); + exit(EXIT_FAILURE); + } + } + + /* run threads */ + workers = calloc(sizeof(struct worker_thread), thread_count); + + clock_gettime(CLOCK_MONOTONIC, &t0); + for(i = 0; i < thread_count; ++i) { + workers[i].msg_target = msg_target; + workers[i].hi = &hi; + workers[i].verbose = verbose; + + pthread_create(&workers[i].thread, NULL, + worker_main, &workers[i]); + } + + /* wait for threads to finish */ + for(i = 0; i < thread_count; ++i) { + pthread_join(workers[i].thread, NULL); + total += workers[i].msg_received; + total_bytes += workers[i].byte_count; + } + + /* timing */ + clock_gettime(CLOCK_MONOTONIC, &t1); + float mili0 = t0.tv_sec * 1000 + t0.tv_nsec / 1000000; + float mili1 = t1.tv_sec * 1000 + t1.tv_nsec / 1000000; + + if(total != 0) { + printf("Read %ld messages in %0.2f sec: %0.2f msg/sec (%d MB/sec, %d KB/sec)\n", + (long)total, + (mili1-mili0)/1000.0, + 1000*total/(mili1-mili0), + (int)(total_bytes / (1000*(mili1-mili0))), + (int)(total_bytes / (mili1-mili0))); + return EXIT_SUCCESS; + } else { + printf("No message was read.\n"); + return EXIT_FAILURE; + } +} + diff --git a/tests/websocket.html b/tests/websocket.html new file mode 100644 index 0000000..a30f936 --- /dev/null +++ b/tests/websocket.html @@ -0,0 +1,153 @@ + + + + WebSocket example + + + + + +
Webdis with HTML5 WebSockets
+ +
+

JSON

+
+ Connecting... +
+
+
+

Raw

+
+ Connecting... +
+
+ + + + + diff --git a/version.h b/version.h new file mode 100644 index 0000000..4d07457 --- /dev/null +++ b/version.h @@ -0,0 +1,8 @@ +#ifndef VERSION_H +#define VERSION_H + +#ifndef WEBDIS_VERSION +#define WEBDIS_VERSION "0.1.2" +#endif + +#endif /* VERSION_H */ diff --git a/webdis.c b/webdis.c new file mode 100644 index 0000000..34e88e3 --- /dev/null +++ b/webdis.c @@ -0,0 +1,20 @@ +#include "server.h" + +#include + +int +main(int argc, char *argv[]) { + + struct server *s; + + if(argc > 1) { + s = server_new(argv[1]); + } else { + s = server_new("webdis.json"); + } + + server_start(s); + + return EXIT_SUCCESS; +} + diff --git a/webdis.json b/webdis.json new file mode 100644 index 0000000..f4b0714 --- /dev/null +++ b/webdis.json @@ -0,0 +1,31 @@ +{ + "redis_host": "127.0.0.1", + + "redis_port": 6379, + "redis_auth": null, + + "http_host": "0.0.0.0", + "http_port": 7379, + + "threads": 5, + "pool_size": 20, + + "daemonize": false, + "websockets": false, + + "database": 0, + + "acl": [ + { + "disabled": ["DEBUG"] + }, + + { + "http_basic_auth": "user:password", + "enabled": ["DEBUG"] + } + ], + + "verbosity": 6, + "logfile": "webdis.log" +} diff --git a/webdis.prod.json b/webdis.prod.json new file mode 100644 index 0000000..de35ca8 --- /dev/null +++ b/webdis.prod.json @@ -0,0 +1,28 @@ +{ + "redis_host": "127.0.0.1", + + "redis_port": 6379, + "redis_auth": null, + + "http_host": "0.0.0.0", + "http_port": 7379, + "threads": 4, + + "daemonize": true, + + "database": 0, + + "acl": [ + { + "disabled": ["DEBUG"] + }, + + { + "http_basic_auth": "user:password", + "enabled": ["DEBUG"] + } + ], + + "verbosity": 3, + "logfile": "/var/log/webdis.log" +} diff --git a/websocket.c b/websocket.c new file mode 100644 index 0000000..9b0223c --- /dev/null +++ b/websocket.c @@ -0,0 +1,380 @@ +#include "sha1/sha1.h" +#include +#include "websocket.h" +#include "client.h" +#include "cmd.h" +#include "worker.h" +#include "pool.h" +#include "http.h" + +/* message parsers */ +#include "formats/json.h" +#include "formats/raw.h" + +#include +#include +#include +#include +#include +#include + +/** + * This code uses the WebSocket specification from RFC 6455. + * A copy is available at http://www.rfc-editor.org/rfc/rfc6455.txt + */ + +/* custom 64-bit encoding functions to avoid portability issues */ +#define webdis_ntohl64(p) \ + ((((uint64_t)((p)[0])) << 0) + (((uint64_t)((p)[1])) << 8) +\ + (((uint64_t)((p)[2])) << 16) + (((uint64_t)((p)[3])) << 24) +\ + (((uint64_t)((p)[4])) << 32) + (((uint64_t)((p)[5])) << 40) +\ + (((uint64_t)((p)[6])) << 48) + (((uint64_t)((p)[7])) << 56)) + +#define webdis_htonl64(p) {\ + (char)(((p & ((uint64_t)0xff << 0)) >> 0) & 0xff), (char)(((p & ((uint64_t)0xff << 8)) >> 8) & 0xff), \ + (char)(((p & ((uint64_t)0xff << 16)) >> 16) & 0xff), (char)(((p & ((uint64_t)0xff << 24)) >> 24) & 0xff), \ + (char)(((p & ((uint64_t)0xff << 32)) >> 32) & 0xff), (char)(((p & ((uint64_t)0xff << 40)) >> 40) & 0xff), \ + (char)(((p & ((uint64_t)0xff << 48)) >> 48) & 0xff), (char)(((p & ((uint64_t)0xff << 56)) >> 56) & 0xff) } +static int +ws_compute_handshake(struct http_client *c, char *out, size_t *out_sz) { + + unsigned char *buffer, sha1_output[20]; + char magic[] = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; + SHA1Context ctx; + base64_encodestate b64_ctx; + int pos, i; + + // websocket handshake + const char *key = client_get_header(c, "Sec-WebSocket-Key"); + size_t key_sz = key?strlen(key):0, buffer_sz = key_sz + sizeof(magic) - 1; + buffer = calloc(buffer_sz, 1); + + // concatenate key and guid in buffer + memcpy(buffer, key, key_sz); + memcpy(buffer+key_sz, magic, sizeof(magic)-1); + + // compute sha-1 + SHA1Reset(&ctx); + SHA1Input(&ctx, buffer, buffer_sz); + SHA1Result(&ctx); + for(i = 0; i < 5; ++i) { // put in correct byte order before memcpy. + ctx.Message_Digest[i] = ntohl(ctx.Message_Digest[i]); + } + memcpy(sha1_output, (unsigned char*)ctx.Message_Digest, 20); + + // encode `sha1_output' in base 64, into `out'. + base64_init_encodestate(&b64_ctx); + pos = base64_encode_block((const char*)sha1_output, 20, out, &b64_ctx); + base64_encode_blockend(out + pos, &b64_ctx); + + // compute length, without \n + *out_sz = strlen(out); + if(out[*out_sz-1] == '\n') + (*out_sz)--; + + free(buffer); + + return 0; +} + +int +ws_handshake_reply(struct http_client *c) { + + int ret; + char sha1_handshake[40]; + char *buffer = NULL, *p; + const char *origin = NULL, *host = NULL; + size_t origin_sz = 0, host_sz = 0, handshake_sz = 0, sz; + + char template0[] = "HTTP/1.1 101 Switching Protocols\r\n" + "Upgrade: websocket\r\n" + "Connection: Upgrade\r\n" + "Sec-WebSocket-Origin: "; /* %s */ + char template1[] = "\r\n" + "Sec-WebSocket-Location: ws://"; /* %s%s */ + char template2[] = "\r\n" + "Origin: http://"; /* %s */ + char template3[] = "\r\n" + "Sec-WebSocket-Accept: "; /* %s */ + char template4[] = "\r\n\r\n"; + + if((origin = client_get_header(c, "Origin"))) { + origin_sz = strlen(origin); + } else if((origin = client_get_header(c, "Sec-WebSocket-Origin"))) { + origin_sz = strlen(origin); + } + if((host = client_get_header(c, "Host"))) { + host_sz = strlen(host); + } + + /* need those headers */ + if(!origin || !origin_sz || !host || !host_sz || !c->path || !c->path_sz) { + return -1; + } + + memset(sha1_handshake, 0, sizeof(sha1_handshake)); + if(ws_compute_handshake(c, &sha1_handshake[0], &handshake_sz) != 0) { + /* failed to compute handshake. */ + return -1; + } + + sz = sizeof(template0)-1 + origin_sz + + sizeof(template1)-1 + host_sz + c->path_sz + + sizeof(template2)-1 + host_sz + + sizeof(template3)-1 + handshake_sz + + sizeof(template4)-1; + + p = buffer = malloc(sz); + + /* Concat all */ + + /* template0 */ + memcpy(p, template0, sizeof(template0)-1); + p += sizeof(template0)-1; + memcpy(p, origin, origin_sz); + p += origin_sz; + + /* template1 */ + memcpy(p, template1, sizeof(template1)-1); + p += sizeof(template1)-1; + memcpy(p, host, host_sz); + p += host_sz; + memcpy(p, c->path, c->path_sz); + p += c->path_sz; + + /* template2 */ + memcpy(p, template2, sizeof(template2)-1); + p += sizeof(template2)-1; + memcpy(p, host, host_sz); + p += host_sz; + + /* template3 */ + memcpy(p, template3, sizeof(template3)-1); + p += sizeof(template3)-1; + memcpy(p, &sha1_handshake[0], handshake_sz); + p += handshake_sz; + + /* template4 */ + memcpy(p, template4, sizeof(template4)-1); + p += sizeof(template4)-1; + + /* send data to client */ + ret = write(c->fd, buffer, sz); + (void)ret; + free(buffer); + + return 0; +} + + +static int +ws_execute(struct http_client *c, const char *frame, size_t frame_len) { + + struct cmd*(*fun_extract)(struct http_client *, const char *, size_t) = NULL; + formatting_fun fun_reply = NULL; + + if((c->path_sz == 1 && strncmp(c->path, "/", 1) == 0) || + strncmp(c->path, "/.json", 6) == 0) { + fun_extract = json_ws_extract; + fun_reply = json_reply; + } else if(strncmp(c->path, "/.raw", 5) == 0) { + fun_extract = raw_ws_extract; + fun_reply = raw_reply; + } + + if(fun_extract) { + + /* Parse websocket frame into a cmd object. */ + struct cmd *cmd = fun_extract(c, frame, frame_len); + + if(cmd) { + /* copy client info into cmd. */ + cmd_setup(cmd, c); + cmd->is_websocket = 1; + + if (c->pub_sub != NULL) { + /* This client already has its own connection + * to Redis due to a subscription; use it from + * now on. */ + cmd->ac = c->pub_sub->ac; + } else if (cmd_is_subscribe(cmd)) { + /* New subscribe command; make new Redis context + * for this client */ + cmd->ac = pool_connect(c->w->pool, cmd->database, 0); + c->pub_sub = cmd; + cmd->pub_sub_client = c; + } else { + /* get Redis connection from pool */ + cmd->ac = (redisAsyncContext*)pool_get_context(c->w->pool); + } + + /* send it off */ + cmd_send(cmd, fun_reply); + + return 0; + } + } + + return -1; +} + +static struct ws_msg * +ws_msg_new() { + return calloc(1, sizeof(struct ws_msg)); +} + +static void +ws_msg_add(struct ws_msg *m, const char *p, size_t psz, const unsigned char *mask) { + + /* add data to frame */ + size_t i; + m->payload = realloc(m->payload, m->payload_sz + psz); + memcpy(m->payload + m->payload_sz, p, psz); + + /* apply mask */ + for(i = 0; i < psz && mask; ++i) { + m->payload[m->payload_sz + i] = (unsigned char)p[i] ^ mask[i%4]; + } + + /* save new size */ + m->payload_sz += psz; +} + +static void +ws_msg_free(struct ws_msg **m) { + + free((*m)->payload); + free(*m); + *m = NULL; +} + +static enum ws_state +ws_parse_data(const char *frame, size_t sz, struct ws_msg **msg) { + + int has_mask; + uint64_t len; + const char *p; + unsigned char mask[4]; + + /* parse frame and extract contents */ + if(sz < 8) { + return WS_READING; + } + + has_mask = frame[1] & 0x80 ? 1:0; + + /* get payload length */ + len = frame[1] & 0x7f; /* remove leftmost bit */ + if(len <= 125) { /* data starts right after the mask */ + p = frame + 2 + (has_mask ? 4 : 0); + if(has_mask) memcpy(&mask, frame + 2, sizeof(mask)); + } else if(len == 126) { + uint16_t sz16; + memcpy(&sz16, frame + 2, sizeof(uint16_t)); + len = ntohs(sz16); + p = frame + 4 + (has_mask ? 4 : 0); + if(has_mask) memcpy(&mask, frame + 4, sizeof(mask)); + } else if(len == 127) { + len = webdis_ntohl64(frame+2); + p = frame + 10 + (has_mask ? 4 : 0); + if(has_mask) memcpy(&mask, frame + 10, sizeof(mask)); + } else { + return WS_ERROR; + } + + /* we now have the (possibly masked) data starting in p, and its length. */ + if(len > sz - (p - frame)) { /* not enough data */ + return WS_READING; + } + + if(!*msg) + *msg = ws_msg_new(); + ws_msg_add(*msg, p, len, has_mask ? mask : NULL); + (*msg)->total_sz += len + (p - frame); + + if(frame[0] & 0x80) { /* FIN bit set */ + return WS_MSG_COMPLETE; + } else { + return WS_READING; /* need more data */ + } +} + + +/** + * Process some data just received on the socket. + */ +enum ws_state +ws_add_data(struct http_client *c) { + + enum ws_state state; + + state = ws_parse_data(c->buffer, c->sz, &c->frame); + + if(state == WS_MSG_COMPLETE) { + int ret = ws_execute(c, c->frame->payload, c->frame->payload_sz); + + /* remove frame from client buffer */ + http_client_remove_data(c, c->frame->total_sz); + + /* free frame and set back to NULL */ + ws_msg_free(&c->frame); + + if(ret != 0) { + /* can't process frame. */ + return WS_ERROR; + } + } + return state; +} + +int +ws_reply(struct cmd *cmd, const char *p, size_t sz) { + + char *frame = malloc(sz + 8); /* create frame by prepending header */ + size_t frame_sz = 0; + struct http_response *r; + if (frame == NULL) + return -1; + + /* + The length of the "Payload data", in bytes: if 0-125, that is the + payload length. If 126, the following 2 bytes interpreted as a + 16-bit unsigned integer are the payload length. If 127, the + following 8 bytes interpreted as a 64-bit unsigned integer (the + most significant bit MUST be 0) are the payload length. + */ + frame[0] = '\x81'; + if(sz <= 125) { + frame[1] = sz; + memcpy(frame + 2, p, sz); + frame_sz = sz + 2; + } else if (sz > 125 && sz <= 65536) { + uint16_t sz16 = htons(sz); + frame[1] = 126; + memcpy(frame + 2, &sz16, 2); + memcpy(frame + 4, p, sz); + frame_sz = sz + 4; + } else if (sz > 65536) { + char sz64[8] = webdis_htonl64(sz); + frame[1] = 127; + memcpy(frame + 2, sz64, 8); + memcpy(frame + 10, p, sz); + frame_sz = sz + 10; + } + + /* send WS frame */ + r = http_response_init(cmd->w, 0, NULL); + if (cmd_is_subscribe(cmd)) { + r->keep_alive = 1; + } + + if (r == NULL) + return -1; + + r->out = frame; + r->out_sz = frame_sz; + r->sent = 0; + http_schedule_write(cmd->fd, r); + + return 0; +} diff --git a/websocket.h b/websocket.h new file mode 100644 index 0000000..dc9d764 --- /dev/null +++ b/websocket.h @@ -0,0 +1,30 @@ +#ifndef WEBSOCKET_H +#define WEBSOCKET_H + +#include +#include + +struct http_client; +struct cmd; + +enum ws_state { + WS_ERROR, + WS_READING, + WS_MSG_COMPLETE}; + +struct ws_msg { + char *payload; + size_t payload_sz; + size_t total_sz; +}; + +int +ws_handshake_reply(struct http_client *c); + +enum ws_state +ws_add_data(struct http_client *c); + +int +ws_reply(struct cmd *cmd, const char *p, size_t sz); + +#endif diff --git a/worker.c b/worker.c new file mode 100644 index 0000000..2bf29d3 --- /dev/null +++ b/worker.c @@ -0,0 +1,237 @@ +#include "worker.h" +#include "client.h" +#include "http.h" +#include "cmd.h" +#include "pool.h" +#include "slog.h" +#include "websocket.h" +#include "conf.h" +#include "server.h" + +#include +#include +#include +#include +#include + + +struct worker * +worker_new(struct server *s) { + + int ret; + struct worker *w = calloc(1, sizeof(struct worker)); + w->s = s; + + /* setup communication link */ + ret = pipe(w->link); + (void)ret; + + /* Redis connection pool */ + w->pool = pool_new(w, s->cfg->pool_size_per_thread); + + return w; + +} + +void +worker_can_read(int fd, short event, void *p) { + + struct http_client *c = p; + int ret, nparsed; + + (void)fd; + (void)event; + + ret = http_client_read(c); + if(ret <= 0) { + if((client_error_t)ret == CLIENT_DISCONNECTED) { + return; + } else if (c->failed_alloc || (client_error_t)ret == CLIENT_OOM) { + slog(c->w->s, WEBDIS_DEBUG, "503", 3); + http_send_error(c, 503, "Service Unavailable"); + return; + } + } + + if(c->is_websocket) { + /* Got websocket data */ + ws_add_data(c); + } else { + /* run parser */ + nparsed = http_client_execute(c); + + if(c->failed_alloc) { + slog(c->w->s, WEBDIS_DEBUG, "503", 3); + http_send_error(c, 503, "Service Unavailable"); + } else if (c->parser.flags & F_CONNECTION_CLOSE) { + c->broken = 1; + } else if(c->is_websocket) { + /* we need to use the remaining (unparsed) data as the body. */ + if(nparsed < ret) { + http_client_add_to_body(c, c->buffer + nparsed + 1, c->sz - nparsed - 1); + ws_handshake_reply(c); + } else { + c->broken = 1; + } + free(c->buffer); + c->buffer = NULL; + c->sz = 0; + } else if(nparsed != ret) { + slog(c->w->s, WEBDIS_DEBUG, "400", 3); + http_send_error(c, 400, "Bad Request"); + } else if(c->request_sz > c->s->cfg->http_max_request_size) { + slog(c->w->s, WEBDIS_DEBUG, "413", 3); + http_send_error(c, 413, "Request Entity Too Large"); + } + } + + if(c->broken) { /* terminate client */ + http_client_free(c); + } else { + /* start monitoring input again */ + worker_monitor_input(c); + } +} + +/** + * Monitor client FD for possible reads. + */ +void +worker_monitor_input(struct http_client *c) { + + event_set(&c->ev, c->fd, EV_READ, worker_can_read, c); + event_base_set(c->w->base, &c->ev); + event_add(&c->ev, NULL); +} + +/** + * Called when a client is sent to this worker. + */ +static void +worker_on_new_client(int pipefd, short event, void *ptr) { + + struct http_client *c; + unsigned long addr; + + (void)event; + (void)ptr; + + /* Get client from messaging pipe */ + int ret = read(pipefd, &addr, sizeof(addr)); + if(ret == sizeof(addr)) { + c = (struct http_client*)addr; + + /* monitor client for input */ + worker_monitor_input(c); + } +} + +static void +worker_pool_connect(struct worker *w) { + + int i; + /* create connections */ + for(i = 0; i < w->pool->count; ++i) { + pool_connect(w->pool, w->s->cfg->database, 1); + } + +} + +static void* +worker_main(void *p) { + + struct worker *w = p; + struct event ev; + + /* setup libevent */ + w->base = event_base_new(); + + /* monitor pipe link */ + event_set(&ev, w->link[0], EV_READ | EV_PERSIST, worker_on_new_client, w); + event_base_set(w->base, &ev); + event_add(&ev, NULL); + + /* connect to Redis */ + worker_pool_connect(w); + + /* loop */ + event_base_dispatch(w->base); + + return NULL; +} + +void +worker_start(struct worker *w) { + + pthread_create(&w->thread, NULL, worker_main, w); +} + +/** + * Queue new client to process + */ +void +worker_add_client(struct worker *w, struct http_client *c) { + + /* write into pipe link */ + unsigned long addr = (unsigned long)c; + int ret = write(w->link[1], &addr, sizeof(addr)); + (void)ret; +} + +/** + * Called when a client has finished reading input and can create a cmd + */ +void +worker_process_client(struct http_client *c) { + + /* check that the command can be executed */ + struct worker *w = c->w; + cmd_response_t ret = CMD_PARAM_ERROR; + switch(c->parser.method) { + case HTTP_GET: + if(c->path_sz == 16 && memcmp(c->path, "/crossdomain.xml", 16) == 0) { + http_crossdomain(c); + return; + } + slog(w->s, WEBDIS_DEBUG, c->path, c->path_sz); + ret = cmd_run(c->w, c, 1+c->path, c->path_sz-1, NULL, 0); + break; + + case HTTP_POST: + slog(w->s, WEBDIS_DEBUG, c->path, c->path_sz); + ret = cmd_run(c->w, c, c->body, c->body_sz, NULL, 0); + break; + + case HTTP_PUT: + slog(w->s, WEBDIS_DEBUG, c->path, c->path_sz); + ret = cmd_run(c->w, c, 1+c->path, c->path_sz-1, + c->body, c->body_sz); + break; + + case HTTP_OPTIONS: + http_send_options(c); + return; + + default: + slog(w->s, WEBDIS_DEBUG, "405", 3); + http_send_error(c, 405, "Method Not Allowed"); + return; + } + + switch(ret) { + case CMD_ACL_FAIL: + case CMD_PARAM_ERROR: + slog(w->s, WEBDIS_DEBUG, "403", 3); + http_send_error(c, 403, "Forbidden"); + break; + + case CMD_REDIS_UNAVAIL: + slog(w->s, WEBDIS_DEBUG, "503", 3); + http_send_error(c, 503, "Service Unavailable"); + break; + default: + break; + } + +} + diff --git a/worker.h b/worker.h new file mode 100644 index 0000000..b4e9cde --- /dev/null +++ b/worker.h @@ -0,0 +1,41 @@ +#ifndef WORKER_H +#define WORKER_H + +#include + +struct http_client; +struct pool; + +struct worker { + + /* self */ + pthread_t thread; + struct event_base *base; + + /* connection dispatcher */ + struct server *s; + int link[2]; + + /* Redis connection pool */ + struct pool *pool; +}; + +struct worker * +worker_new(struct server *s); + +void +worker_start(struct worker *w); + +void +worker_add_client(struct worker *w, struct http_client *c); + +void +worker_monitor_input(struct http_client *c); + +void +worker_can_read(int fd, short event, void *p); + +void +worker_process_client(struct http_client *c); + +#endif