From f6861f8d8e76c14484577e8e67e5f2ee3bea9f44 Mon Sep 17 00:00:00 2001 From: Jeremy Williams Date: Thu, 30 Jan 2020 17:41:44 -0500 Subject: [PATCH 01/20] Major refactor of the parser and request processing --- httpserver.h | 1004 ++++++++++++++++++++++++-------------------------- 1 file changed, 491 insertions(+), 513 deletions(-) diff --git a/httpserver.h b/httpserver.h index e2aa6b5..beb47b0 100644 --- a/httpserver.h +++ b/httpserver.h @@ -253,6 +253,8 @@ void http_request_read_chunk( // the next call to `http_request_read_chunk`. struct http_string_s http_request_chunk(struct http_request_s* request); +#define http_request_read_body http_request_read_chunk + #ifdef __cplusplus } #endif @@ -320,6 +322,7 @@ int main() { #define HTTP_MAX_CONTENT_LENGTH 8388608 // 8mb #define HTTP_MAX_TOKEN_LENGTH 8192 // 8kb #define HTTP_MAX_TOTAL_EST_MEM_USAGE 4294967296 // 4gb +#define HTTP_MAX_REQUEST_BUF_SIZE 8388608 // 8mb #define HTTP_MAX_HEADER_COUNT 127 @@ -327,61 +330,24 @@ int main() { #define HTTP_FLAG_CLEAR(var, flag) var &= ~flag #define HTTP_FLAG_CHECK(var, flag) (var & flag) -// token types / parser states -#define HTTP_METHOD 0 -#define HTTP_TARGET 1 -#define HTTP_VERSION 2 -#define HTTP_HEADER_KEY 3 -#define HTTP_HEADER_VALUE 4 -#define HTTP_HEADER_END 5 -#define HTTP_NONE 6 -#define HTTP_BODY 7 -#define HTTP_PARSE_ERROR 13 - -// error sub types -#define HTTP_ERR_PAYLOAD_TOO_LARGE 0 -#define HTTP_ERR_BAD_REQUEST 1 - -// chunked token types / parser state -#define HTTP_CHUNK_SIZE 8 -#define HTTP_CHUNK_EXTN 9 -#define HTTP_CHUNK_BODY 10 -#define HTTP_CHUNK_BODY_END 11 -#define HTTP_CHUNK_BODY_PARTIAL 12 +// stream flags +#define HS_SF_CONSUMED 0x1 // parser flags -#define HS_PF_TRANSFER_ENCODING 0x1 -#define HS_PF_CONTENT_LENGTH 0x2 +#define HS_PF_IN_CONTENT_LEN 0x1 +#define HS_PF_IN_TRANSFER_ENC 0x2 #define HS_PF_CHUNKED 0x4 - -// parser sub states -#define HTTP_LWS 2 -#define HTTP_CR 3 -#define HTTP_CRLF 4 - -// parser comparisons -#define HS_CONTENT_LENGTH_LOW "content-length" -#define HS_CONTENT_LENGTH_UP "CONTENT-LENGTH" -#define HS_TRANSFER_ENCODING_LOW "transfer-encoding" -#define HS_TRANSFER_ENCODING_UP "TRANSFER-ENCODING" -#define HS_CHUNKED_LOW "chunked" -#define HS_CHUNKED_UP "CHUNKED" - -// parser sentinel lengths -#define HTTP_CHUNKED_LEN -1 +#define HS_PF_CKEND 0x8 +#define HS_PF_REQ_END 0x10 // http session states #define HTTP_SESSION_INIT 0 -#define HTTP_SESSION_READ_HEADERS 1 -#define HTTP_SESSION_READ_BODY 2 -#define HTTP_SESSION_WRITE 3 -#define HTTP_SESSION_READ_CHUNK 4 -#define HTTP_SESSION_NOP 5 +#define HTTP_SESSION_READ 1 +#define HTTP_SESSION_WRITE 2 +#define HTTP_SESSION_NOP 3 // http session flags -#define HTTP_RESPONSE_READY 0x4 #define HTTP_AUTOMATIC 0x8 -#define HTTP_RESPONSE_PAUSED 0x10 #define HTTP_CHUNKED_RESPONSE 0x20 // http version indicators @@ -398,20 +364,6 @@ typedef struct { int type; } http_token_t; -typedef struct { - int content_length; - int len; - int token_start_index; - int start; - int body_start_index; - char header_count; - char content_length_i; - char transfer_encoding_i; - char flags; - char state; - char sub_state; -} http_parser_t; - typedef struct { http_token_t* buf; int capacity; @@ -430,6 +382,25 @@ typedef struct http_ev_cb_s { #endif } ev_cb_t; +typedef struct { + char* buf; + int64_t total_bytes; + int capacity; + int length; + int index; + int anchor; + http_token_t token; + uint8_t flags; +} hs_stream_t; + +typedef struct { + int64_t content_length; + int match_index; + int inject_token; + int8_t state; + uint8_t flags; +} http_parser_t; + typedef struct http_request_s { #ifdef KQUEUE void (*handler)(struct kevent* ev); @@ -440,16 +411,13 @@ typedef struct http_request_s { #endif void (*chunk_cb)(struct http_request_s*); void* data; + hs_stream_t stream; http_parser_t parser; + int i; int state; int socket; - char* buf; - int bytes; - int written; - int capacity; int timeout; struct http_server_s* server; - http_token_t token; http_token_dyn_t tokens; char flags; } http_request_t; @@ -487,6 +455,24 @@ typedef struct http_response_s { typedef struct http_string_s http_string_t; +// enums + +enum hs_token { + HS_TOK_NONE, HS_TOK_METHOD, HS_TOK_TARGET, HS_TOK_VERSION, + HS_TOK_HEADER_KEY, HS_TOK_HEADER_VAL, HS_TOK_CHUNK_BODY, HS_TOK_BODY, + HS_TOK_BODY_STREAM, HS_TOK_REQ_END, HS_TOK_EOF, HS_TOK_ERROR +}; + +enum hs_state { + ST, MT, MS, TR, TS, VN, RR, RN, HK, HS, HV, CL, TE, HR, HE, + ER, HN, BD, CS, CB, CE, CR, CN, CD, C1, C2, BR, HS_STATE_LEN +}; + +enum hs_char_type { + HS_SPC, HS_NL, HS_CR, HS_COLN, HS_TAB, HS_SCOLN, + HS_DIGIT, HS_HEX, HS_ALPHA, HS_TCHAR, HS_VCHAR, HS_ETC, HS_CHAR_TYPE_LEN +}; + // prototypes void hs_add_server_sock_events(struct http_server_s* serv); @@ -494,8 +480,7 @@ void hs_server_init(struct http_server_s* serv); void hs_delete_events(struct http_request_s* request); void hs_add_events(struct http_request_s* request); void hs_add_write_event(struct http_request_s* request); - -void hs_exec_response_handler(http_request_t* request, void (*handler)(http_request_t*)); +void hs_process_tokens(http_request_t* request); #ifdef KQUEUE @@ -596,276 +581,340 @@ char const * hs_status_text[] = { "", "", "", "", "", "", "", "", "", "" }; -// *** http parser *** +static int const hs_transitions[] = { +// spc \n \r : \t ; 0-9 a-fA-f g-zG-Z tchar vchar etc +/* ST start */ BR, BR, BR, BR, BR, BR, BR, MT, MT, MT, BR, BR, +/* MT method */ MS, BR, BR, BR, BR, BR, MT, MT, MT, MT, BR, BR, +/* MS methodsp */ BR, BR, BR, BR, BR, BR, TR, TR, TR, TR, TR, BR, +/* TR target */ TS, BR, BR, TR, BR, TR, TR, TR, TR, TR, TR, BR, +/* TS targetsp */ BR, BR, BR, BR, BR, BR, VN, VN, VN, VN, VN, BR, +/* VN version */ BR, BR, RR, BR, BR, BR, VN, VN, VN, VN, VN, BR, +/* RR rl \r */ BR, RN, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, +/* RN rl \n */ BR, BR, BR, BR, BR, BR, HK, HK, HK, HK, BR, BR, +/* HK headkey */ BR, BR, BR, HS, BR, BR, HK, HK, HK, HK, BR, BR, +/* HS headspc */ HS, HS, HS, HV, HS, HV, HV, HV, HV, HV, HV, BR, +/* HV headval */ HV, BR, HR, HV, HV, HV, HV, HV, HV, HV, HV, BR, +/* CL clval */ BR, BR, HR, BR, BR, BR, CL, BR, BR, BR, BR, BR, +/* TE teval */ TE, BR, HR, TE, TE, TE, TE, TE, TE, TE, TE, BR, +/* HR head\r */ BR, HE, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, +/* HE head\n */ BR, BR, ER, BR, BR, BR, HK, HK, HK, HK, BR, BR, +/* ER hend\r */ BR, HN, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, +/* HN hend\n */ BD, BD, BD, BD, BD, BD, BD, BD, BD, BD, BD, BD, +/* BD body */ BD, BD, BD, BD, BD, BD, BD, BD, BD, BD, BD, BD, +/* CS chksz */ BR, BR, CR, BR, BR, CE, CS, CS, BR, BR, BR, BR, +/* CB chkbd */ CB, CB, CB, CB, CB, CB, CB, CB, CB, CB, CB, CB, +/* CE chkext */ BR, BR, CR, CE, CE, CE, CE, CE, CE, CE, CE, BR, +/* CR chksz\r */ BR, CN, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, +/* CN chksz\n */ CB, CB, CB, CB, CB, CB, CB, CB, CB, CB, CB, CB, +/* CD chkend */ BR, BR, C1, BR, BR, BR, BR, BR, BR, BR, BR, BR, +/* C1 chkend\r */ BR, C2, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, +/* C2 chkend\n */ BR, BR, BR, BR, BR, BR, CS, CS, BR, BR, BR, BR +}; -#define HS_P_MATCH_HEADER(up, low, i) \ - if ((c == up[(int)i] || c == low[(int)i]) && i < (char)(sizeof(up) - 1)) i++; +static int const hs_ctype[] = { + HS_ETC, HS_ETC, HS_ETC, HS_ETC, HS_ETC, HS_ETC, HS_ETC, + HS_ETC, HS_ETC, HS_TAB, HS_NL, HS_ETC, HS_ETC, HS_CR, + HS_ETC, HS_ETC, HS_ETC, HS_ETC, HS_ETC, HS_ETC, HS_ETC, + HS_ETC, HS_ETC, HS_ETC, HS_ETC, HS_ETC, HS_ETC, HS_ETC, + HS_ETC, HS_ETC, HS_ETC, HS_ETC, HS_SPC, HS_TCHAR, HS_VCHAR, + HS_TCHAR, HS_TCHAR, HS_TCHAR, HS_TCHAR, HS_TCHAR, HS_VCHAR, HS_VCHAR, + HS_TCHAR, HS_TCHAR, HS_TCHAR, HS_TCHAR, HS_TCHAR, HS_VCHAR, HS_DIGIT, + HS_DIGIT, HS_DIGIT, HS_DIGIT, HS_DIGIT, HS_DIGIT, HS_DIGIT, HS_DIGIT, + HS_DIGIT, HS_DIGIT, HS_COLN, HS_SCOLN, HS_VCHAR, HS_VCHAR, HS_VCHAR, + HS_VCHAR, HS_VCHAR, HS_HEX, HS_HEX, HS_HEX, HS_HEX, HS_HEX, + HS_HEX, HS_ALPHA, HS_ALPHA, HS_ALPHA, HS_ALPHA, HS_ALPHA, HS_ALPHA, + HS_ALPHA, HS_ALPHA, HS_ALPHA, HS_ALPHA, HS_ALPHA, HS_ALPHA, HS_ALPHA, + HS_ALPHA, HS_ALPHA, HS_ALPHA, HS_ALPHA, HS_ALPHA, HS_ALPHA, HS_ALPHA, + HS_VCHAR, HS_VCHAR, HS_VCHAR, HS_TCHAR, HS_TCHAR, HS_TCHAR, HS_HEX, + HS_HEX, HS_HEX, HS_HEX, HS_HEX, HS_HEX, HS_ALPHA, HS_ALPHA, + HS_ALPHA, HS_ALPHA, HS_ALPHA, HS_ALPHA, HS_ALPHA, HS_ALPHA, HS_ALPHA, + HS_ALPHA, HS_ALPHA, HS_ALPHA, HS_ALPHA, HS_ALPHA, HS_ALPHA, HS_ALPHA, + HS_ALPHA, HS_ALPHA, HS_ALPHA, HS_ALPHA, HS_VCHAR, HS_TCHAR, HS_VCHAR, + HS_TCHAR, HS_ETC +}; -http_token_t hs_parse_error(http_parser_t* parser, int subtype) { - parser->len = 0; - parser->state = HTTP_PARSE_ERROR; - http_token_t token; - token.index = subtype; - token.type = HTTP_PARSE_ERROR; - return token; -} +static int const hs_token_start_states[] = { + 0, HS_TOK_METHOD, 0, HS_TOK_TARGET, 0, HS_TOK_VERSION, 0, 0, HS_TOK_HEADER_KEY, + 0, HS_TOK_HEADER_VAL, HS_TOK_HEADER_VAL, HS_TOK_HEADER_VAL, 0, 0, 0, 0, + HS_TOK_BODY, 0, HS_TOK_CHUNK_BODY, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +// *** input stream *** -http_token_t http_parse(http_parser_t* parser, char* input, int n) { - for (int i = parser->start; i < n; ++i, parser->start = i + 1, parser->len++) { - char c = input[i]; - switch (parser->state) { - case HTTP_METHOD: - if (c == ' ') { - http_token_t token; - token.index = parser->token_start_index; - token.type = parser->state; - token.len = parser->len; - parser->state = HTTP_TARGET; - parser->len = 0; - parser->token_start_index = i + 1; - return token; - } - break; - case HTTP_TARGET: - if (c == ' ') { - http_token_t token; - token.index = parser->token_start_index; - token.type = parser->state; - token.len = parser->len; - parser->state = HTTP_VERSION; - parser->token_start_index = i + 1; - parser->len = 0; - return token; - } - break; - case HTTP_VERSION: - if (c == '\r') { - parser->sub_state = HTTP_CR; - http_token_t token; - token.index = parser->token_start_index; - token.type = HTTP_VERSION; - token.len = parser->len; - return token; - } else if (parser->sub_state == HTTP_CR && c == '\n') { - parser->sub_state = 0; - parser->len = 0; - parser->token_start_index = i + 1; - parser->state = HTTP_HEADER_KEY; - } - break; - case HTTP_HEADER_KEY: - if (c == ':') { - parser->state = HTTP_HEADER_VALUE; - parser->sub_state = HTTP_LWS; - if (parser->len == parser->content_length_i + 1) { - HTTP_FLAG_SET(parser->flags, HS_PF_CONTENT_LENGTH); - } else if (parser->len == parser->transfer_encoding_i + 1) { - HTTP_FLAG_SET(parser->flags, HS_PF_TRANSFER_ENCODING); - } - parser->content_length_i = 0; - parser->transfer_encoding_i = 0; - http_token_t token; - token.index = parser->token_start_index; - token.type = HTTP_HEADER_KEY; - token.len = parser->len - 1; - return token; - } - HS_P_MATCH_HEADER( - HS_CONTENT_LENGTH_UP, - HS_CONTENT_LENGTH_LOW, - parser->content_length_i - ) - HS_P_MATCH_HEADER( - HS_TRANSFER_ENCODING_UP, - HS_TRANSFER_ENCODING_LOW, - parser->transfer_encoding_i - ) - break; - case HTTP_HEADER_VALUE: - if (parser->sub_state == HTTP_LWS && (c == ' ' || c == '\t' || c == '\r' || c == '\n')) { - continue; - } else if (parser->sub_state == HTTP_LWS) { - parser->sub_state = 0; - parser->len = 0; - parser->token_start_index = i; - if (HTTP_FLAG_CHECK(parser->flags, HS_PF_CONTENT_LENGTH)) { - parser->content_length *= 10; - parser->content_length += c - '0'; - } else if (HTTP_FLAG_CHECK(parser->flags, HS_PF_TRANSFER_ENCODING)) { - HS_P_MATCH_HEADER(HS_CHUNKED_UP, HS_CHUNKED_LOW, parser->transfer_encoding_i) - } - } else if (parser->sub_state != HTTP_LWS && c == '\r') { - parser->sub_state = HTTP_CR; - parser->state = HTTP_HEADER_END; - HTTP_FLAG_CLEAR(parser->flags, HS_PF_TRANSFER_ENCODING); - HTTP_FLAG_CLEAR(parser->flags, HS_PF_CONTENT_LENGTH); - if (parser->len == parser->transfer_encoding_i) { - HTTP_FLAG_SET(parser->flags, HS_PF_CHUNKED); - } - parser->transfer_encoding_i = 0; - if (parser->header_count == HTTP_MAX_HEADER_COUNT) { - return hs_parse_error(parser, HTTP_ERR_BAD_REQUEST); - } - parser->header_count++; - http_token_t token; - token.index = parser->token_start_index; - token.type = HTTP_HEADER_VALUE; - token.len = parser->len; - return token; - } else if (HTTP_FLAG_CHECK(parser->flags, HS_PF_CONTENT_LENGTH)) { - int64_t new_content_length = parser->content_length * 10l; - new_content_length += c - '0'; - if (new_content_length > HTTP_MAX_CONTENT_LENGTH) { - return hs_parse_error(parser, HTTP_ERR_PAYLOAD_TOO_LARGE); - } else { - parser->content_length = (int)new_content_length; - } - } else if (HTTP_FLAG_CHECK(parser->flags, HS_PF_TRANSFER_ENCODING)) { - HS_P_MATCH_HEADER(HS_CHUNKED_UP, HS_CHUNKED_LOW, parser->transfer_encoding_i) - } - break; - case HTTP_HEADER_END: - if (parser->sub_state == 0 && c == '\r') { - parser->sub_state = HTTP_CR; - } else if (parser->sub_state == HTTP_CR && c == '\n') { - parser->sub_state = HTTP_CRLF; - } else if (parser->sub_state == HTTP_CRLF && c == '\r') { - parser->sub_state = 0; - parser->state = HTTP_BODY; - http_token_t token; - token.index = i + 2; - parser->body_start_index = token.index; - token.type = HTTP_BODY; - if (HTTP_FLAG_CHECK(parser->flags, HS_PF_CHUNKED)) { - token.len = HTTP_CHUNKED_LEN; - } else { - token.len = parser->content_length; - } - parser->start++; - return token; - } else if (parser->sub_state == HTTP_CRLF && c != '\r') { - parser->sub_state = 0; - parser->len = 0; - parser->token_start_index = i; - i--; - parser->state = HTTP_HEADER_KEY; - } - break; +int hs_stream_read_socket(hs_stream_t* stream, int socket) { + if (!stream->buf) { + //stream->server->memused += HTTP_REQUEST_BUF_SIZE; + stream->buf = (char*)calloc(1, HTTP_REQUEST_BUF_SIZE); + assert(stream->buf != NULL); + stream->capacity = HTTP_REQUEST_BUF_SIZE; + } + int bytes; + do { + bytes = read( + socket, + stream->buf + stream->length, + stream->capacity - stream->length + ); + if (bytes > 0) { + stream->length += bytes; + stream->total_bytes += bytes; } - if (parser->len >= HTTP_MAX_TOKEN_LENGTH && parser->state != HTTP_BODY) { - return hs_parse_error(parser, HTTP_ERR_BAD_REQUEST); + if ( + stream->length == stream->capacity && + stream->capacity != HTTP_MAX_REQUEST_BUF_SIZE + ) { + //stream->server->memused -= stream->capacity; + stream->capacity *= 2; + if (stream->capacity > HTTP_MAX_REQUEST_BUF_SIZE) { + stream->capacity = HTTP_MAX_REQUEST_BUF_SIZE; + } + //stream->server->memused += stream->capacity; + stream->buf = (char*)realloc(stream->buf, stream->capacity); + assert(stream->buf != NULL); } - } - http_token_t token = { 0, 0, 0 }; - token.type = HTTP_NONE; - return token; + //printf("bytes: %d\n", bytes); + } while (bytes > 0); + return bytes == 0 ? 0 : 1; } -// When we detect that the full chunk exists in the read buffer we can -// immediately emit the HTTP_CHUNK_BODY token and skip reading forward to the -// end of the chunk -http_token_t http_gen_body_token(http_parser_t* parser) { - http_token_t token; - token.index = parser->token_start_index; - token.type = HTTP_CHUNK_BODY; - token.len = parser->content_length; - parser->start = parser->token_start_index + parser->content_length; - parser->len = parser->content_length; - parser->state = HTTP_CHUNK_BODY_END; +int hs_stream_next(hs_stream_t* stream, char* c) { + HTTP_FLAG_CLEAR(stream->flags, HS_SF_CONSUMED); + if (stream->index >= stream->length) return 0; + *c = stream->buf[stream->index]; + return 1; +} + +void hs_stream_consume(hs_stream_t* stream) { + if (HTTP_FLAG_CHECK(stream->flags, HS_SF_CONSUMED)) return; + HTTP_FLAG_SET(stream->flags, HS_SF_CONSUMED); + stream->index++; + int new_len = stream->token.len + 1; + stream->token.len = stream->token.type == 0 ? 0 : new_len; +} + +void hs_stream_begin_token(hs_stream_t* stream, int token_type) { + stream->token.index = stream->index; + stream->token.type = token_type; +} + +int hs_stream_jump(hs_stream_t* stream, int offset) { + if (stream->index + offset > stream->length) return 0; + stream->index += offset; + int new_len = stream->token.len + offset; + stream->token.len = stream->token.type == 0 ? 0 : new_len; + HTTP_FLAG_SET(stream->flags, HS_SF_CONSUMED); + return 1; +} + +int hs_stream_jumpall(hs_stream_t* stream) { + int offset = stream->length - stream->index; + stream->index += offset; + int new_len = stream->token.len + offset; + HTTP_FLAG_SET(stream->flags, HS_SF_CONSUMED); + stream->token.len = stream->token.type == 0 ? 0 : new_len; + return offset; +} + +void hs_stream_anchor(hs_stream_t* stream) { + stream->anchor = stream->index; +} + +http_token_t hs_stream_emit(hs_stream_t* stream) { + http_token_t token = stream->token; + http_token_t none = {0}; + stream->token = none; return token; } -http_token_t http_chunk_parse(http_request_t* request, char* input, int n) { - http_parser_t* parser = &request->parser; - for (int i = parser->start; i < n; ++i, parser->start = i + 1, parser->len++) { - char c = input[i]; - int remaining = n - (i + 1); - switch (parser->state) { - case HTTP_CHUNK_SIZE: - if (c == ';') { - parser->state = HTTP_CHUNK_EXTN; - } else if (c == '\n') { - parser->token_start_index = i + 1; - parser->len = 0; - if (remaining >= parser->content_length) { - // Full chunk exists in buffer - return http_gen_body_token(parser); - } - parser->state = HTTP_CHUNK_BODY; - } else if (c == '\r') { - break; - } else if (c >= 'A' && c <= 'F') { - parser->content_length *= 0x10; - parser->content_length += c - 55; - } else if (c >= 'a' && c <= 'f') { - parser->content_length *= 0x10; - parser->content_length += c - 87; - } else if (c >= '0' && c <= '9') { - parser->content_length *= 0x10; - parser->content_length += c - '0'; - } - break; - case HTTP_CHUNK_EXTN: - if (c == '\n') { - if (remaining >= parser->content_length) { - // Full chunk exists in buffer - parser->token_start_index = i + 1; - return http_gen_body_token(parser); - } - parser->token_start_index = i + 1; - parser->state = HTTP_CHUNK_BODY; - } - break; - case HTTP_CHUNK_BODY: - if (remaining >= parser->content_length) { - // The remaining portion of the chunk exists in the read buffer - return http_gen_body_token(parser); - } - break; - case HTTP_CHUNK_BODY_END: - if (c == '\n') { - parser->state = HTTP_CHUNK_SIZE; - parser->content_length = 0; - parser->len = 0; - parser->token_start_index = i + 1; - } - break; - } - } - // This code is reached when we come to the end of our read buffer and no - // token has been emitted during this call. If we are part way through parsing - // a token at the end of the buffer or the next token would require us to grow - // the buffer then we want to reset the parser to overwrite old chunks so that - // we don't need to grow the buffer. - if (parser->token_start_index != parser->body_start_index) { - // Next time, start parsing as if the current token has been shifted to the - // start of the http body. - parser->start = parser->body_start_index + parser->len - 1; - int tsi = parser->token_start_index; - parser->token_start_index = parser->body_start_index; - // This will make the next read overwrite the old bytes - request->bytes = parser->start; - // If parser->len is 1 then there is no current partial token. This is kind - // of ugly because len gets incremented in the for loop before we get here - // so we need to check for 1 instead of 0 - if (parser->len > 1) { - char* dst = input + parser->body_start_index; - char const* src = input + tsi; - // Copy partial token to beginning of body - memcpy(dst, src, n - tsi); - } +int hs_stream_is_maxcap(hs_stream_t* stream) { + return stream->length == HTTP_MAX_REQUEST_BUF_SIZE; +} + +int hs_stream_can_contain(hs_stream_t* stream, int64_t size) { + return HTTP_MAX_REQUEST_BUF_SIZE - stream->anchor >= size; +} + +void hs_stream_shift(hs_stream_t* stream) { + if (stream->token.index == stream->anchor) return; + if (stream->token.len > 0) { + char* dst = stream->buf + stream->anchor; + char const* src = stream->buf + stream->token.index; + int bytes = stream->length - stream->token.index; + memcpy(dst, src, bytes); } - http_token_t token = { 0, 0, 0 }; - token.type = HTTP_NONE; - return token; + stream->token.index = stream->anchor; + stream->index = stream->token.len + stream->anchor; + stream->length = stream->anchor + stream->token.len; } -void http_parse_start_chunk_mode(http_parser_t* parser) { - parser->token_start_index = parser->start; - parser->content_length = 0; - parser->state = HTTP_CHUNK_SIZE; +// *** http parser *** + +int hs_augment_transition(int to, uint8_t* flags) { + switch (to) { + case C2: + if (*flags & HS_PF_CKEND) { + HTTP_FLAG_CLEAR(*flags, HS_PF_CKEND); + return ST; + } + break; + case BD: + if (*flags & HS_PF_CHUNKED) { + HTTP_FLAG_CLEAR(*flags, HS_PF_CHUNKED); + return CS; + } + break; + case HV: + if (*flags & HS_PF_IN_CONTENT_LEN) { + HTTP_FLAG_CLEAR(*flags, HS_PF_IN_CONTENT_LEN); + return CL; + } else if (*flags & HS_PF_IN_TRANSFER_ENC) { + HTTP_FLAG_CLEAR(*flags, HS_PF_IN_TRANSFER_ENC); + return TE; + } + } + return to; } +#define HS_MATCH(str, flag) \ + in_bounds = parser->match_index < (int)sizeof(str) - 1; \ + m = in_bounds ? str[parser->match_index] : m; \ + low = c >= 'A' && c <= 'Z' ? c + 32 : c; \ + if (low != m) HTTP_FLAG_CLEAR(parser->flags, flag); \ + +http_token_t hs_transition_action( + http_parser_t* parser, + hs_stream_t* stream, + char c, + int8_t from, + int8_t to +) { + http_token_t emitted = {0}; + switch (from) { + case RN: + case HE: + HTTP_FLAG_SET(parser->flags, HS_PF_IN_CONTENT_LEN); + HTTP_FLAG_SET(parser->flags, HS_PF_IN_TRANSFER_ENC); + break; + case HN: + hs_stream_anchor(stream); + break; + case C1: + HTTP_FLAG_SET(parser->flags, HS_PF_CKEND); + break; + } + if (from == HN && to == CS) { + HTTP_FLAG_SET(parser->flags, HS_PF_CKEND); + emitted.type = HS_TOK_BODY_STREAM; + } + if (from != to) { + int type = hs_token_start_states[to]; + if (type != HS_TOK_NONE) hs_stream_begin_token(stream, type); + if (to == TE) HTTP_FLAG_SET(parser->flags, HS_PF_CHUNKED); + if (to == BD && !hs_stream_can_contain(stream, parser->content_length)) { + parser->inject_token = HS_TOK_BODY_STREAM; + } + parser->match_index = 0; + } + char low, m = '\0'; + int in_bounds = 0; + // generic to + switch (to) { + case ST: + emitted.type = HS_TOK_REQ_END; + break; + case MS: + case TS: + case RR: + case HS: + case HR: + emitted = hs_stream_emit(stream); + break; + case HN: + if (parser->content_length == 0) emitted.type = HS_TOK_BODY; + parser->inject_token = HS_TOK_REQ_END; + break; + case HK: + HS_MATCH("transfer-encoding", HS_PF_IN_TRANSFER_ENC) + HS_MATCH("content-length", HS_PF_IN_CONTENT_LEN) + parser->match_index++; + break; + case TE: + HS_MATCH("chunked", HS_PF_CHUNKED) + parser->match_index++; + break; + case CL: + parser->content_length *= 10; + parser->content_length += c - '0'; + break; + case CS: + if (c != '0') HTTP_FLAG_CLEAR(parser->flags, HS_PF_CKEND); + if (c >= 'A' && c <= 'F') { + parser->content_length *= 0x10; + parser->content_length += c - 55; + } else if (c >= 'a' && c <= 'f') { + parser->content_length *= 0x10; + parser->content_length += c - 87; + } else if (c >= '0' && c <= '9') { + parser->content_length *= 0x10; + parser->content_length += c - '0'; + } + break; + case CB: + if (hs_stream_jump(stream, parser->content_length)) { + emitted = hs_stream_emit(stream); + parser->state = CD; + parser->content_length = 0; + } else { + int bytes = hs_stream_jumpall(stream); + parser->content_length -= bytes; + } + break; + case BD: + if (hs_stream_jump(stream, parser->content_length)) { + emitted = hs_stream_emit(stream); + parser->state = ST; + parser->inject_token = HS_TOK_REQ_END; + parser->content_length = 0; + } else if (hs_stream_is_maxcap(stream)) { + hs_stream_begin_token(stream, HS_TOK_CHUNK_BODY); + hs_stream_jumpall(stream); + emitted = hs_stream_emit(stream); + hs_stream_shift(stream); + } else { + hs_stream_jumpall(stream); + } + break; + case BR: + // bad request + break; + } + return emitted; +} + +int hs_inject_token(http_parser_t* parser, http_token_t* token) { + token->type = parser->inject_token; + parser->inject_token = 0; + return token->type; +} + +http_token_t http_parse(http_parser_t* parser, hs_stream_t* stream) { + char c = 0; + http_token_t token = {0}; + if (hs_inject_token(parser, &token)) return token; + while (hs_stream_next(stream, &c)) { + int type = hs_ctype[(int)c]; + int to = hs_transitions[parser->state * HS_CHAR_TYPE_LEN + type]; + to = hs_augment_transition(to, &parser->flags); + int from = parser->state; + parser->state = to; + http_token_t emitted = hs_transition_action(parser, stream, c, from, to); + hs_stream_consume(stream); + if (emitted.type != HS_TOK_NONE) return emitted; + } + if (parser->state >= CS && parser->state <= C2) hs_stream_shift(stream); + hs_inject_token(parser, &token); + return token; +} // *** http server *** void http_token_dyn_push(http_token_dyn_t* dyn, http_token_t a) { @@ -895,98 +944,42 @@ void hs_bind_localhost(int s, struct sockaddr_in* addr, int port) { } } -int hs_read_client_socket(http_request_t* session) { - if (!session->buf) { - session->server->memused += HTTP_REQUEST_BUF_SIZE; - session->buf = (char*)calloc(1, HTTP_REQUEST_BUF_SIZE); - assert(session->buf != NULL); - session->capacity = HTTP_REQUEST_BUF_SIZE; - http_token_dyn_init(&session->tokens, 32); - } - int bytes; - do { - bytes = read( - session->socket, - session->buf + session->bytes, - session->capacity - session->bytes - ); - if (bytes > 0) session->bytes += bytes; - if (session->bytes == session->capacity) { - session->server->memused -= session->capacity; - session->capacity *= 2; - session->server->memused += session->capacity; - session->buf = (char*)realloc(session->buf, session->capacity); - assert(session->buf != NULL); - } - } while (bytes > 0); - return bytes == 0 ? 0 : 1; -} - int hs_write_client_socket(http_request_t* session) { int bytes = write( session->socket, - session->buf + session->written, - session->bytes - session->written + session->stream.buf + session->stream.total_bytes, + session->stream.length - session->stream.total_bytes ); - if (bytes > 0) session->written += bytes; + if (bytes > 0) session->stream.total_bytes += bytes; return errno == EPIPE ? 0 : 1; } void hs_free_buffer(http_request_t* session) { - if (session->buf) { - free(session->buf); - session->server->memused -= session->capacity; - session->buf = NULL; - free(session->tokens.buf); - session->tokens.buf = NULL; + if (session->stream.buf) { + free(session->stream.buf); + //session->server->memused -= session->stream.capacity; + session->stream.buf = NULL; } } -void hs_parse_tokens(http_request_t* session) { - http_token_t token; - int chunk_start = 0; - do { - token = http_parse(&session->parser, session->buf, session->bytes); - if (token.type != HTTP_NONE) { - session->token = token; - http_token_dyn_push(&session->tokens, token); - } - chunk_start = token.type == HTTP_BODY && token.len == HTTP_CHUNKED_LEN; - } while (token.type != HTTP_NONE && !chunk_start); -} - void hs_init_session(http_request_t* session) { - session->flags = 0; - session->flags |= HTTP_AUTOMATIC; + session->flags = HTTP_AUTOMATIC; + session->i = 0; session->parser = (http_parser_t){ }; - session->bytes = 0; - session->written = 0; - session->buf = NULL; - session->token.len = 0; - session->token.index = 0; - session->token.type = HTTP_NONE; -} - -int hs_parsing_headers(http_request_t* request) { - return request->token.type != HTTP_BODY; -} - -int hs_reading_body(http_request_t* request) { - if ( - request->token.type != HTTP_BODY || - request->token.len == 0 || - request->token.len == HTTP_CHUNKED_LEN - ) { - return 0; + session->stream = (hs_stream_t){ }; + if (session->tokens.buf) { + free(session->tokens.buf); + session->tokens.buf = NULL; } - int size = request->token.index + request->token.len; - return request->bytes < size; + http_token_dyn_init(&session->tokens, 32); } void hs_end_session(http_request_t* session) { hs_delete_events(session); close(session->socket); hs_free_buffer(session); + free(session->tokens.buf); + session->tokens.buf = NULL; free(session); } @@ -995,8 +988,9 @@ void hs_reset_timeout(http_request_t* request, int time) { } void hs_write_response(http_request_t* request) { + //printf("writing!\n"); if (!hs_write_client_socket(request)) { return hs_end_session(request); } - if (request->written != request->bytes) { + if (request->stream.total_bytes != request->stream.length) { // All bytes of the body were not written and we need to wait until the // socket is writable again to complete the write hs_add_write_event(request); @@ -1008,29 +1002,9 @@ void hs_write_response(http_request_t* request) { request->state = HTTP_SESSION_WRITE; hs_reset_timeout(request, HTTP_REQUEST_TIMEOUT); hs_free_buffer(request); - HTTP_FLAG_CLEAR(request->flags, HTTP_RESPONSE_READY); - hs_exec_response_handler(request, request->chunk_cb); - } else if (HTTP_FLAG_CHECK(request->flags, HTTP_KEEP_ALIVE)) { - // All bytes of the response were successfully written. However the - // keep-alive flag was set so we don't close the connection, we just clean - // up - request->state = HTTP_SESSION_INIT; - hs_free_buffer(request); - hs_reset_timeout(request, HTTP_KEEP_ALIVE_TIMEOUT); + request->chunk_cb(request); } else { - // All response bytes were written and the connection should be closed - return hs_end_session(request); - } -} - -void hs_exec_response_handler(http_request_t* request, void (*handler)(http_request_t*)) { - handler(request); - if (HTTP_FLAG_CHECK(request->flags, HTTP_RESPONSE_READY)) { - // The request handler has been called and a response is immediately ready. - hs_write_response(request); - } else { - // The response is not ready immediately and will be written out later. - HTTP_FLAG_SET(request->flags, HTTP_RESPONSE_PAUSED); + hs_process_tokens(request); } } @@ -1043,93 +1017,103 @@ void hs_error_response(http_request_t* request, int code, char const * message) hs_write_response(request); } +int hs_has_buffered_tokens(http_request_t* request) { + return request->i < request->tokens.size; +} + +char const* tokens[] = { + "HS_TOK_NONE", "HS_TOK_METHOD", "HS_TOK_TARGET", "HS_TOK_VERSION", + "HS_TOK_HEADER_KEY", "HS_TOK_HEADER_VAL", "HS_TOK_CHUNK_BODY", "HS_TOK_BODY", + "HS_TOK_BODY_STREAM", "HS_TOK_REQ_END", "HS_TOK_EOF" +}; + +void hs_read_and_parse_tokens(http_request_t* request) { + //printf("read state set\n"); + request->state = HTTP_SESSION_READ; + http_token_t token = {0}; + hs_reset_timeout(request, HTTP_REQUEST_TIMEOUT); + int rc = hs_stream_read_socket(&request->stream, request->socket); + //printf("rc: %d\n", rc); + do { + token = http_parse(&request->parser, &request->stream); + //printf("EMITTED: %s, len: %d UNREAD: %d\n", tokens[token.type], token.len, request->stream.length - request->stream.index); + if (token.type != HS_TOK_NONE) { + http_token_dyn_push(&request->tokens, token); + } + } while (token.type != HS_TOK_NONE); + token.type = HS_TOK_EOF; + if (rc == 0) http_token_dyn_push(&request->tokens, token); +} + +void hs_process_tokens(http_request_t* request) { + while (request->i < request->tokens.size) { + http_token_t token = request->tokens.buf[request->i]; + request->i++; + //printf("PROCESSING: %s, %.*s, UNREAD: %d\n", tokens[token.type], token.len, request->stream.buf + token.index, request->stream.length - request->stream.index); + if (token.type == HS_TOK_EOF) { + return hs_end_session(request); + } else if (token.type == HS_TOK_ERROR) { + hs_error_response(request, 400, "Bad Request"); + } else if (token.type == HS_TOK_BODY) { + //printf("nop state set\n"); + request->state = HTTP_SESSION_NOP; + request->server->request_handler(request); + break; + } else if (token.type == HS_TOK_CHUNK_BODY) { + //printf("nop state set\n"); + request->state = HTTP_SESSION_NOP; + request->chunk_cb(request); + break; + } else if (token.type == HS_TOK_REQ_END) { + if (HTTP_FLAG_CHECK(request->flags, HTTP_KEEP_ALIVE)) { + //printf("keep-alive!\n"); + request->state = HTTP_SESSION_INIT; + hs_free_buffer(request); + hs_reset_timeout(request, HTTP_KEEP_ALIVE_TIMEOUT); + } else { + return hs_end_session(request); + } + } + } +} + // Application requesting next chunk of request body. void http_request_read_chunk( struct http_request_s* request, void (*chunk_cb)(struct http_request_s*) ) { + //printf("reading chunk!\n"); request->chunk_cb = chunk_cb; - http_token_t token = http_chunk_parse(request, request->buf, request->bytes); - if (token.type == HTTP_CHUNK_BODY) { - // The next chunk was in the read buffer and is ready. - request->token = token; - chunk_cb(request); + if (hs_has_buffered_tokens(request)) { + hs_process_tokens(request); } else { - // No chunk is in the read buffer, continue reading the socket. - if (!hs_read_client_socket(request)) { return hs_end_session(request); } - http_token_t token = http_chunk_parse(request, request->buf, request->bytes); - if (token.type == HTTP_CHUNK_BODY) { - // A chunk was in the kernel network buffer - request->token = token; - request->chunk_cb(request); - } else { - // No chunk ready, wait for IO. - request->state = HTTP_SESSION_READ_CHUNK; - } + hs_read_and_parse_tokens(request); + hs_process_tokens(request); } } // This is the heart of the request logic. This is the state machine that // controls what happens when an IO event is received. void http_session(http_request_t* request) { - http_token_t token; switch (request->state) { case HTTP_SESSION_INIT: hs_init_session(request); - request->state = HTTP_SESSION_READ_HEADERS; + request->state = HTTP_SESSION_READ; if (request->server->memused > HTTP_MAX_TOTAL_EST_MEM_USAGE) { return hs_error_response(request, 503, "Service Unavailable"); } // fallthrough - case HTTP_SESSION_READ_HEADERS: - if (!hs_read_client_socket(request)) { return hs_end_session(request); } - hs_reset_timeout(request, HTTP_REQUEST_TIMEOUT); - hs_parse_tokens(request); - if (request->token.type == HTTP_PARSE_ERROR) { - switch (request->token.index) { - case HTTP_ERR_BAD_REQUEST: - return hs_error_response(request, 400, "Bad Request"); - case HTTP_ERR_PAYLOAD_TOO_LARGE: - return hs_error_response(request, 413, "Payload Too Large"); - } - } else if (hs_reading_body(request)) { - // The full request body has not been ready. Need to wait for more IO. - request->state = HTTP_SESSION_READ_BODY; - } else if (!hs_parsing_headers(request)) { - if (request->parser.flags & HS_PF_CHUNKED) { - // Set state to NOP for chunked requests. This means we won't handle - // any read events on the socket until the application has requested - // to read a chunk. - request->state = HTTP_SESSION_NOP; - http_parse_start_chunk_mode(&request->parser); - } - return hs_exec_response_handler(request, request->server->request_handler); - } - break; - case HTTP_SESSION_READ_BODY: - if (!hs_read_client_socket(request)) { return hs_end_session(request); } - hs_reset_timeout(request, HTTP_REQUEST_TIMEOUT); - if (!hs_reading_body(request)) { - // Full body has been read into the read buffer. Call the application - // request handler - return hs_exec_response_handler(request, request->server->request_handler); - } - // Full body has still not been read. Wait for more IO. - break; - case HTTP_SESSION_READ_CHUNK: - if (!hs_read_client_socket(request)) { return hs_end_session(request); } - hs_reset_timeout(request, HTTP_REQUEST_TIMEOUT); - token = http_chunk_parse(request, request->buf, request->bytes); - if (token.type == HTTP_CHUNK_BODY) { - // Chunk is ready call the chunk handler - request->token = token; - request->state = HTTP_SESSION_NOP; - request->chunk_cb(request); - } + case HTTP_SESSION_READ: + //printf("io clabback!\n"); + hs_read_and_parse_tokens(request); + hs_process_tokens(request); break; case HTTP_SESSION_WRITE: hs_write_response(request); break; + case HTTP_SESSION_NOP: + //printf("NOP!\n"); + break; } } @@ -1198,12 +1182,12 @@ int http_server_loop(http_server_t* server) { // *** http request *** http_string_t http_get_token_string(http_request_t* request, int token_type) { - http_string_t str = { 0, 0 }; + http_string_t str = {0}; if (request->tokens.buf == NULL) return str; for (int i = 0; i < request->tokens.size; i++) { http_token_t token = request->tokens.buf[i]; if (token.type == token_type) { - str.buf = &request->buf[token.index]; + str.buf = &request->stream.buf[token.index]; str.len = token.len; return str; } @@ -1221,15 +1205,15 @@ int hs_case_insensitive_cmp(char const * a, char const * b, int len) { } http_string_t http_request_method(http_request_t* request) { - return http_get_token_string(request, HTTP_METHOD); + return http_get_token_string(request, HS_TOK_METHOD); } http_string_t http_request_target(http_request_t* request) { - return http_get_token_string(request, HTTP_TARGET); + return http_get_token_string(request, HS_TOK_TARGET); } http_string_t http_request_body(http_request_t* request) { - return http_get_token_string(request, HTTP_BODY); + return http_get_token_string(request, HS_TOK_BODY); } int hs_assign_iteration_headers( @@ -1239,15 +1223,15 @@ int hs_assign_iteration_headers( int* iter ) { http_token_t token = request->tokens.buf[*iter]; - if (request->tokens.buf[*iter].type == HTTP_BODY) return 0; + if (request->tokens.buf[*iter].type == HS_TOK_BODY) return 0; *key = (http_string_t) { - .buf = &request->buf[token.index], + .buf = &request->stream.buf[token.index], .len = token.len }; (*iter)++; token = request->tokens.buf[*iter]; *val = (http_string_t) { - .buf = &request->buf[token.index], + .buf = &request->stream.buf[token.index], .len = token.len }; return 1; @@ -1262,7 +1246,7 @@ int http_request_iterate_headers( if (*iter == 0) { for ( ; *iter < request->tokens.size; (*iter)++) { http_token_t token = request->tokens.buf[*iter]; - if (token.type == HTTP_HEADER_KEY) { + if (token.type == HS_TOK_HEADER_KEY) { return hs_assign_iteration_headers(request, key, val, iter); } } @@ -1277,11 +1261,11 @@ http_string_t http_request_header(http_request_t* request, char const * key) { int len = strlen(key); for (int i = 0; i < request->tokens.size; i++) { http_token_t token = request->tokens.buf[i]; - if (token.type == HTTP_HEADER_KEY && token.len == len) { - if (hs_case_insensitive_cmp(&request->buf[token.index], key, len)) { + if (token.type == HS_TOK_HEADER_KEY && token.len == len) { + if (hs_case_insensitive_cmp(&request->stream.buf[token.index], key, len)) { token = request->tokens.buf[i + 1]; return (http_string_t) { - .buf = &request->buf[token.index], + .buf = &request->stream.buf[token.index], .len = token.len }; } @@ -1303,7 +1287,7 @@ void http_request_set_userdata(http_request_t* request, void* data) { } void hs_auto_detect_keep_alive(http_request_t* request) { - http_string_t str = http_get_token_string(request, HTTP_VERSION); + http_string_t str = http_get_token_string(request, HS_TOK_VERSION); if (str.buf == NULL) return; int version = str.buf[str.len - 1] == '1'; str = http_request_header(request, "Connection"); @@ -1328,9 +1312,10 @@ void http_request_connection(http_request_t* request, int directive) { } http_string_t http_request_chunk(struct http_request_s* request) { + http_token_t token = request->tokens.buf[request->i - 1]; return (http_string_t) { - .buf = &request->buf[request->token.index], - .len = request->token.len + .buf = &request->stream.buf[token.index], + .len = token.len }; } @@ -1373,16 +1358,16 @@ void grwprintf_init(grwprintf_t* ctx, int capacity, long* memused) { ctx->memused = memused; ctx->size = 0; ctx->buf = (char*)malloc(capacity); - *ctx->memused += capacity; + //*ctx->memused += capacity; assert(ctx->buf != NULL); ctx->capacity = capacity; } void grwmemcpy(grwprintf_t* ctx, char const * src, int size) { if (ctx->size + size > ctx->capacity) { - *ctx->memused -= ctx->capacity; + //*ctx->memused -= ctx->capacity; ctx->capacity = ctx->size + size; - *ctx->memused += ctx->capacity; + //*ctx->memused += ctx->capacity; ctx->buf = (char*)realloc(ctx->buf, ctx->capacity); assert(ctx->buf != NULL); } @@ -1396,9 +1381,9 @@ void grwprintf(grwprintf_t* ctx, char const * fmt, ...) { int bytes = vsnprintf(ctx->buf + ctx->size, ctx->capacity - ctx->size, fmt, args); if (bytes + ctx->size > ctx->capacity) { - *ctx->memused -= ctx->capacity; + //*ctx->memused -= ctx->capacity; while (bytes + ctx->size > ctx->capacity) ctx->capacity *= 2; - *ctx->memused += ctx->capacity; + //*ctx->memused += ctx->capacity; ctx->buf = (char*)realloc(ctx->buf, ctx->capacity); assert(ctx->buf != NULL); bytes += vsnprintf(ctx->buf + ctx->size, ctx->capacity - ctx->size, fmt, args); @@ -1453,18 +1438,12 @@ void http_end_response(http_request_t* request, http_response_t* response, grwpr } hs_free_buffer(request); free(response); - request->buf = printctx->buf; - request->written = 0; - request->bytes = printctx->size; - request->capacity = printctx->capacity; + request->stream.buf = printctx->buf; + request->stream.total_bytes = 0; + request->stream.length = printctx->size; + request->stream.capacity = printctx->capacity; request->state = HTTP_SESSION_WRITE; - // Signal that the response is ready for writing. - HTTP_FLAG_SET(request->flags, HTTP_RESPONSE_READY); - if (HTTP_FLAG_CHECK(request->flags, HTTP_RESPONSE_PAUSED)) { - // If the request has been paused waiting for the response clear the flag - HTTP_FLAG_CLEAR(request->flags, HTTP_RESPONSE_PAUSED); - http_session(request); - } + hs_write_response(request); } void http_respond(http_request_t* request, http_response_t* response) { @@ -1565,8 +1544,7 @@ void hs_delete_events(http_request_t* request) { int http_server_poll(http_server_t* serv) { struct kevent ev; - struct timespec ts; - memset(&ts, 0, sizeof(ts)); + struct timespec ts = {0}; int nev = kevent(serv->loop, NULL, 0, &ev, 1, &ts); if (nev <= 0) return nev; ev_cb_t* ev_cb = (ev_cb_t*)ev.udata; From 1fd1294e492196e5362ecf4a678ccefe91f0bea0 Mon Sep 17 00:00:00 2001 From: Jeremy Williams Date: Fri, 31 Jan 2020 14:40:03 -0500 Subject: [PATCH 02/20] Fix bugs --- httpserver.h | 83 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 51 insertions(+), 32 deletions(-) diff --git a/httpserver.h b/httpserver.h index beb47b0..613fcc4 100644 --- a/httpserver.h +++ b/httpserver.h @@ -385,18 +385,20 @@ typedef struct http_ev_cb_s { typedef struct { char* buf; int64_t total_bytes; - int capacity; - int length; - int index; - int anchor; + int32_t capacity; + int32_t length; + int32_t index; + int32_t anchor; http_token_t token; uint8_t flags; } hs_stream_t; typedef struct { int64_t content_length; - int match_index; - int inject_token; + int64_t body_consumed; + int32_t match_index; + int32_t inject_token; + int16_t header_count; int8_t state; uint8_t flags; } http_parser_t; @@ -429,7 +431,7 @@ typedef struct http_server_s { epoll_cb_t handler; epoll_cb_t timer_handler; #endif - long memused; + int64_t memused; int socket; int port; int loop; @@ -641,9 +643,9 @@ static int const hs_token_start_states[] = { // *** input stream *** -int hs_stream_read_socket(hs_stream_t* stream, int socket) { +int hs_stream_read_socket(hs_stream_t* stream, int socket, int64_t* memused) { if (!stream->buf) { - //stream->server->memused += HTTP_REQUEST_BUF_SIZE; + *memused += HTTP_REQUEST_BUF_SIZE; stream->buf = (char*)calloc(1, HTTP_REQUEST_BUF_SIZE); assert(stream->buf != NULL); stream->capacity = HTTP_REQUEST_BUF_SIZE; @@ -663,12 +665,12 @@ int hs_stream_read_socket(hs_stream_t* stream, int socket) { stream->length == stream->capacity && stream->capacity != HTTP_MAX_REQUEST_BUF_SIZE ) { - //stream->server->memused -= stream->capacity; + *memused -= stream->capacity; stream->capacity *= 2; if (stream->capacity > HTTP_MAX_REQUEST_BUF_SIZE) { stream->capacity = HTTP_MAX_REQUEST_BUF_SIZE; } - //stream->server->memused += stream->capacity; + *memused += stream->capacity; stream->buf = (char*)realloc(stream->buf, stream->capacity); assert(stream->buf != NULL); } @@ -809,14 +811,27 @@ http_token_t hs_transition_action( if (from != to) { int type = hs_token_start_states[to]; if (type != HS_TOK_NONE) hs_stream_begin_token(stream, type); - if (to == TE) HTTP_FLAG_SET(parser->flags, HS_PF_CHUNKED); - if (to == BD && !hs_stream_can_contain(stream, parser->content_length)) { - parser->inject_token = HS_TOK_BODY_STREAM; + switch (to) { + case HK: + parser->header_count++; + if (parser->header_count > HTTP_MAX_HEADER_COUNT) { + emitted.type = HS_TOK_ERROR; + } + break; + case TE: + HTTP_FLAG_SET(parser->flags, HS_PF_CHUNKED); + break; + case BD: + if (!hs_stream_can_contain(stream, parser->content_length)) { + parser->inject_token = HS_TOK_BODY_STREAM; + } + break; } parser->match_index = 0; } char low, m = '\0'; int in_bounds = 0; + int body_left = 0; // generic to switch (to) { case ST: @@ -830,8 +845,10 @@ http_token_t hs_transition_action( emitted = hs_stream_emit(stream); break; case HN: - if (parser->content_length == 0) emitted.type = HS_TOK_BODY; - parser->inject_token = HS_TOK_REQ_END; + if (parser->content_length == 0 && !HTTP_FLAG_CHECK(parser->flags, HS_PF_CHUNKED)) { + emitted.type = HS_TOK_BODY; + parser->inject_token = HS_TOK_REQ_END; + } break; case HK: HS_MATCH("transfer-encoding", HS_PF_IN_TRANSFER_ENC) @@ -870,22 +887,24 @@ http_token_t hs_transition_action( } break; case BD: - if (hs_stream_jump(stream, parser->content_length)) { + body_left = parser->content_length - parser->body_consumed; + if (hs_stream_jump(stream, body_left)) { emitted = hs_stream_emit(stream); parser->state = ST; parser->inject_token = HS_TOK_REQ_END; parser->content_length = 0; + parser->body_consumed = 0; } else if (hs_stream_is_maxcap(stream)) { hs_stream_begin_token(stream, HS_TOK_CHUNK_BODY); - hs_stream_jumpall(stream); + parser->body_consumed += hs_stream_jumpall(stream); emitted = hs_stream_emit(stream); hs_stream_shift(stream); } else { - hs_stream_jumpall(stream); + parser->body_consumed += hs_stream_jumpall(stream); } break; case BR: - // bad request + emitted.type = HS_TOK_ERROR; break; } return emitted; @@ -902,7 +921,7 @@ http_token_t http_parse(http_parser_t* parser, hs_stream_t* stream) { http_token_t token = {0}; if (hs_inject_token(parser, &token)) return token; while (hs_stream_next(stream, &c)) { - int type = hs_ctype[(int)c]; + int type = c < 0 ? HS_ETC : hs_ctype[(int)c]; int to = hs_transitions[parser->state * HS_CHAR_TYPE_LEN + type]; to = hs_augment_transition(to, &parser->flags); int from = parser->state; @@ -957,7 +976,7 @@ int hs_write_client_socket(http_request_t* session) { void hs_free_buffer(http_request_t* session) { if (session->stream.buf) { free(session->stream.buf); - //session->server->memused -= session->stream.capacity; + session->server->memused -= session->stream.capacity; session->stream.buf = NULL; } } @@ -1032,7 +1051,7 @@ void hs_read_and_parse_tokens(http_request_t* request) { request->state = HTTP_SESSION_READ; http_token_t token = {0}; hs_reset_timeout(request, HTTP_REQUEST_TIMEOUT); - int rc = hs_stream_read_socket(&request->stream, request->socket); + int rc = hs_stream_read_socket(&request->stream, request->socket, &request->server->memused); //printf("rc: %d\n", rc); do { token = http_parse(&request->parser, &request->stream); @@ -1054,7 +1073,7 @@ void hs_process_tokens(http_request_t* request) { return hs_end_session(request); } else if (token.type == HS_TOK_ERROR) { hs_error_response(request, 400, "Bad Request"); - } else if (token.type == HS_TOK_BODY) { + } else if (token.type == HS_TOK_BODY || token.type == HS_TOK_BODY_STREAM) { //printf("nop state set\n"); request->state = HTTP_SESSION_NOP; request->server->request_handler(request); @@ -1095,6 +1114,7 @@ void http_request_read_chunk( // This is the heart of the request logic. This is the state machine that // controls what happens when an IO event is received. void http_session(http_request_t* request) { + //printf("io clabback!\n"); switch (request->state) { case HTTP_SESSION_INIT: hs_init_session(request); @@ -1104,7 +1124,6 @@ void http_session(http_request_t* request) { } // fallthrough case HTTP_SESSION_READ: - //printf("io clabback!\n"); hs_read_and_parse_tokens(request); hs_process_tokens(request); break; @@ -1351,23 +1370,23 @@ typedef struct { char* buf; int capacity; int size; - long* memused; + int64_t* memused; } grwprintf_t; -void grwprintf_init(grwprintf_t* ctx, int capacity, long* memused) { +void grwprintf_init(grwprintf_t* ctx, int capacity, int64_t* memused) { ctx->memused = memused; ctx->size = 0; ctx->buf = (char*)malloc(capacity); - //*ctx->memused += capacity; + *ctx->memused += capacity; assert(ctx->buf != NULL); ctx->capacity = capacity; } void grwmemcpy(grwprintf_t* ctx, char const * src, int size) { if (ctx->size + size > ctx->capacity) { - //*ctx->memused -= ctx->capacity; + *ctx->memused -= ctx->capacity; ctx->capacity = ctx->size + size; - //*ctx->memused += ctx->capacity; + *ctx->memused += ctx->capacity; ctx->buf = (char*)realloc(ctx->buf, ctx->capacity); assert(ctx->buf != NULL); } @@ -1381,9 +1400,9 @@ void grwprintf(grwprintf_t* ctx, char const * fmt, ...) { int bytes = vsnprintf(ctx->buf + ctx->size, ctx->capacity - ctx->size, fmt, args); if (bytes + ctx->size > ctx->capacity) { - //*ctx->memused -= ctx->capacity; + *ctx->memused -= ctx->capacity; while (bytes + ctx->size > ctx->capacity) ctx->capacity *= 2; - //*ctx->memused += ctx->capacity; + *ctx->memused += ctx->capacity; ctx->buf = (char*)realloc(ctx->buf, ctx->capacity); assert(ctx->buf != NULL); bytes += vsnprintf(ctx->buf + ctx->size, ctx->capacity - ctx->size, fmt, args); From fd4e83510f1c55b16c63c5f271ab78dae6d032d4 Mon Sep 17 00:00:00 2001 From: Jeremy Williams Date: Tue, 4 Feb 2020 18:07:18 -0500 Subject: [PATCH 03/20] Continue parser improvements --- httpserver.h | 304 +++++++++++++++++++++++++++------------------------ 1 file changed, 161 insertions(+), 143 deletions(-) diff --git a/httpserver.h b/httpserver.h index 613fcc4..5d8f911 100644 --- a/httpserver.h +++ b/httpserver.h @@ -397,10 +397,9 @@ typedef struct { int64_t content_length; int64_t body_consumed; int32_t match_index; - int32_t inject_token; int16_t header_count; int8_t state; - uint8_t flags; + int8_t meta; } http_parser_t; typedef struct http_request_s { @@ -466,7 +465,7 @@ enum hs_token { }; enum hs_state { - ST, MT, MS, TR, TS, VN, RR, RN, HK, HS, HV, CL, TE, HR, HE, + ST, MT, MS, TR, TS, VN, RR, RN, HK, HS, HV, HR, HE, ER, HN, BD, CS, CB, CE, CR, CN, CD, C1, C2, BR, HS_STATE_LEN }; @@ -475,6 +474,25 @@ enum hs_char_type { HS_DIGIT, HS_HEX, HS_ALPHA, HS_TCHAR, HS_VCHAR, HS_ETC, HS_CHAR_TYPE_LEN }; +enum hs_meta_state { + M_WFK, M_ANY, M_MTE, M_MCL, M_CLV, M_MCK, M_SML, M_CHK, M_BIG, M_ZER, M_CSZ, + M_CBD, M_STR, M_BDY, M_LST, M_END, M_ERR +}; + +enum hs_meta_type { + HS_META_NOT_CONTENT_LEN, HS_META_NOT_TRANSFER_ENC, HS_META_END_KEY, + HS_META_END_VALUE, HS_META_END_HEADERS, HS_META_TYPE_LEN +}; + +#define HS_META_NOT_CHUNKED 0 +#define HS_META_LARGE_BODY 0 +#define HS_META_END_BODY 0 +#define HS_META_NON_ZERO 0 +#define HS_META_FINISH 0 +#define HS_META_END_CHK_SIZE 1 +#define HS_META_END_CHUNK 2 +#define HS_META_NEXT 0 + // prototypes void hs_add_server_sock_events(struct http_server_s* serv); @@ -584,33 +602,57 @@ char const * hs_status_text[] = { }; static int const hs_transitions[] = { -// spc \n \r : \t ; 0-9 a-fA-f g-zG-Z tchar vchar etc -/* ST start */ BR, BR, BR, BR, BR, BR, BR, MT, MT, MT, BR, BR, -/* MT method */ MS, BR, BR, BR, BR, BR, MT, MT, MT, MT, BR, BR, -/* MS methodsp */ BR, BR, BR, BR, BR, BR, TR, TR, TR, TR, TR, BR, -/* TR target */ TS, BR, BR, TR, BR, TR, TR, TR, TR, TR, TR, BR, -/* TS targetsp */ BR, BR, BR, BR, BR, BR, VN, VN, VN, VN, VN, BR, -/* VN version */ BR, BR, RR, BR, BR, BR, VN, VN, VN, VN, VN, BR, -/* RR rl \r */ BR, RN, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, -/* RN rl \n */ BR, BR, BR, BR, BR, BR, HK, HK, HK, HK, BR, BR, -/* HK headkey */ BR, BR, BR, HS, BR, BR, HK, HK, HK, HK, BR, BR, -/* HS headspc */ HS, HS, HS, HV, HS, HV, HV, HV, HV, HV, HV, BR, -/* HV headval */ HV, BR, HR, HV, HV, HV, HV, HV, HV, HV, HV, BR, -/* CL clval */ BR, BR, HR, BR, BR, BR, CL, BR, BR, BR, BR, BR, -/* TE teval */ TE, BR, HR, TE, TE, TE, TE, TE, TE, TE, TE, BR, -/* HR head\r */ BR, HE, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, -/* HE head\n */ BR, BR, ER, BR, BR, BR, HK, HK, HK, HK, BR, BR, -/* ER hend\r */ BR, HN, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, -/* HN hend\n */ BD, BD, BD, BD, BD, BD, BD, BD, BD, BD, BD, BD, -/* BD body */ BD, BD, BD, BD, BD, BD, BD, BD, BD, BD, BD, BD, -/* CS chksz */ BR, BR, CR, BR, BR, CE, CS, CS, BR, BR, BR, BR, -/* CB chkbd */ CB, CB, CB, CB, CB, CB, CB, CB, CB, CB, CB, CB, -/* CE chkext */ BR, BR, CR, CE, CE, CE, CE, CE, CE, CE, CE, BR, -/* CR chksz\r */ BR, CN, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, -/* CN chksz\n */ CB, CB, CB, CB, CB, CB, CB, CB, CB, CB, CB, CB, -/* CD chkend */ BR, BR, C1, BR, BR, BR, BR, BR, BR, BR, BR, BR, -/* C1 chkend\r */ BR, C2, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, -/* C2 chkend\n */ BR, BR, BR, BR, BR, BR, CS, CS, BR, BR, BR, BR +// A-Z G-Z +// spc \n \r : \t ; 0-9 a-f g-z tch vch etc +/* ST start */ BR, BR, BR, BR, BR, BR, BR, MT, MT, MT, BR, BR, +/* MT method */ MS, BR, BR, BR, BR, BR, MT, MT, MT, MT, BR, BR, +/* MS methodsp */ BR, BR, BR, BR, BR, BR, TR, TR, TR, TR, TR, BR, +/* TR target */ TS, BR, BR, TR, BR, TR, TR, TR, TR, TR, TR, BR, +/* TS targetsp */ BR, BR, BR, BR, BR, BR, VN, VN, VN, VN, VN, BR, +/* VN version */ BR, BR, RR, BR, BR, BR, VN, VN, VN, VN, VN, BR, +/* RR rl \r */ BR, RN, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, +/* RN rl \n */ BR, BR, BR, BR, BR, BR, HK, HK, HK, HK, BR, BR, +/* HK headkey */ BR, BR, BR, HS, BR, BR, HK, HK, HK, HK, BR, BR, +/* HS headspc */ HS, HS, HS, HV, HS, HV, HV, HV, HV, HV, HV, BR, +/* HV headval */ HV, BR, HR, HV, HV, HV, HV, HV, HV, HV, HV, BR, +/* HR head\r */ BR, HE, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, +/* HE head\n */ BR, BR, ER, BR, BR, BR, HK, HK, HK, HK, BR, BR, +/* ER hend\r */ BR, HN, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, +/* HN hend\n */ BD, BD, BD, BD, BD, BD, BD, BD, BD, BD, BD, BD, +/* BD body */ BD, BD, BD, BD, BD, BD, BD, BD, BD, BD, BD, BD, +/* CS chksz */ BR, BR, CR, BR, BR, CE, CS, CS, BR, BR, BR, BR, +/* CB chkbd */ CB, CB, CB, CB, CB, CB, CB, CB, CB, CB, CB, CB, +/* CE chkext */ BR, BR, CR, CE, CE, CE, CE, CE, CE, CE, CE, BR, +/* CR chksz\r */ BR, CN, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, +/* CN chksz\n */ CB, CB, CB, CB, CB, CB, CB, CB, CB, CB, CB, CB, +/* CD chkend */ BR, BR, C1, BR, BR, BR, BR, BR, BR, BR, BR, BR, +/* C1 chkend\r */ BR, C2, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, +/* C2 chkend\n */ BR, BR, BR, BR, BR, BR, CS, CS, BR, BR, BR, BR +}; + +static int const hs_meta_transitions[] = { +// not cl not te endkey endval end h +/* WFK wait */ M_WFK, M_WFK, M_WFK, M_ANY, M_END, +/* ANY matchkey */ M_MTE, M_MCL, M_WFK, M_ERR, M_END, +/* MTE matchte */ M_MTE, M_WFK, M_MCK, M_ERR, M_ERR, +/* MCL matchcl */ M_WFK, M_MCL, M_CLV, M_ERR, M_ERR, +/* CLV clvalue */ M_ERR, M_ERR, M_ERR, M_SML, M_ERR, +// no chk +/* MCK matchchk */ M_WFK, M_ERR, M_ERR, M_CHK, M_ERR, +// toobig +/* SML smallbdy */ M_BIG, M_SML, M_SML, M_SML, M_BDY, +/* CHK chunkbdy */ M_CHK, M_CHK, M_CHK, M_CHK, M_ZER, +/* BIG bigbody */ M_BIG, M_BIG, M_BIG, M_BIG, M_STR, +// nozer endsz +/* ZER zerochk */ M_CSZ, M_LST, M_ERR, M_ERR, M_ERR, +/* CSZ chksize */ M_CSZ, M_CBD, M_ERR, M_ERR, M_ERR, +// endbdy endchk +/* CBD readchk */ M_CBD, M_CBD, M_ZER, M_ERR, M_ERR, +/* STR readstr */ M_END, M_ERR, M_ERR, M_ERR, M_ERR, +/* BDY readbody */ M_END, M_ERR, M_ERR, M_ERR, M_ERR, +/* LST lastchk */ M_LST, M_END, M_END, M_ERR, M_ERR, +// fin +/* END reqend */ M_END, M_WFK, M_ERR, M_ERR, M_ERR }; static int const hs_ctype[] = { @@ -636,9 +678,12 @@ static int const hs_ctype[] = { }; static int const hs_token_start_states[] = { +//ST MT MS TR TS VN RR RN HK 0, HS_TOK_METHOD, 0, HS_TOK_TARGET, 0, HS_TOK_VERSION, 0, 0, HS_TOK_HEADER_KEY, - 0, HS_TOK_HEADER_VAL, HS_TOK_HEADER_VAL, HS_TOK_HEADER_VAL, 0, 0, 0, 0, - HS_TOK_BODY, 0, HS_TOK_CHUNK_BODY, 0, 0, 0, 0, 0, 0, 0, 0 +//HS HV HR HE ER HN BD CS CB CE CR CN + 0, HS_TOK_HEADER_VAL, 0, 0, 0, 0, HS_TOK_BODY, 0, HS_TOK_CHUNK_BODY, 0, 0, 0, +//CD C1 C2 + 0, 0, 0, }; // *** input stream *** @@ -700,11 +745,11 @@ void hs_stream_begin_token(hs_stream_t* stream, int token_type) { } int hs_stream_jump(hs_stream_t* stream, int offset) { + HTTP_FLAG_SET(stream->flags, HS_SF_CONSUMED); if (stream->index + offset > stream->length) return 0; stream->index += offset; int new_len = stream->token.len + offset; stream->token.len = stream->token.type == 0 ? 0 : new_len; - HTTP_FLAG_SET(stream->flags, HS_SF_CONSUMED); return 1; } @@ -728,12 +773,8 @@ http_token_t hs_stream_emit(hs_stream_t* stream) { return token; } -int hs_stream_is_maxcap(hs_stream_t* stream) { - return stream->length == HTTP_MAX_REQUEST_BUF_SIZE; -} - int hs_stream_can_contain(hs_stream_t* stream, int64_t size) { - return HTTP_MAX_REQUEST_BUF_SIZE - stream->anchor >= size; + return HTTP_MAX_REQUEST_BUF_SIZE - stream->index + 1 >= size; } void hs_stream_shift(hs_stream_t* stream) { @@ -751,37 +792,25 @@ void hs_stream_shift(hs_stream_t* stream) { // *** http parser *** -int hs_augment_transition(int to, uint8_t* flags) { - switch (to) { - case C2: - if (*flags & HS_PF_CKEND) { - HTTP_FLAG_CLEAR(*flags, HS_PF_CKEND); - return ST; - } - break; - case BD: - if (*flags & HS_PF_CHUNKED) { - HTTP_FLAG_CLEAR(*flags, HS_PF_CHUNKED); - return CS; - } - break; - case HV: - if (*flags & HS_PF_IN_CONTENT_LEN) { - HTTP_FLAG_CLEAR(*flags, HS_PF_IN_CONTENT_LEN); - return CL; - } else if (*flags & HS_PF_IN_TRANSFER_ENC) { - HTTP_FLAG_CLEAR(*flags, HS_PF_IN_TRANSFER_ENC); - return TE; - } +char const* mstate[] = { + "M_WFK", "M_ANY", "M_MTE", "M_MCL", "M_CLV", "M_MCK", "M_SML", "M_CHK", "M_BIG", + "M_ZER", "M_CSZ", "M_CBD", "M_STR", "M_BDY", "M_LST", "M_END", "M_EMP", "M_ERR" +}; + +void hs_trigger_meta(http_parser_t* parser, int event) { + int to = hs_meta_transitions[parser->meta * HS_META_TYPE_LEN + event]; + if (to == M_END) { + printf("end\n"); } - return to; + printf("-> %s\n", mstate[to]); + parser->meta = to; } -#define HS_MATCH(str, flag) \ +#define HS_MATCH(str, meta) \ in_bounds = parser->match_index < (int)sizeof(str) - 1; \ m = in_bounds ? str[parser->match_index] : m; \ low = c >= 'A' && c <= 'Z' ? c + 32 : c; \ - if (low != m) HTTP_FLAG_CLEAR(parser->flags, flag); \ + if (low != m) hs_trigger_meta(parser, meta); http_token_t hs_transition_action( http_parser_t* parser, @@ -791,80 +820,69 @@ http_token_t hs_transition_action( int8_t to ) { http_token_t emitted = {0}; - switch (from) { - case RN: - case HE: - HTTP_FLAG_SET(parser->flags, HS_PF_IN_CONTENT_LEN); - HTTP_FLAG_SET(parser->flags, HS_PF_IN_TRANSFER_ENC); - break; - case HN: - hs_stream_anchor(stream); - break; - case C1: - HTTP_FLAG_SET(parser->flags, HS_PF_CKEND); - break; - } - if (from == HN && to == CS) { - HTTP_FLAG_SET(parser->flags, HS_PF_CKEND); - emitted.type = HS_TOK_BODY_STREAM; - } + if (from == HN) hs_stream_anchor(stream); if (from != to) { int type = hs_token_start_states[to]; + // if body is streaming token type is CHUNK_BODY instead of BODY + type = type == HS_TOK_BODY && parser->meta == M_STR ? HS_TOK_CHUNK_BODY : type; if (type != HS_TOK_NONE) hs_stream_begin_token(stream, type); - switch (to) { - case HK: - parser->header_count++; - if (parser->header_count > HTTP_MAX_HEADER_COUNT) { - emitted.type = HS_TOK_ERROR; - } - break; - case TE: - HTTP_FLAG_SET(parser->flags, HS_PF_CHUNKED); - break; - case BD: - if (!hs_stream_can_contain(stream, parser->content_length)) { - parser->inject_token = HS_TOK_BODY_STREAM; - } - break; + + if (from == CS) { + hs_trigger_meta(parser, HS_META_END_CHK_SIZE); + } + if (to == HK) { + parser->header_count++; + if (parser->header_count > HTTP_MAX_HEADER_COUNT) { + emitted.type = HS_TOK_ERROR; + } + } else if (to == HS) { + hs_trigger_meta(parser, HS_META_END_KEY); + emitted = hs_stream_emit(stream); } parser->match_index = 0; } char low, m = '\0'; int in_bounds = 0; int body_left = 0; - // generic to switch (to) { - case ST: - emitted.type = HS_TOK_REQ_END; - break; case MS: case TS: case RR: - case HS: - case HR: emitted = hs_stream_emit(stream); break; - case HN: - if (parser->content_length == 0 && !HTTP_FLAG_CHECK(parser->flags, HS_PF_CHUNKED)) { - emitted.type = HS_TOK_BODY; - parser->inject_token = HS_TOK_REQ_END; - } + case HR: + hs_trigger_meta(parser, HS_META_END_VALUE); + emitted = hs_stream_emit(stream); break; case HK: - HS_MATCH("transfer-encoding", HS_PF_IN_TRANSFER_ENC) - HS_MATCH("content-length", HS_PF_IN_CONTENT_LEN) + HS_MATCH("transfer-encoding", HS_META_NOT_TRANSFER_ENC) + HS_MATCH("content-length", HS_META_NOT_CONTENT_LEN) parser->match_index++; break; - case TE: - HS_MATCH("chunked", HS_PF_CHUNKED) - parser->match_index++; + case HV: + if (parser->meta == M_MCK) { + HS_MATCH("chunked", HS_META_NOT_CHUNKED) + parser->match_index++; + } else if (parser->meta == M_CLV) { + parser->content_length *= 10; + parser->content_length += c - '0'; + } break; - case CL: - parser->content_length *= 10; - parser->content_length += c - '0'; + case HN: + if (parser->meta == M_SML && !hs_stream_can_contain(stream, parser->content_length)) { + hs_trigger_meta(parser, HS_META_LARGE_BODY); + } + if (parser->meta == M_BIG || parser->meta == M_CHK) { + emitted.type = HS_TOK_BODY_STREAM; + } + if (parser->meta == M_CHK) parser->state = CS; + hs_trigger_meta(parser, HS_META_END_HEADERS); + if (parser->meta == M_END) { + emitted.type = HS_TOK_BODY; + } break; case CS: - if (c != '0') HTTP_FLAG_CLEAR(parser->flags, HS_PF_CKEND); + if (c != '0') hs_trigger_meta(parser, HS_META_NON_ZERO); if (c >= 'A' && c <= 'F') { parser->content_length *= 0x10; parser->content_length += c - 55; @@ -877,32 +895,24 @@ http_token_t hs_transition_action( } break; case CB: - if (hs_stream_jump(stream, parser->content_length)) { - emitted = hs_stream_emit(stream); - parser->state = CD; - parser->content_length = 0; - } else { - int bytes = hs_stream_jumpall(stream); - parser->content_length -= bytes; - } - break; case BD: body_left = parser->content_length - parser->body_consumed; if (hs_stream_jump(stream, body_left)) { emitted = hs_stream_emit(stream); - parser->state = ST; - parser->inject_token = HS_TOK_REQ_END; + hs_trigger_meta(parser, HS_META_END_BODY); + if (to == CB) parser->state = CD; parser->content_length = 0; parser->body_consumed = 0; - } else if (hs_stream_is_maxcap(stream)) { - hs_stream_begin_token(stream, HS_TOK_CHUNK_BODY); - parser->body_consumed += hs_stream_jumpall(stream); - emitted = hs_stream_emit(stream); - hs_stream_shift(stream); } else { parser->body_consumed += hs_stream_jumpall(stream); + if (parser->meta == M_STR) { + emitted = hs_stream_emit(stream); + } } break; + case C2: + hs_trigger_meta(parser, HS_META_END_CHUNK); + break; case BR: emitted.type = HS_TOK_ERROR; break; @@ -910,30 +920,38 @@ http_token_t hs_transition_action( return emitted; } -int hs_inject_token(http_parser_t* parser, http_token_t* token) { - token->type = parser->inject_token; - parser->inject_token = 0; - return token->type; -} +char const* pstate[] = { + "ST", "MT", "MS", "TR", "TS", "VN", "RR", "RN", "HK", "HS", "HV", "HR", "HE", + "ER", "HN", "BD", "CS", "CB", "CE", "CR", "CN", "CD", "C1", "C2", "BR", "HS_STATE_LEN" +}; + +#define HS_END_REQUEST_IF(condition) \ + if (condition) { \ + token.type = HS_TOK_REQ_END; \ + parser->state = 0; \ + parser->meta = 0; \ + parser->header_count = 0; \ + return token; \ + } http_token_t http_parse(http_parser_t* parser, hs_stream_t* stream) { char c = 0; http_token_t token = {0}; - if (hs_inject_token(parser, &token)) return token; + HS_END_REQUEST_IF(parser->meta == M_END) while (hs_stream_next(stream, &c)) { int type = c < 0 ? HS_ETC : hs_ctype[(int)c]; int to = hs_transitions[parser->state * HS_CHAR_TYPE_LEN + type]; - to = hs_augment_transition(to, &parser->flags); + printf("to: %s\n", pstate[to]); int from = parser->state; parser->state = to; http_token_t emitted = hs_transition_action(parser, stream, c, from, to); hs_stream_consume(stream); if (emitted.type != HS_TOK_NONE) return emitted; } - if (parser->state >= CS && parser->state <= C2) hs_stream_shift(stream); - hs_inject_token(parser, &token); + HS_END_REQUEST_IF(parser->meta == M_END) return token; } + // *** http server *** void http_token_dyn_push(http_token_dyn_t* dyn, http_token_t a) { @@ -1043,7 +1061,7 @@ int hs_has_buffered_tokens(http_request_t* request) { char const* tokens[] = { "HS_TOK_NONE", "HS_TOK_METHOD", "HS_TOK_TARGET", "HS_TOK_VERSION", "HS_TOK_HEADER_KEY", "HS_TOK_HEADER_VAL", "HS_TOK_CHUNK_BODY", "HS_TOK_BODY", - "HS_TOK_BODY_STREAM", "HS_TOK_REQ_END", "HS_TOK_EOF" + "HS_TOK_BODY_STREAM", "HS_TOK_REQ_END", "HS_TOK_EOF", "HS_TOK_ERROR" }; void hs_read_and_parse_tokens(http_request_t* request) { @@ -1055,7 +1073,7 @@ void hs_read_and_parse_tokens(http_request_t* request) { //printf("rc: %d\n", rc); do { token = http_parse(&request->parser, &request->stream); - //printf("EMITTED: %s, len: %d UNREAD: %d\n", tokens[token.type], token.len, request->stream.length - request->stream.index); + printf("EMITTED: %s, len: %d UNREAD: %d\n", tokens[token.type], token.len, request->stream.length - request->stream.index); if (token.type != HS_TOK_NONE) { http_token_dyn_push(&request->tokens, token); } @@ -1068,7 +1086,7 @@ void hs_process_tokens(http_request_t* request) { while (request->i < request->tokens.size) { http_token_t token = request->tokens.buf[request->i]; request->i++; - //printf("PROCESSING: %s, %.*s, UNREAD: %d\n", tokens[token.type], token.len, request->stream.buf + token.index, request->stream.length - request->stream.index); + printf("PROCESSING: %s, UNREAD: %d\n", tokens[token.type], request->stream.length - request->stream.index); if (token.type == HS_TOK_EOF) { return hs_end_session(request); } else if (token.type == HS_TOK_ERROR) { From e94f46b5fc1f93d916d21e2173f0095897600df8 Mon Sep 17 00:00:00 2001 From: Jeremy Williams Date: Wed, 5 Feb 2020 13:01:30 -0500 Subject: [PATCH 04/20] Continue refactor --- httpserver.h | 102 +++++++++++++++++++++++++++------------------------ test/main.c | 7 ++++ test/run | 29 ++++++++++++--- 3 files changed, 85 insertions(+), 53 deletions(-) diff --git a/httpserver.h b/httpserver.h index 5d8f911..7984dc2 100644 --- a/httpserver.h +++ b/httpserver.h @@ -476,19 +476,17 @@ enum hs_char_type { enum hs_meta_state { M_WFK, M_ANY, M_MTE, M_MCL, M_CLV, M_MCK, M_SML, M_CHK, M_BIG, M_ZER, M_CSZ, - M_CBD, M_STR, M_BDY, M_LST, M_END, M_ERR + M_CBD, M_LST, M_STR, M_SEN, M_BDY, M_END, M_ERR }; enum hs_meta_type { HS_META_NOT_CONTENT_LEN, HS_META_NOT_TRANSFER_ENC, HS_META_END_KEY, - HS_META_END_VALUE, HS_META_END_HEADERS, HS_META_TYPE_LEN + HS_META_END_VALUE, HS_META_END_HEADERS, HS_META_LARGE_BODY, + HS_META_TYPE_LEN }; #define HS_META_NOT_CHUNKED 0 -#define HS_META_LARGE_BODY 0 -#define HS_META_END_BODY 0 #define HS_META_NON_ZERO 0 -#define HS_META_FINISH 0 #define HS_META_END_CHK_SIZE 1 #define HS_META_END_CHUNK 2 #define HS_META_NEXT 0 @@ -631,28 +629,37 @@ static int const hs_transitions[] = { }; static int const hs_meta_transitions[] = { -// not cl not te endkey endval end h -/* WFK wait */ M_WFK, M_WFK, M_WFK, M_ANY, M_END, -/* ANY matchkey */ M_MTE, M_MCL, M_WFK, M_ERR, M_END, -/* MTE matchte */ M_MTE, M_WFK, M_MCK, M_ERR, M_ERR, -/* MCL matchcl */ M_WFK, M_MCL, M_CLV, M_ERR, M_ERR, -/* CLV clvalue */ M_ERR, M_ERR, M_ERR, M_SML, M_ERR, // no chk -/* MCK matchchk */ M_WFK, M_ERR, M_ERR, M_CHK, M_ERR, -// toobig -/* SML smallbdy */ M_BIG, M_SML, M_SML, M_SML, M_BDY, -/* CHK chunkbdy */ M_CHK, M_CHK, M_CHK, M_CHK, M_ZER, -/* BIG bigbody */ M_BIG, M_BIG, M_BIG, M_BIG, M_STR, -// nozer endsz -/* ZER zerochk */ M_CSZ, M_LST, M_ERR, M_ERR, M_ERR, -/* CSZ chksize */ M_CSZ, M_CBD, M_ERR, M_ERR, M_ERR, -// endbdy endchk -/* CBD readchk */ M_CBD, M_CBD, M_ZER, M_ERR, M_ERR, -/* STR readstr */ M_END, M_ERR, M_ERR, M_ERR, M_ERR, -/* BDY readbody */ M_END, M_ERR, M_ERR, M_ERR, M_ERR, -/* LST lastchk */ M_LST, M_END, M_END, M_ERR, M_ERR, -// fin -/* END reqend */ M_END, M_WFK, M_ERR, M_ERR, M_ERR +// not cl not te endkey endval end h toobig +/* WFK wait */ M_WFK, M_WFK, M_WFK, M_ANY, M_END, M_ERR, +/* ANY matchkey */ M_MTE, M_MCL, M_WFK, M_ERR, M_END, M_ERR, +/* MTE matchte */ M_MTE, M_WFK, M_MCK, M_ERR, M_ERR, M_ERR, +/* MCL matchcl */ M_WFK, M_MCL, M_CLV, M_ERR, M_ERR, M_ERR, +/* CLV clvalue */ M_ERR, M_ERR, M_ERR, M_SML, M_ERR, M_ERR, +/* MCK matchchk */ M_WFK, M_ERR, M_ERR, M_CHK, M_ERR, M_ERR, +/* SML smallbdy */ M_SML, M_SML, M_SML, M_SML, M_BDY, M_BIG, +/* CHK chunkbdy */ M_CHK, M_CHK, M_CHK, M_CHK, M_ZER, M_ERR, +/* BIG bigbody */ M_BIG, M_BIG, M_BIG, M_BIG, M_STR, M_ERR, + +// *** chunked body *** + +// nonzer endsz endchk +/* ZER zerochk */ M_CSZ, M_LST, M_ERR, M_ERR, M_ERR, M_ERR, +/* CSZ chksize */ M_CSZ, M_CBD, M_ERR, M_ERR, M_ERR, M_ERR, +/* CBD readchk */ M_CBD, M_CBD, M_ZER, M_ERR, M_ERR, M_ERR, +/* LST lastchk */ M_LST, M_END, M_END, M_ERR, M_ERR, M_ERR, + +// *** streamed body *** + +// next +/* STR readstr */ M_SEN, M_ERR, M_ERR, M_ERR, M_ERR, M_ERR, +/* SEN strend */ M_END, M_ERR, M_ERR, M_ERR, M_ERR, M_ERR, + +// *** small body *** + +// next +/* BDY readbody */ M_END, M_ERR, M_ERR, M_ERR, M_ERR, M_ERR, +/* END reqend */ M_WFK, M_ERR, M_ERR, M_ERR, M_ERR, M_ERR }; static int const hs_ctype[] = { @@ -794,14 +801,11 @@ void hs_stream_shift(hs_stream_t* stream) { char const* mstate[] = { "M_WFK", "M_ANY", "M_MTE", "M_MCL", "M_CLV", "M_MCK", "M_SML", "M_CHK", "M_BIG", - "M_ZER", "M_CSZ", "M_CBD", "M_STR", "M_BDY", "M_LST", "M_END", "M_EMP", "M_ERR" + "M_ZER", "M_CSZ", "M_CBD", "M_STR", "M_BDY", "M_LST", "M_SEN", "M_END", "M_ERR" }; void hs_trigger_meta(http_parser_t* parser, int event) { int to = hs_meta_transitions[parser->meta * HS_META_TYPE_LEN + event]; - if (to == M_END) { - printf("end\n"); - } printf("-> %s\n", mstate[to]); parser->meta = to; } @@ -823,13 +827,8 @@ http_token_t hs_transition_action( if (from == HN) hs_stream_anchor(stream); if (from != to) { int type = hs_token_start_states[to]; - // if body is streaming token type is CHUNK_BODY instead of BODY - type = type == HS_TOK_BODY && parser->meta == M_STR ? HS_TOK_CHUNK_BODY : type; if (type != HS_TOK_NONE) hs_stream_begin_token(stream, type); - - if (from == CS) { - hs_trigger_meta(parser, HS_META_END_CHK_SIZE); - } + if (from == CS) hs_trigger_meta(parser, HS_META_END_CHK_SIZE); if (to == HK) { parser->header_count++; if (parser->header_count > HTTP_MAX_HEADER_COUNT) { @@ -896,10 +895,11 @@ http_token_t hs_transition_action( break; case CB: case BD: + if (parser->meta == M_STR) hs_stream_begin_token(stream, HS_TOK_CHUNK_BODY); body_left = parser->content_length - parser->body_consumed; if (hs_stream_jump(stream, body_left)) { emitted = hs_stream_emit(stream); - hs_trigger_meta(parser, HS_META_END_BODY); + hs_trigger_meta(parser, HS_META_NEXT); if (to == CB) parser->state = CD; parser->content_length = 0; parser->body_consumed = 0; @@ -907,6 +907,7 @@ http_token_t hs_transition_action( parser->body_consumed += hs_stream_jumpall(stream); if (parser->meta == M_STR) { emitted = hs_stream_emit(stream); + hs_stream_shift(stream); } } break; @@ -925,19 +926,25 @@ char const* pstate[] = { "ER", "HN", "BD", "CS", "CB", "CE", "CR", "CN", "CD", "C1", "C2", "BR", "HS_STATE_LEN" }; -#define HS_END_REQUEST_IF(condition) \ - if (condition) { \ - token.type = HS_TOK_REQ_END; \ - parser->state = 0; \ - parser->meta = 0; \ - parser->header_count = 0; \ - return token; \ +http_token_t hs_meta_emit(http_parser_t* parser) { + http_token_t token = {0}; + switch (parser->meta) { + case M_SEN: + token.type = HS_TOK_CHUNK_BODY; + hs_trigger_meta(parser, HS_META_NEXT); + break; + case M_END: + token.type = HS_TOK_REQ_END; + memset(parser, 0, sizeof(http_parser_t)); + break; } + return token; +} http_token_t http_parse(http_parser_t* parser, hs_stream_t* stream) { char c = 0; - http_token_t token = {0}; - HS_END_REQUEST_IF(parser->meta == M_END) + http_token_t token = hs_meta_emit(parser); + if (token.type != HS_TOK_NONE) return token; while (hs_stream_next(stream, &c)) { int type = c < 0 ? HS_ETC : hs_ctype[(int)c]; int to = hs_transitions[parser->state * HS_CHAR_TYPE_LEN + type]; @@ -948,7 +955,8 @@ http_token_t http_parse(http_parser_t* parser, hs_stream_t* stream) { hs_stream_consume(stream); if (emitted.type != HS_TOK_NONE) return emitted; } - HS_END_REQUEST_IF(parser->meta == M_END) + if (parser->state >= CS && parser->state <= C2) hs_stream_shift(stream); + token = hs_meta_emit(parser); return token; } diff --git a/test/main.c b/test/main.c index 4651b0b..fe4776f 100644 --- a/test/main.c +++ b/test/main.c @@ -78,6 +78,13 @@ void handle_request(struct http_request_s* request) { http_request_set_userdata(request, chunk_buffer); http_request_read_chunk(request, chunk_req_cb); return; + } else if (request_target_is(request, "/large")) { + chunk_buf_t* chunk_buffer = (chunk_buf_t*)calloc(1, sizeof(chunk_buf_t)); + chunk_buffer->buf = (char*)malloc(25165824); + chunk_buffer->response = response; + http_request_set_userdata(request, chunk_buffer); + http_request_read_chunk(request, chunk_req_cb); + return; } else if (request_target_is(request, "/headers")) { int iter = 0, i = 0; http_string_t key, val; diff --git a/test/run b/test/run index d2e124c..1fcefb9 100755 --- a/test/run +++ b/test/run @@ -1,52 +1,69 @@ #!/usr/bin/env sh sleep 1 + echo "Small Response Body:" curl http://localhost:8080/ + echo "\n\nEmpty Response:" curl http://localhost:8080/empty -echo "\n\nBody Too Large:" -curl -H'Content-Length: 4000000000' http://localhost:8080/ + echo "\n\nEcho Body:" curl -XPOST http://localhost:8080/echo -d'Echo test' + echo "\n\nGet Header:" curl http://localhost:8080/host + +echo "\n\nRequest Body larger than max in mem size:" +dd if=/dev/urandom of=test.dat bs=25165824 count=1 2> /dev/null +curl -H'Expect:' --data-binary @test.dat -o r1.dat http://localhost:8080/large +diff r1.dat test.dat + echo "\n\nChunked Response:" curl http://localhost:8080/chunked + echo "\n\nChunked Response keep-alive:" curl http://localhost:8080/chunked http://localhost:8080/chunked + echo "\n\nChunked Response close:" curl -H'Connection: close' http://localhost:8080/chunked http://localhost:8080/chunked + echo "\n\nChunked Request keep-alive: (expect empty)" dd if=/dev/urandom of=test.dat bs=262144 count=1 2> /dev/null curl -H'Expect:' -H'Transfer-Encoding: chunked' -XPOST --data-binary @test.dat -o r1.dat http://localhost:8080/chunked-req -o r2.dat http://localhost:8080/chunked-req diff r1.dat test.dat diff r2.dat test.dat + echo "\n\nChunked Request close: (expect empty)" curl -H'Expect:' -H'Transfer-Encoding: chunked' -XPOST --data-binary @test.dat -o r3.dat http://localhost:8080/chunked-req -o r4.dat http://localhost:8080/chunked-req diff r3.dat test.dat diff r4.dat test.dat + echo "\n\nPoll Server:" curl http://localhost:8081/ & sleep 1 curl http://localhost:8080/poll sleep 1 curl http://localhost:8080/poll + echo "\n\nIterate Headers:" curl -H'User-Agent: test-ua' -H'Foo-Header: foo-bar' http://localhost:8080/headers + echo "\nStress Test keep-alive:" ab -k -c 200 -n 100000 http://127.0.0.1:8080/ > keep-alive.bench cat keep-alive.bench | grep Complete -cat large-file.bench | grep Non-2xx +cat keep-alive.bench | grep Non-2xx cat keep-alive.bench 1>&2 + echo "\nStress Test close:" ab -c 20 -n 1000 http://127.0.0.1:8080/ > connection-close.bench cat connection-close.bench | grep Complete -cat large-file.bench | grep Non-2xx +cat connection-close.bench | grep Non-2xx cat connection-close.bench 1>&2 + echo "\nStress Test large body:" -dd if=/dev/urandom of=test.dat bs=8388608 count=1 2> /dev/null -ab -c 2 -n 10 -p test.dat http://127.0.0.1:8080/echo > large-file.bench +dd if=/dev/urandom of=test.dat bs=7388608 count=1 2> /dev/null +ab -c 1 -n 1 -p test.dat http://127.0.0.1:8080/echo > large-file.bench cat large-file.bench | grep Complete cat large-file.bench | grep Non-2xx cat large-file.bench 1>&2 From 0f10992c82185ec2539e3246f2e0e86bd0806020 Mon Sep 17 00:00:00 2001 From: Jeremy Williams Date: Wed, 5 Feb 2020 13:18:38 -0500 Subject: [PATCH 05/20] Fix bug with refactor --- httpserver.h | 10 +++++----- test/run | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/httpserver.h b/httpserver.h index 7984dc2..a35f26c 100644 --- a/httpserver.h +++ b/httpserver.h @@ -806,7 +806,7 @@ char const* mstate[] = { void hs_trigger_meta(http_parser_t* parser, int event) { int to = hs_meta_transitions[parser->meta * HS_META_TYPE_LEN + event]; - printf("-> %s\n", mstate[to]); + //printf("-> %s\n", mstate[to]); parser->meta = to; } @@ -846,9 +846,9 @@ http_token_t hs_transition_action( switch (to) { case MS: case TS: - case RR: emitted = hs_stream_emit(stream); break; + case RR: case HR: hs_trigger_meta(parser, HS_META_END_VALUE); emitted = hs_stream_emit(stream); @@ -948,7 +948,7 @@ http_token_t http_parse(http_parser_t* parser, hs_stream_t* stream) { while (hs_stream_next(stream, &c)) { int type = c < 0 ? HS_ETC : hs_ctype[(int)c]; int to = hs_transitions[parser->state * HS_CHAR_TYPE_LEN + type]; - printf("to: %s\n", pstate[to]); + //printf("to: %s\n", pstate[to]); int from = parser->state; parser->state = to; http_token_t emitted = hs_transition_action(parser, stream, c, from, to); @@ -1081,7 +1081,7 @@ void hs_read_and_parse_tokens(http_request_t* request) { //printf("rc: %d\n", rc); do { token = http_parse(&request->parser, &request->stream); - printf("EMITTED: %s, len: %d UNREAD: %d\n", tokens[token.type], token.len, request->stream.length - request->stream.index); + //printf("EMITTED: %s, len: %d UNREAD: %d\n", tokens[token.type], token.len, request->stream.length - request->stream.index); if (token.type != HS_TOK_NONE) { http_token_dyn_push(&request->tokens, token); } @@ -1094,7 +1094,7 @@ void hs_process_tokens(http_request_t* request) { while (request->i < request->tokens.size) { http_token_t token = request->tokens.buf[request->i]; request->i++; - printf("PROCESSING: %s, UNREAD: %d\n", tokens[token.type], request->stream.length - request->stream.index); + //printf("PROCESSING: %s, UNREAD: %d\n", tokens[token.type], request->stream.length - request->stream.index); if (token.type == HS_TOK_EOF) { return hs_end_session(request); } else if (token.type == HS_TOK_ERROR) { diff --git a/test/run b/test/run index 1fcefb9..8eafe13 100755 --- a/test/run +++ b/test/run @@ -63,7 +63,7 @@ cat connection-close.bench 1>&2 echo "\nStress Test large body:" dd if=/dev/urandom of=test.dat bs=7388608 count=1 2> /dev/null -ab -c 1 -n 1 -p test.dat http://127.0.0.1:8080/echo > large-file.bench +ab -c 2 -n 4 -p test.dat http://127.0.0.1:8080/echo > large-file.bench cat large-file.bench | grep Complete cat large-file.bench | grep Non-2xx cat large-file.bench 1>&2 From 3603faf19fbdd3929f36fd4d3b17f95febeb39ff Mon Sep 17 00:00:00 2001 From: Jeremy Williams Date: Wed, 5 Feb 2020 13:23:28 -0500 Subject: [PATCH 06/20] Fix test suite --- test/results.txt | 6 +++--- test/run | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/results.txt b/test/results.txt index d8e13bb..19d4e70 100644 --- a/test/results.txt +++ b/test/results.txt @@ -4,15 +4,15 @@ Hello, World! Empty Response: -Body Too Large: -Payload Too Large - Echo Body: Echo test Get Header: localhost:8080 +Request Body larger than max in mem size: + + Chunked Response: Hello, World!Hello, World!Hello, World! diff --git a/test/run b/test/run index 8eafe13..f6ed748 100755 --- a/test/run +++ b/test/run @@ -63,7 +63,7 @@ cat connection-close.bench 1>&2 echo "\nStress Test large body:" dd if=/dev/urandom of=test.dat bs=7388608 count=1 2> /dev/null -ab -c 2 -n 4 -p test.dat http://127.0.0.1:8080/echo > large-file.bench +ab -c 2 -n 10 -p test.dat http://127.0.0.1:8080/echo > large-file.bench cat large-file.bench | grep Complete cat large-file.bench | grep Non-2xx cat large-file.bench 1>&2 From 9658bb78981f0465b425c77371daa8f2fca3ddd4 Mon Sep 17 00:00:00 2001 From: Jeremy Williams Date: Wed, 5 Feb 2020 13:55:08 -0500 Subject: [PATCH 07/20] Fix cpp errors --- httpserver.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/httpserver.h b/httpserver.h index a35f26c..74ce0a5 100644 --- a/httpserver.h +++ b/httpserver.h @@ -775,7 +775,7 @@ void hs_stream_anchor(hs_stream_t* stream) { http_token_t hs_stream_emit(hs_stream_t* stream) { http_token_t token = stream->token; - http_token_t none = {0}; + http_token_t none = {0, 0, 0}; stream->token = none; return token; } @@ -823,7 +823,7 @@ http_token_t hs_transition_action( int8_t from, int8_t to ) { - http_token_t emitted = {0}; + http_token_t emitted = {0, 0, 0}; if (from == HN) hs_stream_anchor(stream); if (from != to) { int type = hs_token_start_states[to]; @@ -927,7 +927,7 @@ char const* pstate[] = { }; http_token_t hs_meta_emit(http_parser_t* parser) { - http_token_t token = {0}; + http_token_t token = {0, 0, 0}; switch (parser->meta) { case M_SEN: token.type = HS_TOK_CHUNK_BODY; @@ -1075,7 +1075,7 @@ char const* tokens[] = { void hs_read_and_parse_tokens(http_request_t* request) { //printf("read state set\n"); request->state = HTTP_SESSION_READ; - http_token_t token = {0}; + http_token_t token = {0, 0, 0}; hs_reset_timeout(request, HTTP_REQUEST_TIMEOUT); int rc = hs_stream_read_socket(&request->stream, request->socket, &request->server->memused); //printf("rc: %d\n", rc); @@ -1227,7 +1227,7 @@ int http_server_loop(http_server_t* server) { // *** http request *** http_string_t http_get_token_string(http_request_t* request, int token_type) { - http_string_t str = {0}; + http_string_t str = {0, 0}; if (request->tokens.buf == NULL) return str; for (int i = 0; i < request->tokens.size; i++) { http_token_t token = request->tokens.buf[i]; @@ -1589,7 +1589,7 @@ void hs_delete_events(http_request_t* request) { int http_server_poll(http_server_t* serv) { struct kevent ev; - struct timespec ts = {0}; + struct timespec ts = {0, 0}; int nev = kevent(serv->loop, NULL, 0, &ev, 1, &ts); if (nev <= 0) return nev; ev_cb_t* ev_cb = (ev_cb_t*)ev.udata; From 2adf98e728a49371ba34f08faf364a575f92b06e Mon Sep 17 00:00:00 2001 From: Jeremy Williams Date: Wed, 5 Feb 2020 14:57:58 -0500 Subject: [PATCH 08/20] Fix issue reading into buffer with no space --- httpserver.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/httpserver.h b/httpserver.h index 74ce0a5..635962c 100644 --- a/httpserver.h +++ b/httpserver.h @@ -726,8 +726,8 @@ int hs_stream_read_socket(hs_stream_t* stream, int socket, int64_t* memused) { stream->buf = (char*)realloc(stream->buf, stream->capacity); assert(stream->buf != NULL); } - //printf("bytes: %d\n", bytes); - } while (bytes > 0); + //printf("bytes: %d, total: %ld\n", bytes, stream->total_bytes); + } while (bytes > 0 && stream->capacity < HTTP_MAX_REQUEST_BUF_SIZE); return bytes == 0 ? 0 : 1; } From 6c830e6d463946f582f83486a36c515ce91b9836 Mon Sep 17 00:00:00 2001 From: Jeremy Williams Date: Wed, 5 Feb 2020 16:42:52 -0500 Subject: [PATCH 09/20] Remove debug code --- httpserver.h | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/httpserver.h b/httpserver.h index 635962c..ec4438b 100644 --- a/httpserver.h +++ b/httpserver.h @@ -726,7 +726,6 @@ int hs_stream_read_socket(hs_stream_t* stream, int socket, int64_t* memused) { stream->buf = (char*)realloc(stream->buf, stream->capacity); assert(stream->buf != NULL); } - //printf("bytes: %d, total: %ld\n", bytes, stream->total_bytes); } while (bytes > 0 && stream->capacity < HTTP_MAX_REQUEST_BUF_SIZE); return bytes == 0 ? 0 : 1; } @@ -799,14 +798,8 @@ void hs_stream_shift(hs_stream_t* stream) { // *** http parser *** -char const* mstate[] = { - "M_WFK", "M_ANY", "M_MTE", "M_MCL", "M_CLV", "M_MCK", "M_SML", "M_CHK", "M_BIG", - "M_ZER", "M_CSZ", "M_CBD", "M_STR", "M_BDY", "M_LST", "M_SEN", "M_END", "M_ERR" -}; - void hs_trigger_meta(http_parser_t* parser, int event) { int to = hs_meta_transitions[parser->meta * HS_META_TYPE_LEN + event]; - //printf("-> %s\n", mstate[to]); parser->meta = to; } @@ -921,11 +914,6 @@ http_token_t hs_transition_action( return emitted; } -char const* pstate[] = { - "ST", "MT", "MS", "TR", "TS", "VN", "RR", "RN", "HK", "HS", "HV", "HR", "HE", - "ER", "HN", "BD", "CS", "CB", "CE", "CR", "CN", "CD", "C1", "C2", "BR", "HS_STATE_LEN" -}; - http_token_t hs_meta_emit(http_parser_t* parser) { http_token_t token = {0, 0, 0}; switch (parser->meta) { @@ -948,7 +936,6 @@ http_token_t http_parse(http_parser_t* parser, hs_stream_t* stream) { while (hs_stream_next(stream, &c)) { int type = c < 0 ? HS_ETC : hs_ctype[(int)c]; int to = hs_transitions[parser->state * HS_CHAR_TYPE_LEN + type]; - //printf("to: %s\n", pstate[to]); int from = parser->state; parser->state = to; http_token_t emitted = hs_transition_action(parser, stream, c, from, to); @@ -1033,7 +1020,6 @@ void hs_reset_timeout(http_request_t* request, int time) { } void hs_write_response(http_request_t* request) { - //printf("writing!\n"); if (!hs_write_client_socket(request)) { return hs_end_session(request); } if (request->stream.total_bytes != request->stream.length) { // All bytes of the body were not written and we need to wait until the @@ -1066,22 +1052,13 @@ int hs_has_buffered_tokens(http_request_t* request) { return request->i < request->tokens.size; } -char const* tokens[] = { - "HS_TOK_NONE", "HS_TOK_METHOD", "HS_TOK_TARGET", "HS_TOK_VERSION", - "HS_TOK_HEADER_KEY", "HS_TOK_HEADER_VAL", "HS_TOK_CHUNK_BODY", "HS_TOK_BODY", - "HS_TOK_BODY_STREAM", "HS_TOK_REQ_END", "HS_TOK_EOF", "HS_TOK_ERROR" -}; - void hs_read_and_parse_tokens(http_request_t* request) { - //printf("read state set\n"); request->state = HTTP_SESSION_READ; http_token_t token = {0, 0, 0}; hs_reset_timeout(request, HTTP_REQUEST_TIMEOUT); int rc = hs_stream_read_socket(&request->stream, request->socket, &request->server->memused); - //printf("rc: %d\n", rc); do { token = http_parse(&request->parser, &request->stream); - //printf("EMITTED: %s, len: %d UNREAD: %d\n", tokens[token.type], token.len, request->stream.length - request->stream.index); if (token.type != HS_TOK_NONE) { http_token_dyn_push(&request->tokens, token); } @@ -1094,24 +1071,20 @@ void hs_process_tokens(http_request_t* request) { while (request->i < request->tokens.size) { http_token_t token = request->tokens.buf[request->i]; request->i++; - //printf("PROCESSING: %s, UNREAD: %d\n", tokens[token.type], request->stream.length - request->stream.index); if (token.type == HS_TOK_EOF) { return hs_end_session(request); } else if (token.type == HS_TOK_ERROR) { hs_error_response(request, 400, "Bad Request"); } else if (token.type == HS_TOK_BODY || token.type == HS_TOK_BODY_STREAM) { - //printf("nop state set\n"); request->state = HTTP_SESSION_NOP; request->server->request_handler(request); break; } else if (token.type == HS_TOK_CHUNK_BODY) { - //printf("nop state set\n"); request->state = HTTP_SESSION_NOP; request->chunk_cb(request); break; } else if (token.type == HS_TOK_REQ_END) { if (HTTP_FLAG_CHECK(request->flags, HTTP_KEEP_ALIVE)) { - //printf("keep-alive!\n"); request->state = HTTP_SESSION_INIT; hs_free_buffer(request); hs_reset_timeout(request, HTTP_KEEP_ALIVE_TIMEOUT); @@ -1127,7 +1100,6 @@ void http_request_read_chunk( struct http_request_s* request, void (*chunk_cb)(struct http_request_s*) ) { - //printf("reading chunk!\n"); request->chunk_cb = chunk_cb; if (hs_has_buffered_tokens(request)) { hs_process_tokens(request); @@ -1140,7 +1112,6 @@ void http_request_read_chunk( // This is the heart of the request logic. This is the state machine that // controls what happens when an IO event is received. void http_session(http_request_t* request) { - //printf("io clabback!\n"); switch (request->state) { case HTTP_SESSION_INIT: hs_init_session(request); @@ -1157,7 +1128,6 @@ void http_session(http_request_t* request) { hs_write_response(request); break; case HTTP_SESSION_NOP: - //printf("NOP!\n"); break; } } From a1cf05e4f10021dd8342e818143adef9270e6b52 Mon Sep 17 00:00:00 2001 From: Jeremy Williams Date: Thu, 6 Feb 2020 11:28:34 -0500 Subject: [PATCH 10/20] Limit size of tokens when parsing --- httpserver.h | 57 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/httpserver.h b/httpserver.h index ec4438b..50869df 100644 --- a/httpserver.h +++ b/httpserver.h @@ -24,7 +24,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* httpserver.h (0.6.0) +* httpserver.h (0.7.0) * * Description: * @@ -54,7 +54,7 @@ * capacity is reached but it certain environments it may be optimal to * change this value. * -* HTTP_RESPONSE_BUF_SIZE - default 512 - Same as above except for the +* HTTP_RESPONSE_BUF_SIZE - default 1024 - Same as above except for the * response buffer. * * HTTP_REQUEST_TIMEOUT - default 20 - The amount of seconds the request will @@ -65,12 +65,6 @@ * HTTP_KEEP_ALIVE_TIMEOUT - default 120 - The amount of seconds to keep a * connection alive a keep-alive request has completed. * -* HTTP_MAX_CONTENT_LENGTH - default 8388608 (8MB) - The max size in bytes -* of the request content length. It should be noted that the request body -* will be fully read into memory so while this could be redefined to a -* value as large as INT_MAX it will allocate a lot of memory. I would -* reccommend using chunked encoding for large requests. -* * HTTP_MAX_TOTAL_EST_MEM_USAGE - default 4294967296 (4GB) - This is the * amount of read/write buffer space that is allowed to be allocated across * all requests before new requests will get 503 responses. @@ -78,6 +72,11 @@ * HTTP_MAX_TOKEN_LENGTH - default 8192 (8KB) - This is the max size of any * non body http tokens. i.e: header names, header values, url length, etc. * +* HTTP_MAX_REQUEST_BUF_SIZE - default 8388608 (8MB) - This is the maximum +* amount of bytes that the request buffer will grow to. If the body of the +* request + headers cannot fit in this size the request body will be +* streamed in. +* * For more details see the documentation of the interface and the example * below. * @@ -147,6 +146,15 @@ int http_server_listen_poll(struct http_server_s* server); // 0. int http_server_poll(struct http_server_s* server); +// Returns 1 if the flag is set and false otherwise. The flags that can be +// queried are listed below +int http_request_has_flag(struct http_request_s* request, int flag); + +// This flag will be set when the request body is chunked or the body is too +// large to fit in memory are once. This means that the http_request_read_chunk +// function must be used to read the body piece by piece. +#define HTTP_FLG_STREAMED 0x1 + // Returns the request method as it was read from the HTTP request line. struct http_string_s http_request_method(struct http_request_s* request); @@ -237,9 +245,12 @@ void http_respond_chunk( // response headers. void http_respond_chunk_end(struct http_request_s* request, struct http_response_s* response); -// If a request has Transfer-Encoding: chunked you cannot read the body in the -// typical way. Instead you need to call this function to read one chunk at a -// time. You pass a callback that will be called when the chunk is ready. When +// If a request has Transfer-Encoding: chunked or the body is too big to fit in +// memory all at once you cannot read the body in the typical way. Instead you +// need to call this function to read one chunk at a time. To check if the +// request requires this type of reading you can call the http_request_has_flag +// function to check if the HTTP_FLG_STREAMED flag is set. To read a streamed body +// you pass a callback that will be called when the chunk is ready. When // the callback is called you can use `http_request_chunk` to get the current // chunk. When done with that chunk call this function again to request the // next chunk. If the chunk has size 0 then the request body has been completely @@ -316,10 +327,9 @@ int main() { // Application configurable #define HTTP_REQUEST_BUF_SIZE 1024 -#define HTTP_RESPONSE_BUF_SIZE 512 +#define HTTP_RESPONSE_BUF_SIZE 1024 #define HTTP_REQUEST_TIMEOUT 20 #define HTTP_KEEP_ALIVE_TIMEOUT 120 -#define HTTP_MAX_CONTENT_LENGTH 8388608 // 8mb #define HTTP_MAX_TOKEN_LENGTH 8192 // 8kb #define HTTP_MAX_TOTAL_EST_MEM_USAGE 4294967296 // 4gb #define HTTP_MAX_REQUEST_BUF_SIZE 8388608 // 8mb @@ -396,7 +406,7 @@ typedef struct { typedef struct { int64_t content_length; int64_t body_consumed; - int32_t match_index; + int16_t match_index; int16_t header_count; int8_t state; int8_t meta; @@ -779,6 +789,10 @@ http_token_t hs_stream_emit(hs_stream_t* stream) { return token; } +http_token_t hs_stream_current_token(hs_stream_t* stream) { + return stream->token; +} + int hs_stream_can_contain(hs_stream_t* stream, int64_t size) { return HTTP_MAX_REQUEST_BUF_SIZE - stream->index + 1 >= size; } @@ -944,6 +958,14 @@ http_token_t http_parse(http_parser_t* parser, hs_stream_t* stream) { } if (parser->state >= CS && parser->state <= C2) hs_stream_shift(stream); token = hs_meta_emit(parser); + http_token_t current = hs_stream_current_token(stream); + if ( + current.type != HS_TOK_CHUNK_BODY && + current.type != HS_TOK_BODY && + current.len > HTTP_MAX_TOKEN_LENGTH + ) { + token.type = HS_TOK_ERROR; + } return token; } @@ -1076,6 +1098,9 @@ void hs_process_tokens(http_request_t* request) { } else if (token.type == HS_TOK_ERROR) { hs_error_response(request, 400, "Bad Request"); } else if (token.type == HS_TOK_BODY || token.type == HS_TOK_BODY_STREAM) { + if (token.type == HS_TOK_BODY_STREAM) { + HTTP_FLAG_SET(request->flags, HTTP_FLG_STREAMED); + } request->state = HTTP_SESSION_NOP; request->server->request_handler(request); break; @@ -1210,6 +1235,10 @@ http_string_t http_get_token_string(http_request_t* request, int token_type) { return str; } +int http_request_has_flag(http_request_t* request, int flag) { + return HTTP_FLAG_CHECK(request->flags, flag); +} + int hs_case_insensitive_cmp(char const * a, char const * b, int len) { for (int i = 0; i < len; i++) { char c1 = a[i] >= 'A' && a[i] <= 'Z' ? a[i] + 32 : a[i]; From e3a39fb08336f2f62211b2cd56dc4e82e5927df0 Mon Sep 17 00:00:00 2001 From: Jeremy Williams Date: Mon, 10 Feb 2020 12:21:33 -0500 Subject: [PATCH 11/20] Fix chunked issue --- httpserver.h | 100 +++++++++++++++++++++++---------------------------- 1 file changed, 44 insertions(+), 56 deletions(-) diff --git a/httpserver.h b/httpserver.h index 50869df..c402754 100644 --- a/httpserver.h +++ b/httpserver.h @@ -424,7 +424,6 @@ typedef struct http_request_s { void* data; hs_stream_t stream; http_parser_t parser; - int i; int state; int socket; int timeout; @@ -706,6 +705,7 @@ static int const hs_token_start_states[] = { // *** input stream *** int hs_stream_read_socket(hs_stream_t* stream, int socket, int64_t* memused) { + if (stream->index < stream->length) return 1; if (!stream->buf) { *memused += HTTP_REQUEST_BUF_SIZE; stream->buf = (char*)calloc(1, HTTP_REQUEST_BUF_SIZE); @@ -742,7 +742,7 @@ int hs_stream_read_socket(hs_stream_t* stream, int socket, int64_t* memused) { int hs_stream_next(hs_stream_t* stream, char* c) { HTTP_FLAG_CLEAR(stream->flags, HS_SF_CONSUMED); - if (stream->index >= stream->length) return 0; + if (stream->index >= stream->length || stream->buf == NULL) return 0; *c = stream->buf[stream->index]; return 1; } @@ -831,7 +831,9 @@ http_token_t hs_transition_action( int8_t to ) { http_token_t emitted = {0, 0, 0}; - if (from == HN) hs_stream_anchor(stream); + if (from == HN) { + hs_stream_anchor(stream); + } if (from != to) { int type = hs_token_start_states[to]; if (type != HS_TOK_NONE) hs_stream_begin_token(stream, type); @@ -881,7 +883,7 @@ http_token_t hs_transition_action( if (parser->meta == M_BIG || parser->meta == M_CHK) { emitted.type = HS_TOK_BODY_STREAM; } - if (parser->meta == M_CHK) parser->state = CS; + //if (parser->meta == M_CHK) parser->state = CS; hs_trigger_meta(parser, HS_META_END_HEADERS); if (parser->meta == M_END) { emitted.type = HS_TOK_BODY; @@ -950,13 +952,16 @@ http_token_t http_parse(http_parser_t* parser, hs_stream_t* stream) { while (hs_stream_next(stream, &c)) { int type = c < 0 ? HS_ETC : hs_ctype[(int)c]; int to = hs_transitions[parser->state * HS_CHAR_TYPE_LEN + type]; + if (parser->meta == M_ZER && parser->state == HN && to == BD) { + to = CS; + } int from = parser->state; parser->state = to; http_token_t emitted = hs_transition_action(parser, stream, c, from, to); hs_stream_consume(stream); if (emitted.type != HS_TOK_NONE) return emitted; } - if (parser->state >= CS && parser->state <= C2) hs_stream_shift(stream); + if (parser->state == CB) hs_stream_shift(stream); token = hs_meta_emit(parser); http_token_t current = hs_stream_current_token(stream); if ( @@ -1018,7 +1023,6 @@ void hs_free_buffer(http_request_t* session) { void hs_init_session(http_request_t* session) { session->flags = HTTP_AUTOMATIC; - session->i = 0; session->parser = (http_parser_t){ }; session->stream = (hs_stream_t){ }; if (session->tokens.buf) { @@ -1041,6 +1045,8 @@ void hs_reset_timeout(http_request_t* request, int time) { request->timeout = time; } +void hs_read_and_process_request(http_request_t* request); + void hs_write_response(http_request_t* request) { if (!hs_write_client_socket(request)) { return hs_end_session(request); } if (request->stream.total_bytes != request->stream.length) { @@ -1057,7 +1063,7 @@ void hs_write_response(http_request_t* request) { hs_free_buffer(request); request->chunk_cb(request); } else { - hs_process_tokens(request); + hs_read_and_process_request(request); } } @@ -1070,54 +1076,42 @@ void hs_error_response(http_request_t* request, int code, char const * message) hs_write_response(request); } -int hs_has_buffered_tokens(http_request_t* request) { - return request->i < request->tokens.size; -} - -void hs_read_and_parse_tokens(http_request_t* request) { +void hs_read_and_process_request(http_request_t* request) { request->state = HTTP_SESSION_READ; http_token_t token = {0, 0, 0}; hs_reset_timeout(request, HTTP_REQUEST_TIMEOUT); int rc = hs_stream_read_socket(&request->stream, request->socket, &request->server->memused); + if (rc == 0) return hs_end_session(request); do { token = http_parse(&request->parser, &request->stream); - if (token.type != HS_TOK_NONE) { - http_token_dyn_push(&request->tokens, token); + if (token.type != HS_TOK_NONE) http_token_dyn_push(&request->tokens, token); + switch (token.type) { + case HS_TOK_ERROR: + hs_error_response(request, 400, "Bad Request"); + break; + case HS_TOK_BODY: + case HS_TOK_BODY_STREAM: + if (token.type == HS_TOK_BODY_STREAM) { + HTTP_FLAG_SET(request->flags, HTTP_FLG_STREAMED); + } + request->state = HTTP_SESSION_NOP; + request->server->request_handler(request); + break; + case HS_TOK_CHUNK_BODY: + request->state = HTTP_SESSION_NOP; + request->chunk_cb(request); + break; + case HS_TOK_REQ_END: + if (HTTP_FLAG_CHECK(request->flags, HTTP_KEEP_ALIVE)) { + request->state = HTTP_SESSION_INIT; + hs_free_buffer(request); + hs_reset_timeout(request, HTTP_KEEP_ALIVE_TIMEOUT); + } else { + hs_end_session(request); + } + return; } } while (token.type != HS_TOK_NONE); - token.type = HS_TOK_EOF; - if (rc == 0) http_token_dyn_push(&request->tokens, token); -} - -void hs_process_tokens(http_request_t* request) { - while (request->i < request->tokens.size) { - http_token_t token = request->tokens.buf[request->i]; - request->i++; - if (token.type == HS_TOK_EOF) { - return hs_end_session(request); - } else if (token.type == HS_TOK_ERROR) { - hs_error_response(request, 400, "Bad Request"); - } else if (token.type == HS_TOK_BODY || token.type == HS_TOK_BODY_STREAM) { - if (token.type == HS_TOK_BODY_STREAM) { - HTTP_FLAG_SET(request->flags, HTTP_FLG_STREAMED); - } - request->state = HTTP_SESSION_NOP; - request->server->request_handler(request); - break; - } else if (token.type == HS_TOK_CHUNK_BODY) { - request->state = HTTP_SESSION_NOP; - request->chunk_cb(request); - break; - } else if (token.type == HS_TOK_REQ_END) { - if (HTTP_FLAG_CHECK(request->flags, HTTP_KEEP_ALIVE)) { - request->state = HTTP_SESSION_INIT; - hs_free_buffer(request); - hs_reset_timeout(request, HTTP_KEEP_ALIVE_TIMEOUT); - } else { - return hs_end_session(request); - } - } - } } // Application requesting next chunk of request body. @@ -1126,12 +1120,7 @@ void http_request_read_chunk( void (*chunk_cb)(struct http_request_s*) ) { request->chunk_cb = chunk_cb; - if (hs_has_buffered_tokens(request)) { - hs_process_tokens(request); - } else { - hs_read_and_parse_tokens(request); - hs_process_tokens(request); - } + hs_read_and_process_request(request); } // This is the heart of the request logic. This is the state machine that @@ -1146,8 +1135,7 @@ void http_session(http_request_t* request) { } // fallthrough case HTTP_SESSION_READ: - hs_read_and_parse_tokens(request); - hs_process_tokens(request); + hs_read_and_process_request(request); break; case HTTP_SESSION_WRITE: hs_write_response(request); @@ -1356,7 +1344,7 @@ void http_request_connection(http_request_t* request, int directive) { } http_string_t http_request_chunk(struct http_request_s* request) { - http_token_t token = request->tokens.buf[request->i - 1]; + http_token_t token = request->tokens.buf[request->tokens.size - 1]; return (http_string_t) { .buf = &request->stream.buf[token.index], .len = token.len From fd481d2d2358505462f2ae7fb2f9a0cf062e9507 Mon Sep 17 00:00:00 2001 From: Jeremy Williams Date: Mon, 10 Feb 2020 12:25:07 -0500 Subject: [PATCH 12/20] Revert "Remove debug code" This reverts commit 6c830e6d463946f582f83486a36c515ce91b9836. --- httpserver.h | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/httpserver.h b/httpserver.h index c402754..b9db847 100644 --- a/httpserver.h +++ b/httpserver.h @@ -736,6 +736,7 @@ int hs_stream_read_socket(hs_stream_t* stream, int socket, int64_t* memused) { stream->buf = (char*)realloc(stream->buf, stream->capacity); assert(stream->buf != NULL); } + //printf("bytes: %d, total: %ld\n", bytes, stream->total_bytes); } while (bytes > 0 && stream->capacity < HTTP_MAX_REQUEST_BUF_SIZE); return bytes == 0 ? 0 : 1; } @@ -812,8 +813,14 @@ void hs_stream_shift(hs_stream_t* stream) { // *** http parser *** +char const* mstate[] = { + "M_WFK", "M_ANY", "M_MTE", "M_MCL", "M_CLV", "M_MCK", "M_SML", "M_CHK", "M_BIG", + "M_ZER", "M_CSZ", "M_CBD", "M_STR", "M_BDY", "M_LST", "M_SEN", "M_END", "M_ERR" +}; + void hs_trigger_meta(http_parser_t* parser, int event) { int to = hs_meta_transitions[parser->meta * HS_META_TYPE_LEN + event]; + //printf("-> %s\n", mstate[to]); parser->meta = to; } @@ -930,6 +937,11 @@ http_token_t hs_transition_action( return emitted; } +char const* pstate[] = { + "ST", "MT", "MS", "TR", "TS", "VN", "RR", "RN", "HK", "HS", "HV", "HR", "HE", + "ER", "HN", "BD", "CS", "CB", "CE", "CR", "CN", "CD", "C1", "C2", "BR", "HS_STATE_LEN" +}; + http_token_t hs_meta_emit(http_parser_t* parser) { http_token_t token = {0, 0, 0}; switch (parser->meta) { @@ -955,6 +967,7 @@ http_token_t http_parse(http_parser_t* parser, hs_stream_t* stream) { if (parser->meta == M_ZER && parser->state == HN && to == BD) { to = CS; } + //printf("to: %s\n", pstate[to]); int from = parser->state; parser->state = to; http_token_t emitted = hs_transition_action(parser, stream, c, from, to); @@ -1048,6 +1061,7 @@ void hs_reset_timeout(http_request_t* request, int time) { void hs_read_and_process_request(http_request_t* request); void hs_write_response(http_request_t* request) { + //printf("writing!\n"); if (!hs_write_client_socket(request)) { return hs_end_session(request); } if (request->stream.total_bytes != request->stream.length) { // All bytes of the body were not written and we need to wait until the @@ -1076,7 +1090,15 @@ void hs_error_response(http_request_t* request, int code, char const * message) hs_write_response(request); } + +char const* tokens[] = { + "HS_TOK_NONE", "HS_TOK_METHOD", "HS_TOK_TARGET", "HS_TOK_VERSION", + "HS_TOK_HEADER_KEY", "HS_TOK_HEADER_VAL", "HS_TOK_CHUNK_BODY", "HS_TOK_BODY", + "HS_TOK_BODY_STREAM", "HS_TOK_REQ_END", "HS_TOK_EOF", "HS_TOK_ERROR" +}; + void hs_read_and_process_request(http_request_t* request) { + //printf("read state set\n"); request->state = HTTP_SESSION_READ; http_token_t token = {0, 0, 0}; hs_reset_timeout(request, HTTP_REQUEST_TIMEOUT); @@ -1084,6 +1106,7 @@ void hs_read_and_process_request(http_request_t* request) { if (rc == 0) return hs_end_session(request); do { token = http_parse(&request->parser, &request->stream); + //printf("EMITTED: %s, len: %d UNREAD: %d\n", tokens[token.type], token.len, request->stream.length - request->stream.index); if (token.type != HS_TOK_NONE) http_token_dyn_push(&request->tokens, token); switch (token.type) { case HS_TOK_ERROR: @@ -1119,6 +1142,7 @@ void http_request_read_chunk( struct http_request_s* request, void (*chunk_cb)(struct http_request_s*) ) { + //printf("reading chunk!\n"); request->chunk_cb = chunk_cb; hs_read_and_process_request(request); } @@ -1126,6 +1150,7 @@ void http_request_read_chunk( // This is the heart of the request logic. This is the state machine that // controls what happens when an IO event is received. void http_session(http_request_t* request) { + //printf("io clabback!\n"); switch (request->state) { case HTTP_SESSION_INIT: hs_init_session(request); @@ -1141,6 +1166,7 @@ void http_session(http_request_t* request) { hs_write_response(request); break; case HTTP_SESSION_NOP: + //printf("NOP!\n"); break; } } From 1710de7ad119bcbac90b6736631d7e6122381870 Mon Sep 17 00:00:00 2001 From: Jeremy Williams Date: Mon, 10 Feb 2020 13:36:45 -0500 Subject: [PATCH 13/20] Fix bug --- httpserver.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/httpserver.h b/httpserver.h index b9db847..c12208c 100644 --- a/httpserver.h +++ b/httpserver.h @@ -1022,6 +1022,7 @@ int hs_write_client_socket(http_request_t* session) { session->stream.buf + session->stream.total_bytes, session->stream.length - session->stream.total_bytes ); + //printf("bytes: %d\n", bytes); if (bytes > 0) session->stream.total_bytes += bytes; return errno == EPIPE ? 0 : 1; } @@ -1069,6 +1070,7 @@ void hs_write_response(http_request_t* request) { hs_add_write_event(request); request->state = HTTP_SESSION_WRITE; hs_reset_timeout(request, HTTP_REQUEST_TIMEOUT); + //printf("continue writing\n"); } else if (HTTP_FLAG_CHECK(request->flags, HTTP_CHUNKED_RESPONSE)) { // All bytes of the chunk were written and we need to get the next chunk // from the application. @@ -1134,7 +1136,7 @@ void hs_read_and_process_request(http_request_t* request) { } return; } - } while (token.type != HS_TOK_NONE); + } while (token.type != HS_TOK_NONE && request->state == HTTP_SESSION_READ); } // Application requesting next chunk of request body. From b95046f4bddeb660ade14f56207965b5938ad4e9 Mon Sep 17 00:00:00 2001 From: Jeremy Williams Date: Mon, 10 Feb 2020 13:38:05 -0500 Subject: [PATCH 14/20] Remove debug code --- httpserver.h | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/httpserver.h b/httpserver.h index c12208c..b2abbaf 100644 --- a/httpserver.h +++ b/httpserver.h @@ -736,7 +736,6 @@ int hs_stream_read_socket(hs_stream_t* stream, int socket, int64_t* memused) { stream->buf = (char*)realloc(stream->buf, stream->capacity); assert(stream->buf != NULL); } - //printf("bytes: %d, total: %ld\n", bytes, stream->total_bytes); } while (bytes > 0 && stream->capacity < HTTP_MAX_REQUEST_BUF_SIZE); return bytes == 0 ? 0 : 1; } @@ -813,14 +812,8 @@ void hs_stream_shift(hs_stream_t* stream) { // *** http parser *** -char const* mstate[] = { - "M_WFK", "M_ANY", "M_MTE", "M_MCL", "M_CLV", "M_MCK", "M_SML", "M_CHK", "M_BIG", - "M_ZER", "M_CSZ", "M_CBD", "M_STR", "M_BDY", "M_LST", "M_SEN", "M_END", "M_ERR" -}; - void hs_trigger_meta(http_parser_t* parser, int event) { int to = hs_meta_transitions[parser->meta * HS_META_TYPE_LEN + event]; - //printf("-> %s\n", mstate[to]); parser->meta = to; } @@ -937,11 +930,6 @@ http_token_t hs_transition_action( return emitted; } -char const* pstate[] = { - "ST", "MT", "MS", "TR", "TS", "VN", "RR", "RN", "HK", "HS", "HV", "HR", "HE", - "ER", "HN", "BD", "CS", "CB", "CE", "CR", "CN", "CD", "C1", "C2", "BR", "HS_STATE_LEN" -}; - http_token_t hs_meta_emit(http_parser_t* parser) { http_token_t token = {0, 0, 0}; switch (parser->meta) { @@ -967,7 +955,6 @@ http_token_t http_parse(http_parser_t* parser, hs_stream_t* stream) { if (parser->meta == M_ZER && parser->state == HN && to == BD) { to = CS; } - //printf("to: %s\n", pstate[to]); int from = parser->state; parser->state = to; http_token_t emitted = hs_transition_action(parser, stream, c, from, to); @@ -1022,7 +1009,6 @@ int hs_write_client_socket(http_request_t* session) { session->stream.buf + session->stream.total_bytes, session->stream.length - session->stream.total_bytes ); - //printf("bytes: %d\n", bytes); if (bytes > 0) session->stream.total_bytes += bytes; return errno == EPIPE ? 0 : 1; } @@ -1062,7 +1048,6 @@ void hs_reset_timeout(http_request_t* request, int time) { void hs_read_and_process_request(http_request_t* request); void hs_write_response(http_request_t* request) { - //printf("writing!\n"); if (!hs_write_client_socket(request)) { return hs_end_session(request); } if (request->stream.total_bytes != request->stream.length) { // All bytes of the body were not written and we need to wait until the @@ -1070,7 +1055,6 @@ void hs_write_response(http_request_t* request) { hs_add_write_event(request); request->state = HTTP_SESSION_WRITE; hs_reset_timeout(request, HTTP_REQUEST_TIMEOUT); - //printf("continue writing\n"); } else if (HTTP_FLAG_CHECK(request->flags, HTTP_CHUNKED_RESPONSE)) { // All bytes of the chunk were written and we need to get the next chunk // from the application. @@ -1093,14 +1077,7 @@ void hs_error_response(http_request_t* request, int code, char const * message) } -char const* tokens[] = { - "HS_TOK_NONE", "HS_TOK_METHOD", "HS_TOK_TARGET", "HS_TOK_VERSION", - "HS_TOK_HEADER_KEY", "HS_TOK_HEADER_VAL", "HS_TOK_CHUNK_BODY", "HS_TOK_BODY", - "HS_TOK_BODY_STREAM", "HS_TOK_REQ_END", "HS_TOK_EOF", "HS_TOK_ERROR" -}; - void hs_read_and_process_request(http_request_t* request) { - //printf("read state set\n"); request->state = HTTP_SESSION_READ; http_token_t token = {0, 0, 0}; hs_reset_timeout(request, HTTP_REQUEST_TIMEOUT); @@ -1108,7 +1085,6 @@ void hs_read_and_process_request(http_request_t* request) { if (rc == 0) return hs_end_session(request); do { token = http_parse(&request->parser, &request->stream); - //printf("EMITTED: %s, len: %d UNREAD: %d\n", tokens[token.type], token.len, request->stream.length - request->stream.index); if (token.type != HS_TOK_NONE) http_token_dyn_push(&request->tokens, token); switch (token.type) { case HS_TOK_ERROR: @@ -1144,7 +1120,6 @@ void http_request_read_chunk( struct http_request_s* request, void (*chunk_cb)(struct http_request_s*) ) { - //printf("reading chunk!\n"); request->chunk_cb = chunk_cb; hs_read_and_process_request(request); } @@ -1152,7 +1127,6 @@ void http_request_read_chunk( // This is the heart of the request logic. This is the state machine that // controls what happens when an IO event is received. void http_session(http_request_t* request) { - //printf("io clabback!\n"); switch (request->state) { case HTTP_SESSION_INIT: hs_init_session(request); @@ -1168,7 +1142,6 @@ void http_session(http_request_t* request) { hs_write_response(request); break; case HTTP_SESSION_NOP: - //printf("NOP!\n"); break; } } From f63f9ef0371991ed2ecfd1d8adf4e3b11dcfa985 Mon Sep 17 00:00:00 2001 From: Jeremy Williams Date: Mon, 10 Feb 2020 13:51:49 -0500 Subject: [PATCH 15/20] Revert "Remove debug code" This reverts commit b95046f4bddeb660ade14f56207965b5938ad4e9. --- httpserver.h | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/httpserver.h b/httpserver.h index b2abbaf..c12208c 100644 --- a/httpserver.h +++ b/httpserver.h @@ -736,6 +736,7 @@ int hs_stream_read_socket(hs_stream_t* stream, int socket, int64_t* memused) { stream->buf = (char*)realloc(stream->buf, stream->capacity); assert(stream->buf != NULL); } + //printf("bytes: %d, total: %ld\n", bytes, stream->total_bytes); } while (bytes > 0 && stream->capacity < HTTP_MAX_REQUEST_BUF_SIZE); return bytes == 0 ? 0 : 1; } @@ -812,8 +813,14 @@ void hs_stream_shift(hs_stream_t* stream) { // *** http parser *** +char const* mstate[] = { + "M_WFK", "M_ANY", "M_MTE", "M_MCL", "M_CLV", "M_MCK", "M_SML", "M_CHK", "M_BIG", + "M_ZER", "M_CSZ", "M_CBD", "M_STR", "M_BDY", "M_LST", "M_SEN", "M_END", "M_ERR" +}; + void hs_trigger_meta(http_parser_t* parser, int event) { int to = hs_meta_transitions[parser->meta * HS_META_TYPE_LEN + event]; + //printf("-> %s\n", mstate[to]); parser->meta = to; } @@ -930,6 +937,11 @@ http_token_t hs_transition_action( return emitted; } +char const* pstate[] = { + "ST", "MT", "MS", "TR", "TS", "VN", "RR", "RN", "HK", "HS", "HV", "HR", "HE", + "ER", "HN", "BD", "CS", "CB", "CE", "CR", "CN", "CD", "C1", "C2", "BR", "HS_STATE_LEN" +}; + http_token_t hs_meta_emit(http_parser_t* parser) { http_token_t token = {0, 0, 0}; switch (parser->meta) { @@ -955,6 +967,7 @@ http_token_t http_parse(http_parser_t* parser, hs_stream_t* stream) { if (parser->meta == M_ZER && parser->state == HN && to == BD) { to = CS; } + //printf("to: %s\n", pstate[to]); int from = parser->state; parser->state = to; http_token_t emitted = hs_transition_action(parser, stream, c, from, to); @@ -1009,6 +1022,7 @@ int hs_write_client_socket(http_request_t* session) { session->stream.buf + session->stream.total_bytes, session->stream.length - session->stream.total_bytes ); + //printf("bytes: %d\n", bytes); if (bytes > 0) session->stream.total_bytes += bytes; return errno == EPIPE ? 0 : 1; } @@ -1048,6 +1062,7 @@ void hs_reset_timeout(http_request_t* request, int time) { void hs_read_and_process_request(http_request_t* request); void hs_write_response(http_request_t* request) { + //printf("writing!\n"); if (!hs_write_client_socket(request)) { return hs_end_session(request); } if (request->stream.total_bytes != request->stream.length) { // All bytes of the body were not written and we need to wait until the @@ -1055,6 +1070,7 @@ void hs_write_response(http_request_t* request) { hs_add_write_event(request); request->state = HTTP_SESSION_WRITE; hs_reset_timeout(request, HTTP_REQUEST_TIMEOUT); + //printf("continue writing\n"); } else if (HTTP_FLAG_CHECK(request->flags, HTTP_CHUNKED_RESPONSE)) { // All bytes of the chunk were written and we need to get the next chunk // from the application. @@ -1077,7 +1093,14 @@ void hs_error_response(http_request_t* request, int code, char const * message) } +char const* tokens[] = { + "HS_TOK_NONE", "HS_TOK_METHOD", "HS_TOK_TARGET", "HS_TOK_VERSION", + "HS_TOK_HEADER_KEY", "HS_TOK_HEADER_VAL", "HS_TOK_CHUNK_BODY", "HS_TOK_BODY", + "HS_TOK_BODY_STREAM", "HS_TOK_REQ_END", "HS_TOK_EOF", "HS_TOK_ERROR" +}; + void hs_read_and_process_request(http_request_t* request) { + //printf("read state set\n"); request->state = HTTP_SESSION_READ; http_token_t token = {0, 0, 0}; hs_reset_timeout(request, HTTP_REQUEST_TIMEOUT); @@ -1085,6 +1108,7 @@ void hs_read_and_process_request(http_request_t* request) { if (rc == 0) return hs_end_session(request); do { token = http_parse(&request->parser, &request->stream); + //printf("EMITTED: %s, len: %d UNREAD: %d\n", tokens[token.type], token.len, request->stream.length - request->stream.index); if (token.type != HS_TOK_NONE) http_token_dyn_push(&request->tokens, token); switch (token.type) { case HS_TOK_ERROR: @@ -1120,6 +1144,7 @@ void http_request_read_chunk( struct http_request_s* request, void (*chunk_cb)(struct http_request_s*) ) { + //printf("reading chunk!\n"); request->chunk_cb = chunk_cb; hs_read_and_process_request(request); } @@ -1127,6 +1152,7 @@ void http_request_read_chunk( // This is the heart of the request logic. This is the state machine that // controls what happens when an IO event is received. void http_session(http_request_t* request) { + //printf("io clabback!\n"); switch (request->state) { case HTTP_SESSION_INIT: hs_init_session(request); @@ -1142,6 +1168,7 @@ void http_session(http_request_t* request) { hs_write_response(request); break; case HTTP_SESSION_NOP: + //printf("NOP!\n"); break; } } From 1fc1f0cdd2cd4b30e584570d417633573f39fdca Mon Sep 17 00:00:00 2001 From: Jeremy Williams Date: Mon, 10 Feb 2020 14:30:04 -0500 Subject: [PATCH 16/20] Fix memory leak --- httpserver.h | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/httpserver.h b/httpserver.h index c12208c..2e65339 100644 --- a/httpserver.h +++ b/httpserver.h @@ -1079,7 +1079,13 @@ void hs_write_response(http_request_t* request) { hs_free_buffer(request); request->chunk_cb(request); } else { - hs_read_and_process_request(request); + if (HTTP_FLAG_CHECK(request->flags, HTTP_KEEP_ALIVE)) { + request->state = HTTP_SESSION_INIT; + hs_free_buffer(request); + hs_reset_timeout(request, HTTP_KEEP_ALIVE_TIMEOUT); + } else { + hs_end_session(request); + } } } @@ -1126,15 +1132,6 @@ void hs_read_and_process_request(http_request_t* request) { request->state = HTTP_SESSION_NOP; request->chunk_cb(request); break; - case HS_TOK_REQ_END: - if (HTTP_FLAG_CHECK(request->flags, HTTP_KEEP_ALIVE)) { - request->state = HTTP_SESSION_INIT; - hs_free_buffer(request); - hs_reset_timeout(request, HTTP_KEEP_ALIVE_TIMEOUT); - } else { - hs_end_session(request); - } - return; } } while (token.type != HS_TOK_NONE && request->state == HTTP_SESSION_READ); } From 63ecfa4ea84aa05c5527f72d41c4352bfe54fd12 Mon Sep 17 00:00:00 2001 From: Jeremy Williams Date: Mon, 10 Feb 2020 15:04:05 -0500 Subject: [PATCH 17/20] fix use after free --- httpserver.h | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/httpserver.h b/httpserver.h index 2e65339..375a440 100644 --- a/httpserver.h +++ b/httpserver.h @@ -357,6 +357,7 @@ int main() { #define HTTP_SESSION_NOP 3 // http session flags +#define HTTP_END_SESSION 0x1 #define HTTP_AUTOMATIC 0x8 #define HTTP_CHUNKED_RESPONSE 0x20 @@ -1063,7 +1064,10 @@ void hs_read_and_process_request(http_request_t* request); void hs_write_response(http_request_t* request) { //printf("writing!\n"); - if (!hs_write_client_socket(request)) { return hs_end_session(request); } + if (!hs_write_client_socket(request)) { + HTTP_FLAG_SET(request->flags, HTTP_END_SESSION); + return; + } if (request->stream.total_bytes != request->stream.length) { // All bytes of the body were not written and we need to wait until the // socket is writable again to complete the write @@ -1084,7 +1088,7 @@ void hs_write_response(http_request_t* request) { hs_free_buffer(request); hs_reset_timeout(request, HTTP_KEEP_ALIVE_TIMEOUT); } else { - hs_end_session(request); + HTTP_FLAG_SET(request->flags, HTTP_END_SESSION); } } } @@ -1111,7 +1115,10 @@ void hs_read_and_process_request(http_request_t* request) { http_token_t token = {0, 0, 0}; hs_reset_timeout(request, HTTP_REQUEST_TIMEOUT); int rc = hs_stream_read_socket(&request->stream, request->socket, &request->server->memused); - if (rc == 0) return hs_end_session(request); + if (rc == 0) { + HTTP_FLAG_SET(request->flags, HTTP_END_SESSION); + return; + } do { token = http_parse(&request->parser, &request->stream); //printf("EMITTED: %s, len: %d UNREAD: %d\n", tokens[token.type], token.len, request->stream.length - request->stream.index); @@ -1164,9 +1171,9 @@ void http_session(http_request_t* request) { case HTTP_SESSION_WRITE: hs_write_response(request); break; - case HTTP_SESSION_NOP: - //printf("NOP!\n"); - break; + } + if (HTTP_FLAG_CHECK(request->flags, HTTP_END_SESSION)) { + hs_end_session(request); } } From 4a18e43a4e5c10e491dae2f75a4e80dc14587a9f Mon Sep 17 00:00:00 2001 From: Jeremy Williams Date: Mon, 10 Feb 2020 15:47:28 -0500 Subject: [PATCH 18/20] Fix flag collision --- httpserver.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/httpserver.h b/httpserver.h index 375a440..ee7370a 100644 --- a/httpserver.h +++ b/httpserver.h @@ -357,7 +357,7 @@ int main() { #define HTTP_SESSION_NOP 3 // http session flags -#define HTTP_END_SESSION 0x1 +#define HTTP_END_SESSION 0x2 #define HTTP_AUTOMATIC 0x8 #define HTTP_CHUNKED_RESPONSE 0x20 @@ -816,7 +816,7 @@ void hs_stream_shift(hs_stream_t* stream) { char const* mstate[] = { "M_WFK", "M_ANY", "M_MTE", "M_MCL", "M_CLV", "M_MCK", "M_SML", "M_CHK", "M_BIG", - "M_ZER", "M_CSZ", "M_CBD", "M_STR", "M_BDY", "M_LST", "M_SEN", "M_END", "M_ERR" + "M_ZER", "M_CSZ", "M_CBD", "M_LST", "M_STR", "M_SEN", "M_BDY", "M_END", "M_ERR" }; void hs_trigger_meta(http_parser_t* parser, int event) { From 869e398bf30b9829464f76a5c95378f231f651a9 Mon Sep 17 00:00:00 2001 From: Jeremy Williams Date: Mon, 10 Feb 2020 16:06:16 -0500 Subject: [PATCH 19/20] remove debug code --- httpserver.h | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/httpserver.h b/httpserver.h index ee7370a..0b3d097 100644 --- a/httpserver.h +++ b/httpserver.h @@ -737,7 +737,6 @@ int hs_stream_read_socket(hs_stream_t* stream, int socket, int64_t* memused) { stream->buf = (char*)realloc(stream->buf, stream->capacity); assert(stream->buf != NULL); } - //printf("bytes: %d, total: %ld\n", bytes, stream->total_bytes); } while (bytes > 0 && stream->capacity < HTTP_MAX_REQUEST_BUF_SIZE); return bytes == 0 ? 0 : 1; } @@ -814,14 +813,8 @@ void hs_stream_shift(hs_stream_t* stream) { // *** http parser *** -char const* mstate[] = { - "M_WFK", "M_ANY", "M_MTE", "M_MCL", "M_CLV", "M_MCK", "M_SML", "M_CHK", "M_BIG", - "M_ZER", "M_CSZ", "M_CBD", "M_LST", "M_STR", "M_SEN", "M_BDY", "M_END", "M_ERR" -}; - void hs_trigger_meta(http_parser_t* parser, int event) { int to = hs_meta_transitions[parser->meta * HS_META_TYPE_LEN + event]; - //printf("-> %s\n", mstate[to]); parser->meta = to; } @@ -938,11 +931,6 @@ http_token_t hs_transition_action( return emitted; } -char const* pstate[] = { - "ST", "MT", "MS", "TR", "TS", "VN", "RR", "RN", "HK", "HS", "HV", "HR", "HE", - "ER", "HN", "BD", "CS", "CB", "CE", "CR", "CN", "CD", "C1", "C2", "BR", "HS_STATE_LEN" -}; - http_token_t hs_meta_emit(http_parser_t* parser) { http_token_t token = {0, 0, 0}; switch (parser->meta) { @@ -968,7 +956,6 @@ http_token_t http_parse(http_parser_t* parser, hs_stream_t* stream) { if (parser->meta == M_ZER && parser->state == HN && to == BD) { to = CS; } - //printf("to: %s\n", pstate[to]); int from = parser->state; parser->state = to; http_token_t emitted = hs_transition_action(parser, stream, c, from, to); @@ -1023,7 +1010,6 @@ int hs_write_client_socket(http_request_t* session) { session->stream.buf + session->stream.total_bytes, session->stream.length - session->stream.total_bytes ); - //printf("bytes: %d\n", bytes); if (bytes > 0) session->stream.total_bytes += bytes; return errno == EPIPE ? 0 : 1; } @@ -1063,7 +1049,6 @@ void hs_reset_timeout(http_request_t* request, int time) { void hs_read_and_process_request(http_request_t* request); void hs_write_response(http_request_t* request) { - //printf("writing!\n"); if (!hs_write_client_socket(request)) { HTTP_FLAG_SET(request->flags, HTTP_END_SESSION); return; @@ -1074,7 +1059,6 @@ void hs_write_response(http_request_t* request) { hs_add_write_event(request); request->state = HTTP_SESSION_WRITE; hs_reset_timeout(request, HTTP_REQUEST_TIMEOUT); - //printf("continue writing\n"); } else if (HTTP_FLAG_CHECK(request->flags, HTTP_CHUNKED_RESPONSE)) { // All bytes of the chunk were written and we need to get the next chunk // from the application. @@ -1102,15 +1086,7 @@ void hs_error_response(http_request_t* request, int code, char const * message) hs_write_response(request); } - -char const* tokens[] = { - "HS_TOK_NONE", "HS_TOK_METHOD", "HS_TOK_TARGET", "HS_TOK_VERSION", - "HS_TOK_HEADER_KEY", "HS_TOK_HEADER_VAL", "HS_TOK_CHUNK_BODY", "HS_TOK_BODY", - "HS_TOK_BODY_STREAM", "HS_TOK_REQ_END", "HS_TOK_EOF", "HS_TOK_ERROR" -}; - void hs_read_and_process_request(http_request_t* request) { - //printf("read state set\n"); request->state = HTTP_SESSION_READ; http_token_t token = {0, 0, 0}; hs_reset_timeout(request, HTTP_REQUEST_TIMEOUT); @@ -1121,7 +1097,6 @@ void hs_read_and_process_request(http_request_t* request) { } do { token = http_parse(&request->parser, &request->stream); - //printf("EMITTED: %s, len: %d UNREAD: %d\n", tokens[token.type], token.len, request->stream.length - request->stream.index); if (token.type != HS_TOK_NONE) http_token_dyn_push(&request->tokens, token); switch (token.type) { case HS_TOK_ERROR: @@ -1148,7 +1123,6 @@ void http_request_read_chunk( struct http_request_s* request, void (*chunk_cb)(struct http_request_s*) ) { - //printf("reading chunk!\n"); request->chunk_cb = chunk_cb; hs_read_and_process_request(request); } @@ -1156,7 +1130,6 @@ void http_request_read_chunk( // This is the heart of the request logic. This is the state machine that // controls what happens when an IO event is received. void http_session(http_request_t* request) { - //printf("io clabback!\n"); switch (request->state) { case HTTP_SESSION_INIT: hs_init_session(request); From 25a04393777c737d2b7157a3b8e05ac07008176c Mon Sep 17 00:00:00 2001 From: Jeremy Williams Date: Mon, 10 Feb 2020 16:25:58 -0500 Subject: [PATCH 20/20] Remove unnecessary null check --- httpserver.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/httpserver.h b/httpserver.h index 0b3d097..12309e9 100644 --- a/httpserver.h +++ b/httpserver.h @@ -743,7 +743,7 @@ int hs_stream_read_socket(hs_stream_t* stream, int socket, int64_t* memused) { int hs_stream_next(hs_stream_t* stream, char* c) { HTTP_FLAG_CLEAR(stream->flags, HS_SF_CONSUMED); - if (stream->index >= stream->length || stream->buf == NULL) return 0; + if (stream->index >= stream->length) return 0; *c = stream->buf[stream->index]; return 1; }