/* HTTP request/response handling Copyright (C) 1999-2010, Joe Orton This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA */ /* This is the HTTP client request/response implementation. * The goal of this code is to be modular and simple. */ #include "config.h" #include #include #include #ifdef HAVE_STRING_H #include #endif #ifdef HAVE_STRINGS_H #include #endif #ifdef HAVE_STDLIB_H #include #endif #ifdef HAVE_UNISTD_H #include #endif #include "ne_internal.h" #include "ne_alloc.h" #include "ne_request.h" #include "ne_string.h" /* for ne_buffer */ #include "ne_utils.h" #include "ne_socket.h" #include "ne_uri.h" #include "ne_private.h" #define SOCK_ERR(req, op, msg) do { ssize_t sret = (op); \ if (sret < 0) return aborted(req, msg, sret); } while (0) #define EOL "\r\n" struct body_reader { ne_block_reader handler; ne_accept_response accept_response; unsigned int use; void *userdata; struct body_reader *next; }; struct field { char *name, *value; size_t vlen; struct field *next; }; /* Maximum number of header fields per response: */ #define MAX_HEADER_FIELDS (100) /* Size of hash table; 43 is the smallest prime for which the common * header names hash uniquely using the *33 hash function. */ #define HH_HASHSIZE (43) /* Hash iteration step: *33 known to be a good hash for ASCII, see RSE. */ #define HH_ITERATE(hash, ch) (((hash)*33 + (unsigned char)(ch)) % HH_HASHSIZE) /* pre-calculated hash values for given header names: */ #define HH_HV_CONNECTION (0x14) #define HH_HV_PROXY_CONNECTION (0x1A) #define HH_HV_CONTENT_LENGTH (0x13) #define HH_HV_TRANSFER_ENCODING (0x07) struct ne_request_s { char *method, *uri; /* method and Request-URI */ ne_buffer *headers; /* request headers */ /* Request body. */ ne_provide_body body_cb; void *body_ud; /* Request body source: file or buffer (if not callback). */ union { struct { int fd; ne_off_t offset, length; ne_off_t remain; /* remaining bytes to send. */ } file; struct { /* length bytes @ buffer = whole body. * remain bytes @ pnt = remaining bytes to send */ const char *buffer, *pnt; size_t length, remain; } buf; } body; ne_off_t body_length; /* length of request body */ /* temporary store for response lines. */ char respbuf[NE_BUFSIZ]; /**** Response ***/ /* The transfer encoding types */ struct ne_response { enum { R_TILLEOF = 0, /* read till eof */ R_NO_BODY, /* implicitly no body (HEAD, 204, 304) */ R_CHUNKED, /* using chunked transfer-encoding */ R_CLENGTH /* using given content-length */ } mode; union { /* clen: used if mode == R_CLENGTH; total and bytes * remaining to be read of response body. */ struct { ne_off_t total, remain; } clen; /* chunk: used if mode == R_CHUNKED; total and bytes * remaining to be read of current chunk */ struct { size_t total, remain; } chunk; } body; ne_off_t progress; /* number of bytes read of response */ } resp; struct hook *private; /* response header fields */ struct field *response_headers[HH_HASHSIZE]; unsigned int current_index; /* response_headers cursor for iterator */ /* List of callbacks which are passed response body blocks */ struct body_reader *body_readers; /*** Miscellaneous ***/ unsigned int method_is_head; unsigned int can_persist; int flags[NE_REQFLAG_LAST]; ne_session *session; ne_status status; }; static int open_connection(ne_session *sess); /* Returns hash value for header 'name', converting it to lower-case * in-place. */ static inline unsigned int hash_and_lower(char *name) { char *pnt; unsigned int hash = 0; for (pnt = name; *pnt != '\0'; pnt++) { *pnt = ne_tolower(*pnt); hash = HH_ITERATE(hash,*pnt); } return hash; } /* Abort a request due to an non-recoverable HTTP protocol error, * whilst doing 'doing'. 'code', if non-zero, is the socket error * code, NE_SOCK_*, or if zero, is ignored. */ static int aborted(ne_request *req, const char *doing, ssize_t code) { ne_session *sess = req->session; int ret = NE_ERROR; NE_DEBUG(NE_DBG_HTTP, "Aborted request (%" NE_FMT_SSIZE_T "): %s\n", code, doing); switch(code) { case NE_SOCK_CLOSED: if (sess->nexthop->proxy != PROXY_NONE) { ne_set_error(sess, _("%s: connection was closed by proxy server"), doing); } else { ne_set_error(sess, _("%s: connection was closed by server"), doing); } break; case NE_SOCK_TIMEOUT: ne_set_error(sess, _("%s: connection timed out"), doing); ret = NE_TIMEOUT; break; case NE_SOCK_ERROR: case NE_SOCK_RESET: case NE_SOCK_TRUNC: ne_set_error(sess, "%s: %s", doing, ne_sock_error(sess->socket)); break; case 0: ne_set_error(sess, "%s", doing); break; } ne_close_connection(sess); return ret; } static void notify_status(ne_session *sess, ne_session_status status) { if (sess->notify_cb) { sess->notify_cb(sess->notify_ud, status, &sess->status); } } static void *get_private(const struct hook *hk, const char *id) { for (; hk != NULL; hk = hk->next) if (strcmp(hk->id, id) == 0) return hk->userdata; return NULL; } void *ne_get_request_private(ne_request *req, const char *id) { return get_private(req->private, id); } void *ne_get_session_private(ne_session *sess, const char *id) { return get_private(sess->private, id); } void ne_set_request_private(ne_request *req, const char *id, void *userdata) { struct hook *hk = ne_malloc(sizeof (struct hook)), *pos; if (req->private != NULL) { for (pos = req->private; pos->next != NULL; pos = pos->next) /* nullop */; pos->next = hk; } else { req->private = hk; } hk->id = id; hk->fn = NULL; hk->userdata = userdata; hk->next = NULL; } static ssize_t body_string_send(void *userdata, char *buffer, size_t count) { ne_request *req = userdata; if (count == 0) { req->body.buf.remain = req->body.buf.length; req->body.buf.pnt = req->body.buf.buffer; } else { /* if body_left == 0 we fall through and return 0. */ if (req->body.buf.remain < count) count = req->body.buf.remain; memcpy(buffer, req->body.buf.pnt, count); req->body.buf.pnt += count; req->body.buf.remain -= count; } return count; } static ssize_t body_fd_send(void *userdata, char *buffer, size_t count) { ne_request *req = userdata; if (count) { ssize_t ret; if (req->body.file.remain == 0) return 0; /* Casts here are necessary for LFS platforms for safe and * warning-free assignment/comparison between 32-bit size_t * and 64-bit off64_t: */ if ((ne_off_t)count > req->body.file.remain) count = (size_t)req->body.file.remain; ret = read(req->body.file.fd, buffer, count); if (ret > 0) { req->body.file.remain -= ret; return ret; } else if (ret == 0) { ne_set_error(req->session, _("Premature EOF in request body file")); } else if (ret < 0) { char err[200]; int errnum = errno; ne_set_error(req->session, _("Failed reading request body file: %s"), ne_strerror(errnum, err, sizeof err)); } return -1; } else { ne_off_t newoff; /* rewind for next send. */ newoff = ne_lseek(req->body.file.fd, req->body.file.offset, SEEK_SET); if (newoff == req->body.file.offset) { req->body.file.remain = req->body.file.length; return 0; } else { char err[200], offstr[20]; if (newoff == -1) { /* errno was set */ ne_strerror(errno, err, sizeof err); } else { ne_strnzcpy(err, _("offset invalid"), sizeof err); } ne_snprintf(offstr, sizeof offstr, "%" FMT_NE_OFF_T, req->body.file.offset); ne_set_error(req->session, _("Could not seek to offset %s" " of request body file: %s"), offstr, err); return -1; } } } /* For accurate persistent connection handling, for any write() or * read() operation for a new request on an already-open connection, * an EOF or RST error MUST be treated as a persistent connection * timeout, and the request retried on a new connection. Once a * read() operation has succeeded, any subsequent error MUST be * treated as fatal. A 'retry' flag is used; retry=1 represents the * first case, retry=0 the latter. */ /* RETRY_RET() crafts a function return value given the 'retry' flag, * the socket error 'code', and the return value 'acode' from the * aborted() function. */ #define RETRY_RET(retry, code, acode) \ ((((code) == NE_SOCK_CLOSED || (code) == NE_SOCK_RESET || \ (code) == NE_SOCK_TRUNC) && retry) ? NE_RETRY : (acode)) /* For sending chunks, an 8-byte prefix is reserved at the beginning * of the buffer. This is large enough for a trailing \r\n for the * previous chunk, the chunk size, and the \r\n following the * chunk-size. */ #define CHUNK_OFFSET (8) #define CHUNK_TERM "\r\n0\r\n\r\n" #define CHUNK_NULL_TERM "0\r\n\r\n" /* Sends the request body; returns 0 on success or an NE_* error code. * If retry is non-zero; will return NE_RETRY on persistent connection * timeout. On error, the session error string is set and the * connection is closed. */ static int send_request_body(ne_request *req, int retry) { ne_session *const sess = req->session; char buffer[NE_BUFSIZ], *start; ssize_t bytes; size_t buflen; int chunked = req->body_length < 0, chunknum = 0; int ret; NE_DEBUG(NE_DBG_HTTP, "Sending request body:\n"); /* Set up status union and (start, buflen) as the buffer to be * passed the supplied callback. */ if (chunked) { start = buffer + CHUNK_OFFSET; buflen = sizeof(buffer) - CHUNK_OFFSET; req->session->status.sr.total = -1; } else { start = buffer; buflen = sizeof buffer; req->session->status.sr.total = req->body_length; } req->session->status.sr.progress = 0; notify_status(sess, ne_status_sending); /* tell the source to start again from the beginning. */ if (req->body_cb(req->body_ud, NULL, 0) != 0) { ne_close_connection(sess); return NE_ERROR; } while ((bytes = req->body_cb(req->body_ud, start, buflen)) > 0) { req->session->status.sr.progress += bytes; if (chunked) { /* Overwrite the buffer prefix with the appropriate chunk * size; since ne_snprintf always NUL-terminates, the \n * is omitted and placed over the NUL afterwards. */ if (chunknum++ == 0) ne_snprintf(buffer, CHUNK_OFFSET, "%06x\r", (unsigned)bytes); else ne_snprintf(buffer, CHUNK_OFFSET, "\r\n%04x\r", (unsigned)bytes); buffer[CHUNK_OFFSET - 1] = '\n'; bytes += CHUNK_OFFSET; } ret = ne_sock_fullwrite(sess->socket, buffer, bytes); if (ret < 0) { int aret = aborted(req, _("Could not send request body"), ret); return RETRY_RET(retry, ret, aret); } NE_DEBUG(NE_DBG_HTTPBODY, "Body block (%" NE_FMT_SSIZE_T " bytes):\n[%.*s]\n", bytes, (int)bytes, buffer); /* invoke progress callback */ notify_status(sess, ne_status_sending); } if (bytes) { NE_DEBUG(NE_DBG_HTTP, "Request body provider failed with " "%" NE_FMT_SSIZE_T "\n", bytes); ne_close_connection(sess); return NE_ERROR; } if (chunked) { if (chunknum == 0) ret = ne_sock_fullwrite(sess->socket, CHUNK_NULL_TERM, sizeof(CHUNK_NULL_TERM) - 1); else ret = ne_sock_fullwrite(sess->socket, CHUNK_TERM, sizeof(CHUNK_TERM) - 1); if (ret < 0) { int aret = aborted(req, _("Could not send chunked " "request terminator"), ret); return RETRY_RET(retry, ret, aret); } } return NE_OK; } /* Set up buffer for initial request headers. */ static ne_buffer *initial_request_headers(ne_request *req) { ne_session *const sess = req->session; ne_buffer *hdrs = ne_buffer_create(); if (sess->user_agent) { ne_buffer_zappend(hdrs, sess->user_agent); } /* If persistent connections are disabled, just send Connection: * close; otherwise, send Connection: Keep-Alive to pre-1.1 origin * servers to try harder to get a persistent connection, except if * using a proxy as per 2068§19.7.1. Always add TE: trailers. */ if (!sess->flags[NE_SESSFLAG_PERSIST]) { ne_buffer_czappend(hdrs, "Connection: TE, close" EOL); } else if (!sess->is_http11 && !sess->any_proxy_http) { ne_buffer_czappend(hdrs, "Keep-Alive: " EOL "Connection: TE, Keep-Alive" EOL); } else if (!req->session->is_http11 && !sess->any_proxy_http) { ne_buffer_czappend(hdrs, "Keep-Alive: " EOL "Proxy-Connection: Keep-Alive" EOL "Connection: TE" EOL); } else { ne_buffer_czappend(hdrs, "Connection: TE" EOL); } ne_buffer_concat(hdrs, "TE: trailers" EOL "Host: ", req->session->server.hostport, EOL, NULL); return hdrs; } int ne_accept_always(void *userdata, ne_request *req, const ne_status *st) { return 1; } int ne_accept_2xx(void *userdata, ne_request *req, const ne_status *st) { return (st->klass == 2); } ne_request *ne_request_create(ne_session *sess, const char *method, const char *path) { ne_request *req = ne_calloc(sizeof *req); req->session = sess; /* Presume the method is idempotent by default. */ req->flags[NE_REQFLAG_IDEMPOTENT] = 1; /* Expect-100 default follows the corresponding session flag. */ req->flags[NE_REQFLAG_EXPECT100] = sess->flags[NE_SESSFLAG_EXPECT100]; /* Add in the fixed headers */ req->headers = initial_request_headers(req); /* Set the standard stuff */ req->method = ne_strdup(method); req->method_is_head = (strcmp(method, "HEAD") == 0); /* Only use an absoluteURI here when we might be using an HTTP * proxy, and SSL is in use: some servers can't parse them. */ if (sess->any_proxy_http && !req->session->use_ssl && path[0] == '/') req->uri = ne_concat(req->session->scheme, "://", req->session->server.hostport, path, NULL); else req->uri = ne_strdup(path); { struct hook *hk; for (hk = sess->create_req_hooks; hk != NULL; hk = hk->next) { ne_create_request_fn fn = (ne_create_request_fn)hk->fn; fn(req, hk->userdata, req->method, req->uri); } } return req; } /* Set the request body length to 'length' */ static void set_body_length(ne_request *req, ne_off_t length) { req->body_length = length; if (length >= 0) ne_print_request_header(req, "Content-Length", "%" FMT_NE_OFF_T, length); else /* length < 0 => chunked body */ ne_add_request_header(req, "Transfer-Encoding", "chunked"); } void ne_set_request_body_buffer(ne_request *req, const char *buffer, size_t size) { req->body.buf.buffer = buffer; req->body.buf.length = size; req->body_cb = body_string_send; req->body_ud = req; set_body_length(req, size); } void ne_set_request_body_provider(ne_request *req, ne_off_t bodysize, ne_provide_body provider, void *ud) { req->body_cb = provider; req->body_ud = ud; set_body_length(req, bodysize); } void ne_set_request_body_fd(ne_request *req, int fd, ne_off_t offset, ne_off_t length) { req->body.file.fd = fd; req->body.file.offset = offset; req->body.file.length = length; req->body_cb = body_fd_send; req->body_ud = req; set_body_length(req, length); } void ne_set_request_flag(ne_request *req, ne_request_flag flag, int value) { if (flag < (ne_request_flag)NE_REQFLAG_LAST) { req->flags[flag] = value; } } int ne_get_request_flag(ne_request *req, ne_request_flag flag) { if (flag < (ne_request_flag)NE_REQFLAG_LAST) { return req->flags[flag]; } return -1; } void ne_add_request_header(ne_request *req, const char *name, const char *value) { ne_buffer_concat(req->headers, name, ": ", value, EOL, NULL); } void ne_print_request_header(ne_request *req, const char *name, const char *format, ...) { va_list params; char buf[NE_BUFSIZ]; va_start(params, format); ne_vsnprintf(buf, sizeof buf, format, params); va_end(params); ne_buffer_concat(req->headers, name, ": ", buf, EOL, NULL); } /* Returns the value of the response header 'name', for which the hash * value is 'h', or NULL if the header is not found. */ static inline char *get_response_header_hv(ne_request *req, unsigned int h, const char *name) { struct field *f; for (f = req->response_headers[h]; f; f = f->next) if (strcmp(f->name, name) == 0) return f->value; return NULL; } const char *ne_get_response_header(ne_request *req, const char *name) { char *lcname = ne_strdup(name); unsigned int hash = hash_and_lower(lcname); char *value = get_response_header_hv(req, hash, lcname); ne_free(lcname); return value; } /* The return value of the iterator function is a pointer to the * struct field of the previously returned header. */ void *ne_response_header_iterate(ne_request *req, void *iterator, const char **name, const char **value) { struct field *f = iterator; unsigned int n; if (f == NULL) { n = 0; } else if ((f = f->next) == NULL) { n = req->current_index + 1; } if (f == NULL) { while (n < HH_HASHSIZE && req->response_headers[n] == NULL) n++; if (n == HH_HASHSIZE) return NULL; /* no more headers */ f = req->response_headers[n]; req->current_index = n; } *name = f->name; *value = f->value; return f; } /* Removes the response header 'name', which has hash value 'hash'. */ static void remove_response_header(ne_request *req, const char *name, unsigned int hash) { struct field **ptr = req->response_headers + hash; while (*ptr) { struct field *const f = *ptr; if (strcmp(f->name, name) == 0) { *ptr = f->next; ne_free(f->name); ne_free(f->value); ne_free(f); return; } ptr = &f->next; } } /* Free all stored response headers. */ static void free_response_headers(ne_request *req) { int n; for (n = 0; n < HH_HASHSIZE; n++) { struct field **ptr = req->response_headers + n; while (*ptr) { struct field *const f = *ptr; *ptr = f->next; ne_free(f->name); ne_free(f->value); ne_free(f); } } } void ne_add_response_body_reader(ne_request *req, ne_accept_response acpt, ne_block_reader rdr, void *userdata) { struct body_reader *new = ne_malloc(sizeof *new); new->accept_response = acpt; new->handler = rdr; new->userdata = userdata; new->next = req->body_readers; req->body_readers = new; } void ne_request_destroy(ne_request *req) { struct body_reader *rdr, *next_rdr; struct hook *hk, *next_hk; ne_free(req->uri); ne_free(req->method); for (rdr = req->body_readers; rdr != NULL; rdr = next_rdr) { next_rdr = rdr->next; ne_free(rdr); } free_response_headers(req); ne_buffer_destroy(req->headers); NE_DEBUG(NE_DBG_HTTP, "Running destroy hooks.\n"); for (hk = req->session->destroy_req_hooks; hk; hk = next_hk) { ne_destroy_req_fn fn = (ne_destroy_req_fn)hk->fn; next_hk = hk->next; fn(req, hk->userdata); } for (hk = req->private; hk; hk = next_hk) { next_hk = hk->next; ne_free(hk); } if (req->status.reason_phrase) ne_free(req->status.reason_phrase); NE_DEBUG(NE_DBG_HTTP, "Request ends.\n"); ne_free(req); } /* Reads a block of the response into BUFFER, which is of size * *BUFLEN. Returns zero on success or non-zero on error. On * success, *BUFLEN is updated to be the number of bytes read into * BUFFER (which will be 0 to indicate the end of the repsonse). On * error, the connection is closed and the session error string is * set. */ static int read_response_block(ne_request *req, struct ne_response *resp, char *buffer, size_t *buflen) { ne_socket *const sock = req->session->socket; size_t willread; ssize_t readlen; switch (resp->mode) { case R_CHUNKED: /* Chunked transfer-encoding: chunk syntax is "SIZE CRLF CHUNK * CRLF SIZE CRLF CHUNK CRLF ..." followed by zero-length * chunk: "CHUNK CRLF 0 CRLF". resp.chunk.remain contains the * number of bytes left to read in the current chunk. */ if (resp->body.chunk.remain == 0) { unsigned long chunk_len; char *ptr; /* Read the chunk size line into a temporary buffer. */ SOCK_ERR(req, ne_sock_readline(sock, req->respbuf, sizeof req->respbuf), _("Could not read chunk size")); NE_DEBUG(NE_DBG_HTTP, "[chunk] < %s", req->respbuf); chunk_len = strtoul(req->respbuf, &ptr, 16); /* limit chunk size to <= UINT_MAX, so it will probably * fit in a size_t. */ if (ptr == req->respbuf || chunk_len == ULONG_MAX || chunk_len > UINT_MAX) { return aborted(req, _("Could not parse chunk size"), 0); } NE_DEBUG(NE_DBG_HTTP, "Got chunk size: %lu\n", chunk_len); resp->body.chunk.remain = chunk_len; } willread = resp->body.chunk.remain > *buflen ? *buflen : resp->body.chunk.remain; break; case R_CLENGTH: willread = resp->body.clen.remain > (off_t)*buflen ? *buflen : (size_t)resp->body.clen.remain; break; case R_TILLEOF: willread = *buflen; break; case R_NO_BODY: default: willread = 0; break; } if (willread == 0) { *buflen = 0; return 0; } NE_DEBUG(NE_DBG_HTTP, "Reading %" NE_FMT_SIZE_T " bytes of response body.\n", willread); readlen = ne_sock_read(sock, buffer, willread); /* EOF is only valid when response body is delimited by it. * Strictly, an SSL truncation should not be treated as an EOF in * any case, but SSL servers are just too buggy. */ if (resp->mode == R_TILLEOF && (readlen == NE_SOCK_CLOSED || readlen == NE_SOCK_TRUNC)) { NE_DEBUG(NE_DBG_HTTP, "Got EOF.\n"); req->can_persist = 0; readlen = 0; } else if (readlen < 0) { return aborted(req, _("Could not read response body"), readlen); } else { NE_DEBUG(NE_DBG_HTTP, "Got %" NE_FMT_SSIZE_T " bytes.\n", readlen); } /* safe to cast: readlen guaranteed to be >= 0 above */ *buflen = (size_t)readlen; NE_DEBUG(NE_DBG_HTTPBODY, "Read block (%" NE_FMT_SSIZE_T " bytes):\n[%.*s]\n", readlen, (int)readlen, buffer); if (resp->mode == R_CHUNKED) { resp->body.chunk.remain -= readlen; if (resp->body.chunk.remain == 0) { char crlfbuf[2]; /* If we've read a whole chunk, read a CRLF */ readlen = ne_sock_fullread(sock, crlfbuf, 2); if (readlen < 0) return aborted(req, _("Could not read chunk delimiter"), readlen); else if (crlfbuf[0] != '\r' || crlfbuf[1] != '\n') return aborted(req, _("Chunk delimiter was invalid"), 0); } } else if (resp->mode == R_CLENGTH) { resp->body.clen.remain -= readlen; } resp->progress += readlen; return NE_OK; } ssize_t ne_read_response_block(ne_request *req, char *buffer, size_t buflen) { struct body_reader *rdr; size_t readlen = buflen; struct ne_response *const resp = &req->resp; if (read_response_block(req, resp, buffer, &readlen)) return -1; if (readlen) { req->session->status.sr.progress += readlen; notify_status(req->session, ne_status_recving); } for (rdr = req->body_readers; rdr!=NULL; rdr=rdr->next) { if (rdr->use && rdr->handler(rdr->userdata, buffer, readlen) != 0) { ne_close_connection(req->session); return -1; } } return readlen; } /* Build the request string, returning the buffer. */ static ne_buffer *build_request(ne_request *req) { struct hook *hk; ne_buffer *buf = ne_buffer_create(); /* Add Request-Line and headers: */ ne_buffer_concat(buf, req->method, " ", req->uri, " HTTP/1.1" EOL, NULL); /* Add custom headers: */ ne_buffer_append(buf, req->headers->data, ne_buffer_size(req->headers)); if (req->body_length && req->flags[NE_REQFLAG_EXPECT100]) { ne_buffer_czappend(buf, "Expect: 100-continue\r\n"); } NE_DEBUG(NE_DBG_HTTP, "Running pre_send hooks\n"); for (hk = req->session->pre_send_hooks; hk!=NULL; hk = hk->next) { ne_pre_send_fn fn = (ne_pre_send_fn)hk->fn; fn(req, hk->userdata, buf); } ne_buffer_czappend(buf, "\r\n"); return buf; } #ifdef NE_DEBUGGING #define DEBUG_DUMP_REQUEST(x) dump_request(x) static void dump_request(const char *request) { if (ne_debug_mask & NE_DBG_HTTPPLAIN) { /* Display everything mode */ NE_DEBUG(NE_DBG_HTTP, "Sending request headers:\n%s", request); } else if (ne_debug_mask & NE_DBG_HTTP) { /* Blank out the Authorization paramaters */ char *reqdebug = ne_strdup(request), *pnt = reqdebug; while ((pnt = strstr(pnt, "Authorization: ")) != NULL) { for (pnt += 15; *pnt != '\r' && *pnt != '\0'; pnt++) { *pnt = 'x'; } } NE_DEBUG(NE_DBG_HTTP, "Sending request headers:\n%s", reqdebug); ne_free(reqdebug); } } #else #define DEBUG_DUMP_REQUEST(x) #endif /* DEBUGGING */ /* remove trailing EOL from 'buf', where strlen(buf) == *len. *len is * adjusted in accordance with any changes made to the string to * remain equal to strlen(buf). */ static inline void strip_eol(char *buf, ssize_t *len) { char *pnt = buf + *len - 1; while (pnt >= buf && (*pnt == '\r' || *pnt == '\n')) { *pnt-- = '\0'; (*len)--; } } /* Read and parse response status-line into 'status'. 'retry' is non-zero * if an NE_RETRY should be returned if an EOF is received. */ static int read_status_line(ne_request *req, ne_status *status, int retry) { char *buffer = req->respbuf; ssize_t ret; ret = ne_sock_readline(req->session->socket, buffer, sizeof req->respbuf); if (ret <= 0) { int aret = aborted(req, _("Could not read status line"), ret); return RETRY_RET(retry, ret, aret); } NE_DEBUG(NE_DBG_HTTP, "[status-line] < %s", buffer); strip_eol(buffer, &ret); if (status->reason_phrase) ne_free(status->reason_phrase); memset(status, 0, sizeof *status); /* Hack to allow ShoutCast-style servers, if requested. */ if (req->session->flags[NE_SESSFLAG_ICYPROTO] && strncmp(buffer, "ICY ", 4) == 0 && strlen(buffer) > 8 && buffer[7] == ' ') { status->code = atoi(buffer + 4); status->major_version = 1; status->minor_version = 0; status->reason_phrase = ne_strclean(ne_strdup(buffer + 8)); status->klass = buffer[4] - '0'; NE_DEBUG(NE_DBG_HTTP, "[status-line] ICY protocol; code %d\n", status->code); } else if (ne_parse_statusline(buffer, status)) { return aborted(req, _("Could not parse response status line"), 0); } return 0; } /* Discard a set of message headers. */ static int discard_headers(ne_request *req) { do { SOCK_ERR(req, ne_sock_readline(req->session->socket, req->respbuf, sizeof req->respbuf), _("Could not read interim response headers")); NE_DEBUG(NE_DBG_HTTP, "[discard] < %s", req->respbuf); } while (strcmp(req->respbuf, EOL) != 0); return NE_OK; } /* Send the request, and read the response Status-Line. Returns: * NE_RETRY connection closed by server; persistent connection * timeout * NE_OK success * NE_* error * On NE_RETRY and NE_* responses, the connection will have been * closed already. */ static int send_request(ne_request *req, const ne_buffer *request) { ne_session *const sess = req->session; ne_status *const status = &req->status; int sentbody = 0; /* zero until body has been sent. */ int ret, retry; /* retry non-zero whilst the request should be retried */ ssize_t sret; /* Send the Request-Line and headers */ NE_DEBUG(NE_DBG_HTTP, "Sending request-line and headers:\n"); /* Open the connection if necessary */ ret = open_connection(sess); if (ret) return ret; /* Allow retry if a persistent connection has been used. */ retry = sess->persisted; sret = ne_sock_fullwrite(req->session->socket, request->data, ne_buffer_size(request)); if (sret < 0) { int aret = aborted(req, _("Could not send request"), sret); return RETRY_RET(retry, sret, aret); } if (!req->flags[NE_REQFLAG_EXPECT100] && req->body_length) { /* Send request body, if not using 100-continue. */ ret = send_request_body(req, retry); if (ret) { return ret; } } NE_DEBUG(NE_DBG_HTTP, "Request sent; retry is %d.\n", retry); /* Loop eating interim 1xx responses (RFC2616 says these MAY be * sent by the server, even if 100-continue is not used). */ while ((ret = read_status_line(req, status, retry)) == NE_OK && status->klass == 1) { NE_DEBUG(NE_DBG_HTTP, "Interim %d response.\n", status->code); retry = 0; /* successful read() => never retry now. */ /* Discard headers with the interim response. */ if ((ret = discard_headers(req)) != NE_OK) break; if (req->flags[NE_REQFLAG_EXPECT100] && (status->code == 100) && req->body_length && !sentbody) { /* Send the body after receiving the first 100 Continue */ if ((ret = send_request_body(req, 0)) != NE_OK) break; sentbody = 1; } } return ret; } /* Read a message header from sock into buf, which has size 'buflen'. * * Returns: * NE_RETRY: Success, read a header into buf. * NE_OK: End of headers reached. * NE_ERROR: Error (session error is set, connection closed). */ static int read_message_header(ne_request *req, char *buf, size_t buflen) { ssize_t n; ne_socket *sock = req->session->socket; n = ne_sock_readline(sock, buf, buflen); if (n <= 0) return aborted(req, _("Error reading response headers"), n); NE_DEBUG(NE_DBG_HTTP, "[hdr] %s", buf); strip_eol(buf, &n); if (n == 0) { NE_DEBUG(NE_DBG_HTTP, "End of headers.\n"); return NE_OK; } buf += n; buflen -= n; while (buflen > 0) { char ch; /* Collect any extra lines into buffer */ SOCK_ERR(req, ne_sock_peek(sock, &ch, 1), _("Error reading response headers")); if (ch != ' ' && ch != '\t') { /* No continuation of this header: stop reading. */ return NE_RETRY; } /* Otherwise, read the next line onto the end of 'buf'. */ n = ne_sock_readline(sock, buf, buflen); if (n <= 0) { return aborted(req, _("Error reading response headers"), n); } NE_DEBUG(NE_DBG_HTTP, "[cont] %s", buf); strip_eol(buf, &n); /* assert(buf[0] == ch), which implies len(buf) > 0. * Otherwise the TCP stack is lying, but we'll be paranoid. * This might be a \t, so replace it with a space to be * friendly to applications (2616 says we MAY do this). */ if (n) buf[0] = ' '; /* ready for the next header. */ buf += n; buflen -= n; } ne_set_error(req->session, _("Response header too long")); return NE_ERROR; } #define MAX_HEADER_LEN (8192) /* Add a respnose header field for the given request, using * precalculated hash value. */ static void add_response_header(ne_request *req, unsigned int hash, char *name, char *value) { struct field **nextf = &req->response_headers[hash]; size_t vlen = strlen(value); while (*nextf) { struct field *const f = *nextf; if (strcmp(f->name, name) == 0) { if (vlen + f->vlen < MAX_HEADER_LEN) { /* merge the header field */ f->value = ne_realloc(f->value, f->vlen + vlen + 3); memcpy(f->value + f->vlen, ", ", 2); memcpy(f->value + f->vlen + 2, value, vlen + 1); f->vlen += vlen + 2; } return; } nextf = &f->next; } (*nextf) = ne_malloc(sizeof **nextf); (*nextf)->name = ne_strdup(name); (*nextf)->value = ne_strdup(value); (*nextf)->vlen = vlen; (*nextf)->next = NULL; } /* Read response headers. Returns NE_* code, sets session error and * closes connection on error. */ static int read_response_headers(ne_request *req) { char hdr[MAX_HEADER_LEN]; int ret, count = 0; while ((ret = read_message_header(req, hdr, sizeof hdr)) == NE_RETRY && ++count < MAX_HEADER_FIELDS) { char *pnt; unsigned int hash = 0; /* Strip any trailing whitespace */ pnt = hdr + strlen(hdr) - 1; while (pnt > hdr && (*pnt == ' ' || *pnt == '\t')) *pnt-- = '\0'; /* Convert the header name to lower case and hash it. */ for (pnt = hdr; (*pnt != '\0' && *pnt != ':' && *pnt != ' ' && *pnt != '\t'); pnt++) { *pnt = ne_tolower(*pnt); hash = HH_ITERATE(hash,*pnt); } /* Skip over any whitespace before the colon. */ while (*pnt == ' ' || *pnt == '\t') *pnt++ = '\0'; /* ignore header lines which lack a ':'. */ if (*pnt != ':') continue; /* NUL-terminate at the colon (when no whitespace before) */ *pnt++ = '\0'; /* Skip any whitespace after the colon... */ while (*pnt == ' ' || *pnt == '\t') pnt++; /* pnt now points to the header value. */ NE_DEBUG(NE_DBG_HTTP, "Header Name: [%s], Value: [%s]\n", hdr, pnt); add_response_header(req, hash, hdr, pnt); } if (count == MAX_HEADER_FIELDS) ret = aborted( req, _("Response exceeded maximum number of header fields"), 0); return ret; } /* Perform any necessary DNS lookup for the host given by *info; * returns NE_ code with error string set on error. */ static int lookup_host(ne_session *sess, struct host_info *info) { NE_DEBUG(NE_DBG_HTTP, "Doing DNS lookup on %s...\n", info->hostname); sess->status.lu.hostname = info->hostname; notify_status(sess, ne_status_lookup); info->address = ne_addr_resolve(info->hostname, 0); if (ne_addr_result(info->address)) { char buf[256]; ne_set_error(sess, _("Could not resolve hostname `%s': %s"), info->hostname, ne_addr_error(info->address, buf, sizeof buf)); ne_addr_destroy(info->address); info->address = NULL; return NE_LOOKUP; } else { return NE_OK; } } int ne_begin_request(ne_request *req) { struct body_reader *rdr; ne_buffer *data; const ne_status *const st = &req->status; const char *value; struct hook *hk; int ret, forced_closure = 0; /* If a non-idempotent request is sent on a persisted connection, * then it is impossible to distinguish between a server failure * and a connection timeout if an EOF/RST is received. So don't * do that. */ if (!req->flags[NE_REQFLAG_IDEMPOTENT] && req->session->persisted && !req->session->flags[NE_SESSFLAG_CONNAUTH]) { NE_DEBUG(NE_DBG_HTTP, "req: Closing connection for non-idempotent " "request.\n"); ne_close_connection(req->session); } /* Build the request string, and send it */ data = build_request(req); DEBUG_DUMP_REQUEST(data->data); ret = send_request(req, data); /* Retry this once after a persistent connection timeout. */ if (ret == NE_RETRY) { NE_DEBUG(NE_DBG_HTTP, "Persistent connection timed out, retrying.\n"); ret = send_request(req, data); } ne_buffer_destroy(data); if (ret != NE_OK) return ret == NE_RETRY ? NE_ERROR : ret; /* Determine whether server claims HTTP/1.1 compliance. */ req->session->is_http11 = (st->major_version == 1 && st->minor_version > 0) || st->major_version > 1; /* Persistent connections supported implicitly in HTTP/1.1 */ if (req->session->is_http11) req->can_persist = 1; ne_set_error(req->session, "%d %s", st->code, st->reason_phrase); /* Empty the response header hash, in case this request was * retried: */ free_response_headers(req); /* Read the headers */ ret = read_response_headers(req); if (ret) return ret; /* check the Connection header */ value = get_response_header_hv(req, HH_HV_CONNECTION, "connection"); if (value) { char *vcopy = ne_strdup(value), *ptr = vcopy; do { char *token = ne_shave(ne_token(&ptr, ','), " \t"); unsigned int hash = hash_and_lower(token); if (strcmp(token, "close") == 0) { req->can_persist = 0; forced_closure = 1; } else if (strcmp(token, "keep-alive") == 0) { req->can_persist = 1; } else if (!req->session->is_http11 && strcmp(token, "connection")) { /* Strip the header per 2616§14.10, last para. Avoid * danger from "Connection: connection". */ remove_response_header(req, token, hash); } } while (ptr); ne_free(vcopy); } /* Support "Proxy-Connection: keep-alive" for compatibility with * some HTTP/1.0 proxies; it is risky to do this, because an * intermediary proxy may not support this HTTP/1.0 extension, but * will not strip the header either. Persistent connection * support is enabled based on the presence of this header if: * a) it is *necessary* to do so due to the use of a connection-auth * scheme, and * b) connection closure was not forced via "Connection: close". */ if (req->session->nexthop->proxy == PROXY_HTTP && !req->session->is_http11 && !forced_closure && req->session->flags[NE_SESSFLAG_CONNAUTH]) { value = get_response_header_hv(req, HH_HV_PROXY_CONNECTION, "proxy-connection"); if (value && ne_strcasecmp(value, "keep-alive") == 0) { NE_DEBUG(NE_DBG_HTTP, "req: Using persistent connection " "for HTTP/1.0 proxy requiring conn-auth hack.\n"); req->can_persist = 1; } } /* Decide which method determines the response message-length per * 2616§4.4 (multipart/byteranges is not supported): */ #ifdef NE_HAVE_SSL /* Special case for CONNECT handling: the response has no body, * and the connection can persist. */ if (req->session->in_connect && st->klass == 2) { req->resp.mode = R_NO_BODY; req->can_persist = 1; } else #endif /* HEAD requests and 204, 304 responses have no response body, * regardless of what headers are present. */ if (req->method_is_head || st->code == 204 || st->code == 304) { req->resp.mode = R_NO_BODY; } /* Broken intermediaries exist which use "transfer-encoding: identity" * to mean "no transfer-coding". So that case must be ignored. */ else if ((value = get_response_header_hv(req, HH_HV_TRANSFER_ENCODING, "transfer-encoding")) != NULL && ne_strcasecmp(value, "identity") != 0) { /* Otherwise, fail iff an unknown transfer-coding is used. */ if (ne_strcasecmp(value, "chunked") == 0) { req->resp.mode = R_CHUNKED; req->resp.body.chunk.remain = 0; } else { return aborted(req, _("Unknown transfer-coding in response"), 0); } } else if ((value = get_response_header_hv(req, HH_HV_CONTENT_LENGTH, "content-length")) != NULL) { char *endptr = NULL; ne_off_t len = ne_strtoff(value, &endptr, 10); if (*value && len != NE_OFFT_MAX && len >= 0 && endptr && *endptr == '\0') { req->resp.mode = R_CLENGTH; req->resp.body.clen.total = req->resp.body.clen.remain = len; } else { /* fail for an invalid content-length header. */ return aborted(req, _("Invalid Content-Length in response"), 0); } } else { req->resp.mode = R_TILLEOF; /* otherwise: read-till-eof mode */ } NE_DEBUG(NE_DBG_HTTP, "Running post_headers hooks\n"); for (hk = req->session->post_headers_hooks; hk != NULL; hk = hk->next) { ne_post_headers_fn fn = (ne_post_headers_fn)hk->fn; fn(req, hk->userdata, &req->status); } /* Prepare for reading the response entity-body. Call each of the * body readers and ask them whether they want to accept this * response or not. */ for (rdr = req->body_readers; rdr != NULL; rdr=rdr->next) { rdr->use = rdr->accept_response(rdr->userdata, req, st); } req->session->status.sr.progress = 0; req->session->status.sr.total = req->resp.mode == R_CLENGTH ? req->resp.body.clen.total : -1; notify_status(req->session, ne_status_recving); return NE_OK; } int ne_end_request(ne_request *req) { struct hook *hk; int ret; /* Read headers in chunked trailers */ if (req->resp.mode == R_CHUNKED) { ret = read_response_headers(req); if (ret) return ret; } else { ret = NE_OK; } NE_DEBUG(NE_DBG_HTTP, "Running post_send hooks\n"); for (hk = req->session->post_send_hooks; ret == NE_OK && hk != NULL; hk = hk->next) { ne_post_send_fn fn = (ne_post_send_fn)hk->fn; ret = fn(req, hk->userdata, &req->status); } /* Close the connection if persistent connections are disabled or * not supported by the server. */ if (!req->session->flags[NE_SESSFLAG_PERSIST] || !req->can_persist) ne_close_connection(req->session); else req->session->persisted = 1; return ret; } int ne_read_response_to_fd(ne_request *req, int fd) { ssize_t len; while ((len = ne_read_response_block(req, req->respbuf, sizeof req->respbuf)) > 0) { const char *block = req->respbuf; do { ssize_t ret = write(fd, block, len); if (ret == -1 && errno == EINTR) { continue; } else if (ret < 0) { char err[200]; ne_strerror(errno, err, sizeof err); ne_set_error(ne_get_session(req), _("Could not write to file: %s"), err); return NE_ERROR; } else { len -= ret; block += ret; } } while (len > 0); } return len == 0 ? NE_OK : NE_ERROR; } int ne_discard_response(ne_request *req) { ssize_t len; do { len = ne_read_response_block(req, req->respbuf, sizeof req->respbuf); } while (len > 0); return len == 0 ? NE_OK : NE_ERROR; } int ne_request_dispatch(ne_request *req) { int ret; do { ret = ne_begin_request(req); if (ret == NE_OK) ret = ne_discard_response(req); if (ret == NE_OK) ret = ne_end_request(req); } while (ret == NE_RETRY); NE_DEBUG(NE_DBG_HTTP | NE_DBG_FLUSH, "Request ends, status %d class %dxx, error line:\n%s\n", req->status.code, req->status.klass, req->session->error); return ret; } const ne_status *ne_get_status(const ne_request *req) { return &req->status; } ne_session *ne_get_session(const ne_request *req) { return req->session; } #ifdef NE_HAVE_SSL /* Create a CONNECT tunnel through the proxy server. * Returns HTTP_* */ static int proxy_tunnel(ne_session *sess) { /* Hack up an HTTP CONNECT request... */ ne_request *req; int ret = NE_OK; char ruri[200]; /* Can't use server.hostport here; Request-URI must include `:port' */ ne_snprintf(ruri, sizeof ruri, "%s:%u", sess->server.hostname, sess->server.port); req = ne_request_create(sess, "CONNECT", ruri); sess->in_connect = 1; ret = ne_request_dispatch(req); sess->in_connect = 0; sess->persisted = 0; /* don't treat this is a persistent connection. */ if (ret != NE_OK || !sess->connected || req->status.klass != 2) { char *err = ne_strdup(sess->error); ne_set_error(sess, _("Could not create SSL connection " "through proxy server: %s"), err); ne_free(err); if (ret == NE_OK) ret = NE_ERROR; } ne_request_destroy(req); return ret; } #endif /* Return the first resolved address for the given host. */ static const ne_inet_addr *resolve_first(struct host_info *host) { return host->network ? host->network : ne_addr_first(host->address); } /* Return the next resolved address for the given host or NULL if * there are no more addresses. */ static const ne_inet_addr *resolve_next(struct host_info *host) { return host->network ? NULL : ne_addr_next(host->address); } /* Make new TCP connection to server at 'host' of type 'name'. Note * that once a connection to a particular network address has * succeeded, that address will be used first for the next attempt to * connect. */ static int do_connect(ne_session *sess, struct host_info *host) { int ret; /* Resolve hostname if necessary. */ if (host->address == NULL && host->network == NULL) { ret = lookup_host(sess, host); if (ret) return ret; } if ((sess->socket = ne_sock_create()) == NULL) { ne_set_error(sess, _("Could not create socket")); return NE_ERROR; } if (sess->cotimeout) ne_sock_connect_timeout(sess->socket, sess->cotimeout); if (sess->local_addr) ne_sock_prebind(sess->socket, sess->local_addr, 0); if (host->current == NULL) host->current = resolve_first(host); sess->status.ci.hostname = host->hostname; do { sess->status.ci.address = host->current; notify_status(sess, ne_status_connecting); #ifdef NE_DEBUGGING if (ne_debug_mask & NE_DBG_HTTP) { char buf[150]; NE_DEBUG(NE_DBG_HTTP, "req: Connecting to %s:%u\n", ne_iaddr_print(host->current, buf, sizeof buf), host->port); } #endif ret = ne_sock_connect(sess->socket, host->current, host->port); } while (ret && /* try the next address... */ (host->current = resolve_next(host)) != NULL); if (ret) { const char *msg; if (host->proxy == PROXY_NONE) msg = _("Could not connect to server"); else msg = _("Could not connect to proxy server"); ne_set_error(sess, "%s: %s", msg, ne_sock_error(sess->socket)); ne_sock_close(sess->socket); return ret == NE_SOCK_TIMEOUT ? NE_TIMEOUT : NE_CONNECT; } if (sess->rdtimeout) ne_sock_read_timeout(sess->socket, sess->rdtimeout); notify_status(sess, ne_status_connected); sess->nexthop = host; sess->connected = 1; /* clear persistent connection flag. */ sess->persisted = 0; return NE_OK; } /* For a SOCKSv4 proxy only, the IP address of the origin server (in * addition to the proxy) must be known, and must be an IPv4 address. * Returns NE_*; connection closed and error string set on error. */ static int socks_origin_lookup(ne_session *sess) { const ne_inet_addr *ia; int ret; ret = lookup_host(sess, &sess->server); if (ret) { /* lookup_host already set the error string. */ ne_close_connection(sess); return ret; } /* Find the first IPv4 address available for the server. */ for (ia = ne_addr_first(sess->server.address); ia && ne_iaddr_typeof(ia) == ne_iaddr_ipv6; ia = ne_addr_next(sess->server.address)) { /* noop */ } /* ... if any */ if (ia == NULL) { ne_set_error(sess, _("Could not find IPv4 address of " "hostname %s for SOCKS v4 proxy"), sess->server.hostname); ne_close_connection(sess); return NE_LOOKUP; } sess->server.current = ia; return ret; } static int open_connection(ne_session *sess) { int ret; if (sess->connected) return NE_OK; if (!sess->proxies) { ret = do_connect(sess, &sess->server); if (ret) { sess->nexthop = NULL; return ret; } } else { struct host_info *hi; /* Attempt to re-use proxy to avoid iterating through * unnecessarily. */ if (sess->prev_proxy) ret = do_connect(sess, sess->prev_proxy); else ret = NE_ERROR; /* Otherwise, try everything - but omitting prev_proxy if that * has already been tried. */ for (hi = sess->proxies; hi && ret; hi = hi->next) { if (hi != sess->prev_proxy) ret = do_connect(sess, hi); } if (ret == NE_OK && sess->nexthop->proxy == PROXY_SOCKS) { /* Special-case for SOCKS v4 proxies, which require the * client to resolve the origin server IP address. */ if (sess->socks_ver == NE_SOCK_SOCKSV4) { ret = socks_origin_lookup(sess); } if (ret == NE_OK) { /* Perform the SOCKS handshake, instructing the proxy * to set up the connection to the origin server. */ ret = ne_sock_proxy(sess->socket, sess->socks_ver, sess->server.current, sess->server.hostname, sess->server.port, sess->socks_user, sess->socks_password); if (ret) { ne_set_error(sess, _("Could not establish connection from " "SOCKS proxy (%s:%u): %s"), sess->nexthop->hostname, sess->nexthop->port, ne_sock_error(sess->socket)); ne_close_connection(sess); ret = NE_ERROR; } } } if (ret != NE_OK) { sess->nexthop = NULL; sess->prev_proxy = NULL; return ret; } /* Success - make this proxy stick. */ sess->prev_proxy = hi; } #ifdef NE_HAVE_SSL /* Negotiate SSL layer if required. */ if (sess->use_ssl && !sess->in_connect) { /* Set up CONNECT tunnel if using an HTTP proxy. */ if (sess->nexthop->proxy == PROXY_HTTP) ret = proxy_tunnel(sess); if (ret == NE_OK) { ret = ne__negotiate_ssl(sess); if (ret != NE_OK) ne_close_connection(sess); } } #endif return ret; }