diff options
Diffstat (limited to 'buffer.c')
| -rw-r--r-- | buffer.c | 947 |
1 files changed, 603 insertions, 344 deletions
@@ -26,16 +26,18 @@ */ #include "event2/event-config.h" +#include "evconfig-private.h" -#ifdef WIN32 +#ifdef _WIN32 #include <winsock2.h> #include <windows.h> #include <io.h> #endif #ifdef _EVENT_HAVE_VASPRINTF -/* If we have vasprintf, we need to define this before we include stdio.h. */ -#define _GNU_SOURCE +/* If we have vasprintf, we need to define _GNU_SOURCE before we include + * stdio.h. This comes from evconfig-private.h. + */ #endif #include <sys/types.h> @@ -63,6 +65,10 @@ #ifdef _EVENT_HAVE_SYS_SENDFILE_H #include <sys/sendfile.h> #endif +#ifdef _EVENT_HAVE_SYS_STAT_H +#include <sys/stat.h> +#endif + #include <errno.h> #include <stdio.h> @@ -83,7 +89,6 @@ #include "event2/bufferevent_compat.h" #include "event2/bufferevent_struct.h" #include "event2/thread.h" -#include "event2/event-config.h" #include "log-internal.h" #include "mm-internal.h" #include "util-internal.h" @@ -111,14 +116,6 @@ #define SENDFILE_IS_SOLARIS 1 #endif -#ifdef USE_SENDFILE -static int use_sendfile = 1; -#endif -#ifdef _EVENT_HAVE_MMAP -static int use_mmap = 1; -#endif - - /* Mask of user-selectable callback flags. */ #define EVBUFFER_CB_USER_FLAGS 0xffff /* Mask of all internal-use-only flags. */ @@ -135,6 +132,13 @@ static int use_mmap = 1; #define CHAIN_PINNED(ch) (((ch)->flags & EVBUFFER_MEM_PINNED_ANY) != 0) #define CHAIN_PINNED_R(ch) (((ch)->flags & EVBUFFER_MEM_PINNED_R) != 0) +/* evbuffer_ptr support */ +#define PTR_NOT_FOUND(ptr) do { \ + (ptr)->pos = -1; \ + (ptr)->_internal.chain = NULL; \ + (ptr)->_internal.pos_in_chain = 0; \ +} while (0) + static void evbuffer_chain_align(struct evbuffer_chain *chain); static int evbuffer_chain_should_realign(struct evbuffer_chain *chain, size_t datalen); @@ -143,13 +147,10 @@ static int evbuffer_ptr_memcmp(const struct evbuffer *buf, const struct evbuffer_ptr *pos, const char *mem, size_t len); static struct evbuffer_chain *evbuffer_expand_singlechain(struct evbuffer *buf, size_t datlen); - -#ifdef WIN32 -static int evbuffer_readfile(struct evbuffer *buf, evutil_socket_t fd, - ev_ssize_t howmuch); -#else -#define evbuffer_readfile evbuffer_read -#endif +static int evbuffer_ptr_subtract(struct evbuffer *buf, struct evbuffer_ptr *pos, + size_t howfar); +static int evbuffer_file_segment_materialize(struct evbuffer_file_segment *seg); +static inline void evbuffer_chain_incref(struct evbuffer_chain *chain); static struct evbuffer_chain * evbuffer_chain_new(size_t size) @@ -177,50 +178,66 @@ evbuffer_chain_new(size_t size) */ chain->buffer = EVBUFFER_CHAIN_EXTRA(u_char, chain); + chain->refcnt = 1; + return (chain); } static inline void evbuffer_chain_free(struct evbuffer_chain *chain) { + EVUTIL_ASSERT(chain->refcnt > 0); + if (--chain->refcnt > 0) { + /* chain is still referenced by other chains */ + return; + } + if (CHAIN_PINNED(chain)) { + /* will get freed once no longer dangling */ + chain->refcnt++; chain->flags |= EVBUFFER_DANGLING; return; } - if (chain->flags & (EVBUFFER_MMAP|EVBUFFER_SENDFILE| - EVBUFFER_REFERENCE)) { - if (chain->flags & EVBUFFER_REFERENCE) { - struct evbuffer_chain_reference *info = - EVBUFFER_CHAIN_EXTRA( - struct evbuffer_chain_reference, - chain); - if (info->cleanupfn) - (*info->cleanupfn)(chain->buffer, - chain->buffer_len, - info->extra); - } -#ifdef _EVENT_HAVE_MMAP - if (chain->flags & EVBUFFER_MMAP) { - struct evbuffer_chain_fd *info = - EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_fd, - chain); - if (munmap(chain->buffer, chain->buffer_len) == -1) - event_warn("%s: munmap failed", __func__); - if (close(info->fd) == -1) - event_warn("%s: close(%d) failed", - __func__, info->fd); - } + + /* safe to release chain, it's either a referencing + * chain or all references to it have been freed */ + if (chain->flags & EVBUFFER_REFERENCE) { + struct evbuffer_chain_reference *info = + EVBUFFER_CHAIN_EXTRA( + struct evbuffer_chain_reference, + chain); + if (info->cleanupfn) + (*info->cleanupfn)(chain->buffer, + chain->buffer_len, + info->extra); + } + if (chain->flags & EVBUFFER_FILESEGMENT) { + struct evbuffer_chain_file_segment *info = + EVBUFFER_CHAIN_EXTRA( + struct evbuffer_chain_file_segment, + chain); + if (info->segment) { +#ifdef _WIN32 + if (info->segment->is_mapping) + UnmapViewOfFile(chain->buffer); #endif -#ifdef USE_SENDFILE - if (chain->flags & EVBUFFER_SENDFILE) { - struct evbuffer_chain_fd *info = - EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_fd, - chain); - if (close(info->fd) == -1) - event_warn("%s: close(%d) failed", - __func__, info->fd); + evbuffer_file_segment_free(info->segment); } -#endif + } + if (chain->flags & EVBUFFER_MULTICAST) { + struct evbuffer_multicast_parent *info = + EVBUFFER_CHAIN_EXTRA( + struct evbuffer_multicast_parent, + chain); + /* referencing chain is being freed, decrease + * refcounts of source chain and associated + * evbuffer (which get freed once both reach + * zero) */ + EVUTIL_ASSERT(info->source != NULL); + EVUTIL_ASSERT(info->parent != NULL); + EVBUFFER_LOCK(info->source); + evbuffer_chain_free(info->parent); + _evbuffer_decref_and_unlock(info->source); } mm_free(chain); @@ -290,21 +307,11 @@ evbuffer_chain_insert(struct evbuffer *buf, EVUTIL_ASSERT(buf->first == NULL); buf->first = buf->last = chain; } else { - struct evbuffer_chain **ch = buf->last_with_datap; - /* Find the first victim chain. It might be *last_with_datap */ - while ((*ch) && ((*ch)->off != 0 || CHAIN_PINNED(*ch))) - ch = &(*ch)->next; - if (*ch == NULL) { - /* There is no victim; just append this new chain. */ - buf->last->next = chain; - if (chain->off) - buf->last_with_datap = &buf->last->next; - } else { - /* Replace all victim chains with this chain. */ - EVUTIL_ASSERT(evbuffer_chains_all_empty(*ch)); - evbuffer_free_all_chains(*ch); - *ch = chain; - } + struct evbuffer_chain **chp; + chp = evbuffer_free_trailing_empty_chains(buf); + *chp = chain; + if (chain->off) + buf->last_with_datap = chp; buf->last = chain; } buf->total_len += chain->off; @@ -336,6 +343,12 @@ _evbuffer_chain_unpin(struct evbuffer_chain *chain, unsigned flag) evbuffer_chain_free(chain); } +static inline void +evbuffer_chain_incref(struct evbuffer_chain *chain) +{ + ++chain->refcnt; +} + struct evbuffer * evbuffer_new(void) { @@ -345,7 +358,7 @@ evbuffer_new(void) if (buffer == NULL) return (NULL); - TAILQ_INIT(&buffer->callbacks); + LIST_INIT(&buffer->callbacks); buffer->refcnt = 1; buffer->last_with_datap = &buffer->first; @@ -454,7 +467,7 @@ evbuffer_run_callbacks(struct evbuffer *buffer, int running_deferred) ASSERT_EVBUFFER_LOCKED(buffer); - if (TAILQ_EMPTY(&buffer->callbacks)) { + if (LIST_EMPTY(&buffer->callbacks)) { buffer->n_add_for_cb = buffer->n_del_for_cb = 0; return; } @@ -469,12 +482,12 @@ evbuffer_run_callbacks(struct evbuffer *buffer, int running_deferred) buffer->n_add_for_cb = 0; buffer->n_del_for_cb = 0; } - for (cbent = TAILQ_FIRST(&buffer->callbacks); - cbent != TAILQ_END(&buffer->callbacks); + for (cbent = LIST_FIRST(&buffer->callbacks); + cbent != LIST_END(&buffer->callbacks); cbent = next) { /* Get the 'next' pointer now in case this callback decides * to remove itself or something. */ - next = TAILQ_NEXT(cbent, next); + next = LIST_NEXT(cbent, next); if ((cbent->flags & mask) != masked_val) continue; @@ -490,7 +503,7 @@ evbuffer_run_callbacks(struct evbuffer *buffer, int running_deferred) void evbuffer_invoke_callbacks(struct evbuffer *buffer) { - if (TAILQ_EMPTY(&buffer->callbacks)) { + if (LIST_EMPTY(&buffer->callbacks)) { buffer->n_add_for_cb = buffer->n_del_for_cb = 0; return; } @@ -529,9 +542,9 @@ evbuffer_remove_all_callbacks(struct evbuffer *buffer) { struct evbuffer_cb_entry *cbent; - while ((cbent = TAILQ_FIRST(&buffer->callbacks))) { - TAILQ_REMOVE(&buffer->callbacks, cbent, next); - mm_free(cbent); + while ((cbent = LIST_FIRST(&buffer->callbacks))) { + LIST_REMOVE(cbent, next); + mm_free(cbent); } } @@ -609,6 +622,42 @@ evbuffer_get_contiguous_space(const struct evbuffer *buf) return result; } +size_t +evbuffer_add_iovec(struct evbuffer * buf, struct evbuffer_iovec * vec, int n_vec) { + int n; + size_t res; + size_t to_alloc; + + EVBUFFER_LOCK(buf); + + res = to_alloc = 0; + + for (n = 0; n < n_vec; n++) { + to_alloc += vec[n].iov_len; + } + + if (_evbuffer_expand_fast(buf, to_alloc, 2) < 0) { + goto done; + } + + for (n = 0; n < n_vec; n++) { + /* XXX each 'add' call here does a bunch of setup that's + * obviated by _evbuffer_expand_fast, and some cleanup that we + * would like to do only once. Instead we should just extract + * the part of the code that's needed. */ + + if (evbuffer_add(buf, vec[n].iov_base, vec[n].iov_len) < 0) { + goto done; + } + + res += vec[n].iov_len; + } + +done: + EVBUFFER_UNLOCK(buf); + return res; +} + int evbuffer_reserve_space(struct evbuffer *buf, ev_ssize_t size, struct evbuffer_iovec *vec, int n_vecs) @@ -837,6 +886,46 @@ APPEND_CHAIN(struct evbuffer *dst, struct evbuffer *src) dst->total_len += src->total_len; } +static inline void +APPEND_CHAIN_MULTICAST(struct evbuffer *dst, struct evbuffer *src) +{ + struct evbuffer_chain *tmp; + struct evbuffer_chain *chain = src->first; + struct evbuffer_multicast_parent *extra; + + ASSERT_EVBUFFER_LOCKED(dst); + ASSERT_EVBUFFER_LOCKED(src); + + for (; chain; chain = chain->next) { + if (!chain->off || chain->flags & EVBUFFER_DANGLING) { + /* skip empty chains */ + continue; + } + + tmp = evbuffer_chain_new(sizeof(struct evbuffer_multicast_parent)); + if (!tmp) { + event_warn("%s: out of memory", __func__); + return; + } + extra = EVBUFFER_CHAIN_EXTRA(struct evbuffer_multicast_parent, tmp); + /* reference evbuffer containing source chain so it + * doesn't get released while the chain is still + * being referenced to */ + _evbuffer_incref(src); + extra->source = src; + /* reference source chain which now becomes immutable */ + evbuffer_chain_incref(chain); + extra->parent = chain; + chain->flags |= EVBUFFER_IMMUTABLE; + tmp->buffer_len = chain->buffer_len; + tmp->misalign = chain->misalign; + tmp->off = chain->off; + tmp->flags |= EVBUFFER_MULTICAST|EVBUFFER_IMMUTABLE; + tmp->buffer = chain->buffer; + evbuffer_chain_insert(dst, tmp); + } +} + static void PREPEND_CHAIN(struct evbuffer *dst, struct evbuffer *src) { @@ -902,6 +991,49 @@ done: } int +evbuffer_add_buffer_reference(struct evbuffer *outbuf, struct evbuffer *inbuf) +{ + size_t in_total_len, out_total_len; + struct evbuffer_chain *chain; + int result = 0; + + EVBUFFER_LOCK2(inbuf, outbuf); + in_total_len = inbuf->total_len; + out_total_len = outbuf->total_len; + chain = inbuf->first; + + if (in_total_len == 0) + goto done; + + if (outbuf->freeze_end || outbuf == inbuf) { + result = -1; + goto done; + } + + for (; chain; chain = chain->next) { + if ((chain->flags & (EVBUFFER_FILESEGMENT|EVBUFFER_SENDFILE|EVBUFFER_MULTICAST)) != 0) { + /* chain type can not be referenced */ + result = -1; + goto done; + } + } + + if (out_total_len == 0) { + /* There might be an empty chain at the start of outbuf; free + * it. */ + evbuffer_free_all_chains(outbuf->first); + } + APPEND_CHAIN_MULTICAST(outbuf, inbuf); + + outbuf->n_add_for_cb += in_total_len; + evbuffer_invoke_callbacks(outbuf); + +done: + EVBUFFER_UNLOCK2(inbuf, outbuf); + return result; +} + +int evbuffer_prepend_buffer(struct evbuffer *outbuf, struct evbuffer *inbuf) { struct evbuffer_chain *pinned, *last; @@ -1001,10 +1133,8 @@ evbuffer_drain(struct evbuffer *buf, size_t len) } buf->first = chain; - if (chain) { - chain->misalign += remaining; - chain->off -= remaining; - } + chain->misalign += remaining; + chain->off -= remaining; } buf->n_del_for_cb += len; @@ -1022,7 +1152,7 @@ evbuffer_remove(struct evbuffer *buf, void *data_out, size_t datlen) { ev_ssize_t n; EVBUFFER_LOCK(buf); - n = evbuffer_copyout(buf, data_out, datlen); + n = evbuffer_copyout_from(buf, NULL, data_out, datlen); if (n > 0) { if (evbuffer_drain(buf, n)<0) n = -1; @@ -1034,18 +1164,34 @@ evbuffer_remove(struct evbuffer *buf, void *data_out, size_t datlen) ev_ssize_t evbuffer_copyout(struct evbuffer *buf, void *data_out, size_t datlen) { + return evbuffer_copyout_from(buf, NULL, data_out, datlen); +} + +ev_ssize_t +evbuffer_copyout_from(struct evbuffer *buf, const struct evbuffer_ptr *pos, + void *data_out, size_t datlen) +{ /*XXX fails badly on sendfile case. */ struct evbuffer_chain *chain; char *data = data_out; size_t nread; ev_ssize_t result = 0; + size_t pos_in_chain; EVBUFFER_LOCK(buf); - chain = buf->first; + if (pos) { + chain = pos->_internal.chain; + pos_in_chain = pos->_internal.pos_in_chain; + if (datlen + pos->pos > buf->total_len) + datlen = buf->total_len - pos->pos; + } else { + chain = buf->first; + pos_in_chain = 0; + if (datlen > buf->total_len) + datlen = buf->total_len; + } - if (datlen >= buf->total_len) - datlen = buf->total_len; if (datlen == 0) goto done; @@ -1057,18 +1203,23 @@ evbuffer_copyout(struct evbuffer *buf, void *data_out, size_t datlen) nread = datlen; - while (datlen && datlen >= chain->off) { - memcpy(data, chain->buffer + chain->misalign, chain->off); - data += chain->off; - datlen -= chain->off; + while (datlen && datlen >= chain->off - pos_in_chain) { + size_t copylen = chain->off - pos_in_chain; + memcpy(data, + chain->buffer + chain->misalign + pos_in_chain, + copylen); + data += copylen; + datlen -= copylen; chain = chain->next; + pos_in_chain = 0; EVUTIL_ASSERT(chain || datlen==0); } if (datlen) { EVUTIL_ASSERT(chain); - memcpy(data, chain->buffer + chain->misalign, datlen); + memcpy(data, chain->buffer + chain->misalign + pos_in_chain, + datlen); } result = nread; @@ -1372,7 +1523,7 @@ evbuffer_strspn( size_t i = ptr->_internal.pos_in_chain; if (!chain) - return -1; + return 0; while (1) { char *buffer = (char *)chain->buffer + chain->misalign; @@ -1403,13 +1554,16 @@ evbuffer_strspn( } -static inline char +static inline int evbuffer_getchr(struct evbuffer_ptr *it) { struct evbuffer_chain *chain = it->_internal.chain; size_t off = it->_internal.pos_in_chain; - return chain->buffer[chain->misalign + off]; + if (chain == NULL) + return -1; + + return (unsigned char)chain->buffer[chain->misalign + off]; } struct evbuffer_ptr @@ -1421,6 +1575,14 @@ evbuffer_search_eol(struct evbuffer *buffer, size_t extra_drain = 0; int ok = 0; + /* Avoid locking in trivial edge cases */ + if (start && start->_internal.chain == NULL) { + PTR_NOT_FOUND(&it); + if (eol_len_out) + *eol_len_out = extra_drain; + return it; + } + EVBUFFER_LOCK(buffer); if (start) { @@ -1447,29 +1609,37 @@ evbuffer_search_eol(struct evbuffer *buffer, extra_drain = 2; break; } - case EVBUFFER_EOL_CRLF: - while (1) { - if (evbuffer_find_eol_char(&it) < 0) - goto done; - if (evbuffer_getchr(&it) == '\n') { - extra_drain = 1; - break; - } else if (!evbuffer_ptr_memcmp( - buffer, &it, "\r\n", 2)) { - extra_drain = 2; - break; - } else { - if (evbuffer_ptr_set(buffer, &it, 1, - EVBUFFER_PTR_ADD)<0) - goto done; - } + case EVBUFFER_EOL_CRLF: { + ev_ssize_t start_pos = it.pos; + /* Look for a LF ... */ + if (evbuffer_strchr(&it, '\n') < 0) + goto done; + extra_drain = 1; + /* ... optionally preceeded by a CR. */ + if (it.pos == start_pos) + break; /* If the first character is \n, don't back up */ + /* This potentially does an extra linear walk over the first + * few chains. Probably, that's not too expensive unless you + * have a really pathological setup. */ + memcpy(&it2, &it, sizeof(it)); + if (evbuffer_ptr_subtract(buffer, &it2, 1)<0) + break; + if (evbuffer_getchr(&it2) == '\r') { + memcpy(&it, &it2, sizeof(it)); + extra_drain = 2; } break; + } case EVBUFFER_EOL_LF: if (evbuffer_strchr(&it, '\n') < 0) goto done; extra_drain = 1; break; + case EVBUFFER_EOL_NUL: + if (evbuffer_strchr(&it, '\0') < 0) + goto done; + extra_drain = 1; + break; default: goto done; } @@ -1478,9 +1648,8 @@ evbuffer_search_eol(struct evbuffer *buffer, done: EVBUFFER_UNLOCK(buffer); - if (!ok) { - it.pos = -1; - } + if (!ok) + PTR_NOT_FOUND(&it); if (eol_len_out) *eol_len_out = extra_drain; @@ -1949,7 +2118,7 @@ evbuffer_expand(struct evbuffer *buf, size_t datlen) * Reads data from a file descriptor into a buffer. */ -#if defined(_EVENT_HAVE_SYS_UIO_H) || defined(WIN32) +#if defined(_EVENT_HAVE_SYS_UIO_H) || defined(_WIN32) #define USE_IOVEC_IMPL #endif @@ -2037,7 +2206,7 @@ _evbuffer_read_setup_vecs(struct evbuffer *buf, ev_ssize_t howmuch, static int get_n_bytes_readable_on_socket(evutil_socket_t fd) { -#if defined(FIONREAD) && defined(WIN32) +#if defined(FIONREAD) && defined(_WIN32) unsigned long lng = EVBUFFER_MAX_READ; if (ioctlsocket(fd, FIONREAD, &lng) < 0) return -1; @@ -2103,7 +2272,7 @@ evbuffer_read(struct evbuffer *buf, evutil_socket_t fd, int howmuch) WSABUF_FROM_EVBUFFER_IOV(&vecs[i], &ev_vecs[i]); #endif -#ifdef WIN32 +#ifdef _WIN32 { DWORD bytesRead; DWORD flags=0; @@ -2134,7 +2303,7 @@ evbuffer_read(struct evbuffer *buf, evutil_socket_t fd, int howmuch) /* We can append new data at this point */ p = chain->buffer + chain->misalign + chain->off; -#ifndef WIN32 +#ifndef _WIN32 n = read(fd, p, howmuch); #else n = recv(fd, p, howmuch, 0); @@ -2179,56 +2348,6 @@ done: return result; } -#ifdef WIN32 -static int -evbuffer_readfile(struct evbuffer *buf, evutil_socket_t fd, ev_ssize_t howmuch) -{ - int result; - int nchains, n; - struct evbuffer_iovec v[2]; - - EVBUFFER_LOCK(buf); - - if (buf->freeze_end) { - result = -1; - goto done; - } - - if (howmuch < 0) - howmuch = 16384; - - - /* XXX we _will_ waste some space here if there is any space left - * over on buf->last. */ - nchains = evbuffer_reserve_space(buf, howmuch, v, 2); - if (nchains < 1 || nchains > 2) { - result = -1; - goto done; - } - n = read((int)fd, v[0].iov_base, (unsigned int)v[0].iov_len); - if (n <= 0) { - result = n; - goto done; - } - v[0].iov_len = (IOV_LEN_TYPE) n; /* XXXX another problem with big n.*/ - if (nchains > 1) { - n = read((int)fd, v[1].iov_base, (unsigned int)v[1].iov_len); - if (n <= 0) { - result = (unsigned long) v[0].iov_len; - evbuffer_commit_space(buf, v, 1); - goto done; - } - v[1].iov_len = n; - } - evbuffer_commit_space(buf, v, nchains); - - result = n; -done: - EVBUFFER_UNLOCK(buf); - return result; -} -#endif - #ifdef USE_IOVEC_IMPL static inline int evbuffer_write_iovec(struct evbuffer *buffer, evutil_socket_t fd, @@ -2263,7 +2382,7 @@ evbuffer_write_iovec(struct evbuffer *buffer, evutil_socket_t fd, } chain = chain->next; } -#ifdef WIN32 +#ifdef _WIN32 { DWORD bytesSent; if (WSASend(fd, iov, i, &bytesSent, 0, NULL, NULL)) @@ -2280,37 +2399,39 @@ evbuffer_write_iovec(struct evbuffer *buffer, evutil_socket_t fd, #ifdef USE_SENDFILE static inline int -evbuffer_write_sendfile(struct evbuffer *buffer, evutil_socket_t fd, +evbuffer_write_sendfile(struct evbuffer *buffer, evutil_socket_t dest_fd, ev_ssize_t howmuch) { struct evbuffer_chain *chain = buffer->first; - struct evbuffer_chain_fd *info = - EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_fd, chain); + struct evbuffer_chain_file_segment *info = + EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_file_segment, + chain); + const int source_fd = info->segment->fd; #if defined(SENDFILE_IS_MACOSX) || defined(SENDFILE_IS_FREEBSD) int res; - off_t len = chain->off; + ev_off_t len = chain->off; #elif defined(SENDFILE_IS_LINUX) || defined(SENDFILE_IS_SOLARIS) ev_ssize_t res; - off_t offset = chain->misalign; + ev_off_t offset = chain->misalign; #endif ASSERT_EVBUFFER_LOCKED(buffer); #if defined(SENDFILE_IS_MACOSX) - res = sendfile(info->fd, fd, chain->misalign, &len, NULL, 0); + res = sendfile(source_fd, dest_fd, chain->misalign, &len, NULL, 0); if (res == -1 && !EVUTIL_ERR_RW_RETRIABLE(errno)) return (-1); return (len); #elif defined(SENDFILE_IS_FREEBSD) - res = sendfile(info->fd, fd, chain->misalign, chain->off, NULL, &len, 0); + res = sendfile(source_fd, dest_fd, chain->misalign, chain->off, NULL, &len, 0); if (res == -1 && !EVUTIL_ERR_RW_RETRIABLE(errno)) return (-1); return (len); #elif defined(SENDFILE_IS_LINUX) /* TODO(niels): implement splice */ - res = sendfile(fd, info->fd, &offset, chain->off); + res = sendfile(dest_fd, source_fd, &offset, chain->off); if (res == -1 && EVUTIL_ERR_RW_RETRIABLE(errno)) { /* if this is EAGAIN or EINTR return 0; otherwise, -1 */ return (0); @@ -2319,7 +2440,7 @@ evbuffer_write_sendfile(struct evbuffer *buffer, evutil_socket_t fd, #elif defined(SENDFILE_IS_SOLARIS) { const off_t offset_orig = offset; - res = sendfile(fd, info->fd, &offset, chain->off); + res = sendfile(dest_fd, source_fd, &offset, chain->off); if (res == -1 && EVUTIL_ERR_RW_RETRIABLE(errno)) { if (offset - offset_orig) return offset - offset_orig; @@ -2357,7 +2478,7 @@ evbuffer_write_atmost(struct evbuffer *buffer, evutil_socket_t fd, #endif #ifdef USE_IOVEC_IMPL n = evbuffer_write_iovec(buffer, fd, howmuch); -#elif defined(WIN32) +#elif defined(_WIN32) /* XXX(nickm) Don't disable this code until we know if * the WSARecv code above works. */ void *p = evbuffer_pullup(buffer, howmuch); @@ -2405,12 +2526,36 @@ evbuffer_find(struct evbuffer *buffer, const unsigned char *what, size_t len) return search; } +/* Subract <b>howfar</b> from the position of <b>pos</b> within + * <b>buf</b>. Returns 0 on success, -1 on failure. + * + * This isn't exposed yet, because of potential inefficiency issues. + * Maybe it should be. */ +static int +evbuffer_ptr_subtract(struct evbuffer *buf, struct evbuffer_ptr *pos, + size_t howfar) +{ + if (howfar > (size_t)pos->pos) + return -1; + if (pos->_internal.chain && howfar <= pos->_internal.pos_in_chain) { + pos->_internal.pos_in_chain -= howfar; + pos->pos -= howfar; + return 0; + } else { + const size_t newpos = pos->pos - howfar; + /* Here's the inefficient part: it walks over the + * chains until we hit newpos. */ + return evbuffer_ptr_set(buf, pos, newpos, EVBUFFER_PTR_SET); + } +} + int evbuffer_ptr_set(struct evbuffer *buf, struct evbuffer_ptr *pos, size_t position, enum evbuffer_ptr_how how) { size_t left = position; struct evbuffer_chain *chain = NULL; + int result = 0; EVBUFFER_LOCK(buf); @@ -2437,14 +2582,18 @@ evbuffer_ptr_set(struct evbuffer *buf, struct evbuffer_ptr *pos, if (chain) { pos->_internal.chain = chain; pos->_internal.pos_in_chain = position + left; - } else { + } else if (left == 0) { + /* The first byte in the (nonexistent) chain after the last chain */ pos->_internal.chain = NULL; - pos->pos = -1; + pos->_internal.pos_in_chain = 0; + } else { + PTR_NOT_FOUND(pos); + result = -1; } EVBUFFER_UNLOCK(buf); - return chain != NULL ? 0 : -1; + return result; } /** @@ -2549,8 +2698,7 @@ evbuffer_search_range(struct evbuffer *buffer, const char *what, size_t len, con } not_found: - pos.pos = -1; - pos._internal.chain = NULL; + PTR_NOT_FOUND(&pos); done: EVBUFFER_UNLOCK(buffer); return pos; @@ -2565,6 +2713,10 @@ evbuffer_peek(struct evbuffer *buffer, ev_ssize_t len, int idx = 0; ev_ssize_t len_so_far = 0; + /* Avoid locking in trivial edge cases */ + if (start_at && start_at->_internal.chain == NULL) + return 0; + EVBUFFER_LOCK(buffer); if (start_at) { @@ -2722,169 +2874,323 @@ done: return result; } -/* TODO(niels): maybe we don't want to own the fd, however, in that - * case, we should dup it - dup is cheap. Perhaps, we should use a - * callback instead? - */ /* TODO(niels): we may want to add to automagically convert to mmap, in * case evbuffer_remove() or evbuffer_pullup() are being used. */ -int -evbuffer_add_file(struct evbuffer *outbuf, int fd, - ev_off_t offset, ev_off_t length) +struct evbuffer_file_segment * +evbuffer_file_segment_new( + int fd, ev_off_t offset, ev_off_t length, unsigned flags) { -#if defined(USE_SENDFILE) || defined(_EVENT_HAVE_MMAP) - struct evbuffer_chain *chain; - struct evbuffer_chain_fd *info; + struct evbuffer_file_segment *seg = + mm_calloc(sizeof(struct evbuffer_file_segment), 1); + if (!seg) + return NULL; + seg->refcnt = 1; + seg->fd = fd; + seg->flags = flags; + seg->file_offset = offset; + +#ifdef _WIN32 +#define lseek _lseeki64 +#define fstat _fstat +#define stat _stat #endif + if (length == -1) { + struct stat st; + if (fstat(fd, &st) < 0) + goto err; + length = st.st_size; + } + seg->length = length; + #if defined(USE_SENDFILE) - int sendfile_okay = 1; + if (!(flags & EVBUF_FS_DISABLE_SENDFILE)) { + seg->can_sendfile = 1; + goto done; + } #endif - int ok = 1; + + if (evbuffer_file_segment_materialize(seg)<0) + goto err; #if defined(USE_SENDFILE) - if (use_sendfile) { - EVBUFFER_LOCK(outbuf); - sendfile_okay = outbuf->flags & EVBUFFER_FLAG_DRAINS_TO_FD; - EVBUFFER_UNLOCK(outbuf); +done: +#endif + if (!(flags & EVBUF_FS_DISABLE_LOCKING)) { + EVTHREAD_ALLOC_LOCK(seg->lock, 0); } + return seg; +err: + mm_free(seg); + return NULL; +} - if (use_sendfile && sendfile_okay) { - chain = evbuffer_chain_new(sizeof(struct evbuffer_chain_fd)); - if (chain == NULL) { - event_warn("%s: out of memory", __func__); - return (-1); - } - - chain->flags |= EVBUFFER_SENDFILE | EVBUFFER_IMMUTABLE; - chain->buffer = NULL; /* no reading possible */ - chain->buffer_len = length + offset; - chain->off = length; - chain->misalign = offset; +/* DOCDOC */ +/* Requires lock */ +static int +evbuffer_file_segment_materialize(struct evbuffer_file_segment *seg) +{ + const unsigned flags = seg->flags; + const int fd = seg->fd; + const ev_off_t length = seg->length; + const ev_off_t offset = seg->file_offset; - info = EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_fd, chain); - info->fd = fd; + if (seg->contents) + return 0; /* already materialized */ - EVBUFFER_LOCK(outbuf); - if (outbuf->freeze_end) { - mm_free(chain); - ok = 0; - } else { - outbuf->n_add_for_cb += length; - evbuffer_chain_insert(outbuf, chain); - } - } else -#endif #if defined(_EVENT_HAVE_MMAP) - if (use_mmap) { - void *mapped = mmap(NULL, length + offset, PROT_READ, + if (!(flags & EVBUF_FS_DISABLE_MMAP)) { + off_t offset_rounded = 0, offset_leftover = 0; + void *mapped; + if (offset) { + /* mmap implementations don't generally like us + * to have an offset that isn't a round */ +#ifdef SC_PAGE_SIZE + long page_size = sysconf(SC_PAGE_SIZE); +#elif defined(_SC_PAGE_SIZE) + long page_size = sysconf(_SC_PAGE_SIZE); +#else + long page_size = 1; +#endif + if (page_size == -1) + goto err; + offset_leftover = offset % page_size; + offset_rounded = offset - offset_leftover; + } + mapped = mmap(NULL, length + offset_leftover, + PROT_READ, #ifdef MAP_NOCACHE - MAP_NOCACHE | + MAP_NOCACHE | /* ??? */ #endif #ifdef MAP_FILE MAP_FILE | #endif MAP_PRIVATE, - fd, 0); - /* some mmap implementations require offset to be a multiple of - * the page size. most users of this api, are likely to use 0 - * so mapping everything is not likely to be a problem. - * TODO(niels): determine page size and round offset to that - * page size to avoid mapping too much memory. - */ + fd, offset_rounded); if (mapped == MAP_FAILED) { event_warn("%s: mmap(%d, %d, %zu) failed", __func__, fd, 0, (size_t)(offset + length)); - return (-1); + } else { + seg->mapping = mapped; + seg->contents = (char*)mapped+offset_leftover; + seg->mmap_offset = 0; + seg->is_mapping = 1; + goto done; } - chain = evbuffer_chain_new(sizeof(struct evbuffer_chain_fd)); - if (chain == NULL) { - event_warn("%s: out of memory", __func__); - munmap(mapped, length); - return (-1); + } +#endif +#ifdef _WIN32 + if (!(flags & EVBUF_FS_DISABLE_MMAP)) { + long h = (long)_get_osfhandle(fd); + HANDLE m; + ev_uint64_t total_size = length+offset; + if (h == (long)INVALID_HANDLE_VALUE) + goto err; + m = CreateFileMapping((HANDLE)h, NULL, PAGE_READONLY, + (total_size >> 32), total_size & 0xfffffffful, + NULL); + if (m != INVALID_HANDLE_VALUE) { /* Does h leak? */ + seg->mapping_handle = m; + seg->mmap_offset = offset; + seg->is_mapping = 1; + goto done; + } + } +#endif + { + ev_off_t start_pos = lseek(fd, 0, SEEK_CUR), pos; + ev_off_t read_so_far = 0; + char *mem; + int e; + ev_ssize_t n = 0; + if (!(mem = mm_malloc(length))) + goto err; + if (start_pos < 0) { + mm_free(mem); + goto err; + } + if (lseek(fd, offset, SEEK_SET) < 0) { + mm_free(mem); + goto err; + } + while (read_so_far < length) { + n = read(fd, mem+read_so_far, length-read_so_far); + if (n <= 0) + break; + read_so_far += n; } - chain->flags |= EVBUFFER_MMAP | EVBUFFER_IMMUTABLE; - chain->buffer = mapped; - chain->buffer_len = length + offset; - chain->off = length + offset; - - info = EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_fd, chain); - info->fd = fd; + e = errno; + pos = lseek(fd, start_pos, SEEK_SET); + if (n < 0 || (n == 0 && length > read_so_far)) { + mm_free(mem); + errno = e; + goto err; + } else if (pos < 0) { + mm_free(mem); + goto err; + } - EVBUFFER_LOCK(outbuf); - if (outbuf->freeze_end) { - info->fd = -1; - evbuffer_chain_free(chain); - ok = 0; - } else { - outbuf->n_add_for_cb += length; + seg->contents = mem; + } - evbuffer_chain_insert(outbuf, chain); +done: + return 0; +err: + return -1; +} - /* we need to subtract whatever we don't need */ - evbuffer_drain(outbuf, offset); - } - } else +void +evbuffer_file_segment_free(struct evbuffer_file_segment *seg) +{ + int refcnt; + EVLOCK_LOCK(seg->lock, 0); + refcnt = --seg->refcnt; + EVLOCK_UNLOCK(seg->lock, 0); + if (refcnt > 0) + return; + EVUTIL_ASSERT(refcnt == 0); + + if (seg->is_mapping) { +#ifdef _WIN32 + CloseHandle(seg->mapping_handle); +#elif defined (_EVENT_HAVE_MMAP) + if (munmap(seg->mapping, seg->length) == -1) + event_warn("%s: munmap failed", __func__); #endif - { - /* the default implementation */ - struct evbuffer *tmp = evbuffer_new(); - ev_ssize_t read; + } else if (seg->contents) { + mm_free(seg->contents); + } - if (tmp == NULL) - return (-1); + if ((seg->flags & EVBUF_FS_CLOSE_ON_FREE) && seg->fd >= 0) { + close(seg->fd); + } -#ifdef WIN32 -#define lseek _lseeki64 -#endif - if (lseek(fd, offset, SEEK_SET) == -1) { - evbuffer_free(tmp); - return (-1); - } + EVTHREAD_FREE_LOCK(seg->lock, 0); + mm_free(seg); +} - /* we add everything to a temporary buffer, so that we - * can abort without side effects if the read fails. - */ - while (length) { - read = evbuffer_readfile(tmp, fd, (ev_ssize_t)length); - if (read == -1) { - evbuffer_free(tmp); - return (-1); - } +int +evbuffer_add_file_segment(struct evbuffer *buf, + struct evbuffer_file_segment *seg, ev_off_t offset, ev_off_t length) +{ + struct evbuffer_chain *chain; + struct evbuffer_chain_file_segment *extra; + int can_use_sendfile = 0; - length -= read; + EVBUFFER_LOCK(buf); + EVLOCK_LOCK(seg->lock, 0); + if (buf->flags & EVBUFFER_FLAG_DRAINS_TO_FD) { + can_use_sendfile = 1; + } else { + if (!seg->contents) { + if (evbuffer_file_segment_materialize(seg)<0) { + EVLOCK_UNLOCK(seg->lock, 0); + EVBUFFER_UNLOCK(buf); + return -1; + } } + } + ++seg->refcnt; + EVLOCK_UNLOCK(seg->lock, 0); - EVBUFFER_LOCK(outbuf); - if (outbuf->freeze_end) { - evbuffer_free(tmp); - ok = 0; - } else { - evbuffer_add_buffer(outbuf, tmp); - evbuffer_free(tmp); + if (buf->freeze_end) + goto err; -#ifdef WIN32 -#define close _close -#endif - close(fd); + if (length < 0) { + if (offset > seg->length) + goto err; + length = seg->length - offset; + } + + /* Can we actually add this? */ + if (offset+length > seg->length) + goto err; + + chain = evbuffer_chain_new(sizeof(struct evbuffer_chain_file_segment)); + if (!chain) + goto err; + extra = EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_file_segment, chain); + + chain->flags |= EVBUFFER_IMMUTABLE|EVBUFFER_FILESEGMENT; + if (can_use_sendfile && seg->can_sendfile) { + chain->flags |= EVBUFFER_SENDFILE; + chain->misalign = seg->file_offset + offset; + chain->off = length; + chain->buffer_len = chain->misalign + length; + } else if (seg->is_mapping) { +#ifdef _WIN32 + ev_uint64_t total_offset = seg->mmap_offset+offset; + ev_uint64_t offset_rounded=0, offset_remaining=0; + LPVOID data; + if (total_offset) { + SYSTEM_INFO si; + memset(&si, 0, sizeof(si)); /* cargo cult */ + GetSystemInfo(&si); + offset_remaining = total_offset % si.dwAllocationGranularity; + offset_rounded = total_offset - offset_remaining; } + data = MapViewOfFile( + seg->mapping_handle, + FILE_MAP_READ, + offset_rounded >> 32, + offset_rounded & 0xfffffffful, + length + offset_remaining); + if (data == NULL) { + mm_free(chain); + goto err; + } + chain->buffer = (unsigned char*) data; + chain->buffer_len = length+offset_remaining; + chain->misalign = offset_remaining; + chain->off = length; +#else + chain->buffer = (unsigned char*)(seg->contents + offset); + chain->buffer_len = length; + chain->off = length; +#endif + } else { + chain->buffer = (unsigned char*)(seg->contents + offset); + chain->buffer_len = length; + chain->off = length; } - if (ok) - evbuffer_invoke_callbacks(outbuf); - EVBUFFER_UNLOCK(outbuf); + extra->segment = seg; + buf->n_add_for_cb += length; + evbuffer_chain_insert(buf, chain); + + evbuffer_invoke_callbacks(buf); + + EVBUFFER_UNLOCK(buf); - return ok ? 0 : -1; + return 0; +err: + EVBUFFER_UNLOCK(buf); + evbuffer_file_segment_free(seg); + return -1; } +int +evbuffer_add_file(struct evbuffer *buf, int fd, ev_off_t offset, ev_off_t length) +{ + struct evbuffer_file_segment *seg; + unsigned flags = EVBUF_FS_CLOSE_ON_FREE; + int r; + + seg = evbuffer_file_segment_new(fd, offset, length, flags); + if (!seg) + return -1; + r = evbuffer_add_file_segment(buf, seg, 0, length); + evbuffer_file_segment_free(seg); + return r; +} void evbuffer_setcb(struct evbuffer *buffer, evbuffer_cb cb, void *cbarg) { EVBUFFER_LOCK(buffer); - if (!TAILQ_EMPTY(&buffer->callbacks)) + if (!LIST_EMPTY(&buffer->callbacks)) evbuffer_remove_all_callbacks(buffer); if (cb) { @@ -2906,7 +3212,7 @@ evbuffer_add_cb(struct evbuffer *buffer, evbuffer_cb_func cb, void *cbarg) e->cb.cb_func = cb; e->cbarg = cbarg; e->flags = EVBUFFER_CB_ENABLED; - TAILQ_INSERT_HEAD(&buffer->callbacks, e, next); + LIST_INSERT_HEAD(&buffer->callbacks, e, next); EVBUFFER_UNLOCK(buffer); return e; } @@ -2916,7 +3222,7 @@ evbuffer_remove_cb_entry(struct evbuffer *buffer, struct evbuffer_cb_entry *ent) { EVBUFFER_LOCK(buffer); - TAILQ_REMOVE(&buffer->callbacks, ent, next); + LIST_REMOVE(ent, next); EVBUFFER_UNLOCK(buffer); mm_free(ent); return 0; @@ -2928,7 +3234,7 @@ evbuffer_remove_cb(struct evbuffer *buffer, evbuffer_cb_func cb, void *cbarg) struct evbuffer_cb_entry *cbent; int result = -1; EVBUFFER_LOCK(buffer); - TAILQ_FOREACH(cbent, &buffer->callbacks, next) { + LIST_FOREACH(cbent, &buffer->callbacks, next) { if (cb == cbent->cb.cb_func && cbarg == cbent->cbarg) { result = evbuffer_remove_cb_entry(buffer, cbent); goto done; @@ -3013,50 +3319,3 @@ evbuffer_cb_unsuspend(struct evbuffer *buffer, struct evbuffer_cb_entry *cb) } #endif -/* These hooks are exposed so that the unit tests can temporarily disable - * sendfile support in order to test mmap, or both to test linear - * access. Don't use it; if we need to add a way to disable sendfile support - * in the future, it will probably be via an alternate version of - * evbuffer_add_file() with a 'flags' argument. - */ -int _evbuffer_testing_use_sendfile(void); -int _evbuffer_testing_use_mmap(void); -int _evbuffer_testing_use_linear_file_access(void); - -int -_evbuffer_testing_use_sendfile(void) -{ - int ok = 0; -#ifdef USE_SENDFILE - use_sendfile = 1; - ok = 1; -#endif -#ifdef _EVENT_HAVE_MMAP - use_mmap = 0; -#endif - return ok; -} -int -_evbuffer_testing_use_mmap(void) -{ - int ok = 0; -#ifdef USE_SENDFILE - use_sendfile = 0; -#endif -#ifdef _EVENT_HAVE_MMAP - use_mmap = 1; - ok = 1; -#endif - return ok; -} -int -_evbuffer_testing_use_linear_file_access(void) -{ -#ifdef USE_SENDFILE - use_sendfile = 0; -#endif -#ifdef _EVENT_HAVE_MMAP - use_mmap = 0; -#endif - return 1; -} |