From e05ce80502113e7428067eca01124c54a0cc54ae Mon Sep 17 00:00:00 2001 From: Glenn Strauss Date: Tue, 8 Feb 2022 15:01:12 -0500 Subject: [multiple] shared code for struct chunk and mmap chunkqueue_chunk_file_view() reduces size of struct chunk use mmap with mod_deflate libdeflate, if mmap available, even when lighttpd not built with --enable-mmap avoid using mmap on temp files in chunkqueue (c->file.is_temp) since pread() with a reasonable block size is typically as fast or faster than mmap on files read sequentially and used only once, especially when writing results to limited-size socket buffers (and lighttpd temp files are, in most cases, by default about 1 MB) (Exception: sometimes mmap is used for convenience or to fulfill a requirement, e.g. one-shot libdeflate in mod_deflate) (There are many factors which influence speed of mmap versus pread, so this should not be construed as generic usage advice.) --- src/chunk.c | 299 +++++++++++++++++++++++++++++++++++----------------- src/chunk.h | 66 ++++++++++-- src/mod_deflate.c | 83 ++++----------- src/mod_webdav.c | 95 +++++------------ src/network_write.c | 110 +++---------------- src/sys-mmap.h | 2 + 6 files changed, 327 insertions(+), 328 deletions(-) diff --git a/src/chunk.c b/src/chunk.c index 6b6a5178..b1aba5ec 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -22,6 +22,44 @@ #include #include + +#ifdef HAVE_MMAP + +#define MMAP_CHUNK_SIZE (512*1024) + +__attribute_cold__ +/*__attribute_noinline__*/ +static off_t +mmap_pagemask (void) +{ + #ifndef _WIN32 + long pagesize = sysconf(_SC_PAGESIZE); + #else + long pagesize = -1; /*(not implemented (yet))*/ + #endif + if (-1 == pagesize) pagesize = 4096; + force_assert(pagesize < MMAP_CHUNK_SIZE); + return ~((off_t)pagesize - 1); /* pagesize always power-of-2 */ +} + +#if 0 +static off_t +mmap_align_offset (off_t start) +{ + static off_t pagemask = 0; + if (0 == pagemask) + pagemask = mmap_pagemask(); + return (start & pagemask); +} +#endif + +#define mmap_align_offset(offset) ((offset) & chunk_pagemask) +static off_t chunk_pagemask = 0; +static int chunk_mmap_flags = MAP_SHARED; + +#endif /* HAVE_MMAP */ + + /* default 1 MB */ #define DEFAULT_TEMPFILE_SIZE (1 * 1024 * 1024) @@ -44,6 +82,12 @@ void chunkqueue_set_tempdirs_default_reset (void) chunk_buf_sz = 8192; chunkqueue_default_tempdirs = NULL; chunkqueue_default_tempfile_size = DEFAULT_TEMPFILE_SIZE; + + #ifdef HAVE_MMAP /*(abuse this func to initialize statics as startup)*/ + if (0 == chunk_pagemask) + chunk_pagemask = mmap_pagemask(); + chunk_mmap_flags = MAP_SHARED; + #endif } chunkqueue *chunkqueue_init(chunkqueue *cq) { @@ -72,16 +116,16 @@ static chunk *chunk_init(void) { c->next = NULL; c->offset = 0; c->file.length = 0; - c->file.mmap.length = c->file.mmap.offset = 0; c->file.is_temp = 0; + c->file.view = NULL; #endif c->file.fd = -1; - c->file.mmap.start = MAP_FAILED; c->mem = buffer_init(); return c; } +__attribute_noinline__ __attribute_returns_nonnull__ static chunk *chunk_init_sz(size_t sz) { chunk * const restrict c = chunk_init(); @@ -89,6 +133,41 @@ static chunk *chunk_init_sz(size_t sz) { return c; } +#ifdef HAVE_MMAP + +__attribute_malloc__ +__attribute_returns_nonnull__ +static void * chunk_file_view_init (void) { + chunk_file_view * const restrict cfv = calloc(1, sizeof(*cfv)); + force_assert(NULL != cfv); + cfv->mptr = MAP_FAILED; + #if 0 /*(zeroed by calloc())*/ + cfv->mlen = 0; + cfv->foff = 0; + #endif + cfv->refcnt = 1; + return cfv; +} + +__attribute_nonnull__() +static chunk_file_view * chunk_file_view_release (chunk_file_view *cfv) { + if (0 == --cfv->refcnt) { + if (MAP_FAILED != cfv->mptr) + munmap(cfv->mptr, (size_t)cfv->mlen); + free(cfv); + } + return NULL; +} + +__attribute_cold__ +__attribute_noinline__ +__attribute_nonnull__() +static chunk_file_view * chunk_file_view_failed (chunk_file_view *cfv) { + return chunk_file_view_release(cfv); +} + +#endif /* HAVE_MMAP */ + static void chunk_reset_file_chunk(chunk *c) { if (c->file.is_temp) { c->file.is_temp = 0; @@ -103,11 +182,10 @@ static void chunk_reset_file_chunk(chunk *c) { else if (c->file.fd != -1) { close(c->file.fd); } - if (MAP_FAILED != c->file.mmap.start) { - munmap(c->file.mmap.start, c->file.mmap.length); - c->file.mmap.start = MAP_FAILED; - c->file.mmap.length = c->file.mmap.offset = 0; - } + #ifdef HAVE_MMAP + if (c->file.view) + c->file.view = chunk_file_view_release(c->file.view); + #endif c->file.fd = -1; c->file.length = 0; c->type = MEM_CHUNK; @@ -157,6 +235,7 @@ static void chunk_push_oversized(chunk * const c, const size_t sz) { } } +__attribute_noinline__ __attribute_returns_nonnull__ static buffer * chunk_buffer_acquire_sz(const size_t sz) { chunk *c; @@ -238,6 +317,7 @@ size_t chunk_buffer_prepare_append(buffer * const b, size_t sz) { return buffer_string_space(b); } +__attribute_noinline__ __attribute_returns_nonnull__ static chunk * chunk_acquire(size_t sz) { if (sz <= (chunk_buf_sz|1)) { @@ -585,6 +665,10 @@ static void chunkqueue_dup_file_chunk_fd (chunk * const restrict d, const chunk } else d->file.fd = fdevent_dup_cloexec(c->file.fd); + #ifdef HAVE_MMAP + if ((d->file.view = c->file.view)) + ++d->file.view->refcnt; + #endif } } @@ -1290,7 +1374,7 @@ chunkqueue_write_data (const int fd, const void *buf, size_t len) } -#if defined(HAVE_MMAP) || defined(_WIN32) /*(see local sys-mmap.h)*/ +#ifdef HAVE_MMAP __attribute_cold__ #endif __attribute_noinline__ @@ -1308,67 +1392,6 @@ chunkqueue_write_chunk_file_intermed (const int fd, chunk * const restrict c, lo } -#if defined(HAVE_MMAP) || defined(_WIN32) /*(see local sys-mmap.h)*/ - -/*(improved from network_write_mmap.c)*/ -static off_t -mmap_align_offset (off_t start) -{ - static off_t pagemask = 0; - if (0 == pagemask) { - #ifndef _WIN32 - long pagesize = sysconf(_SC_PAGESIZE); - #else - long pagesize = -1; /*(not implemented (yet))*/ - #endif - if (-1 == pagesize) pagesize = 4096; - pagemask = ~((off_t)pagesize - 1); /* pagesize always power-of-2 */ - } - return (start & pagemask); -} - - -__attribute_noinline__ -static char * -chunkqueue_mmap_chunk_len (chunk * const c, off_t len) -{ - /* (re)mmap the buffer to file length if range is not covered completely */ - /*(caller is responsible for handling SIGBUS if chunkqueue might contain - * untrusted file, i.e. any file other than lighttpd-created tempfile)*/ - /*(tempfiles are expected for input, MAP_PRIVATE used for portability)*/ - /*(mmaps and writes complete chunk instead of only small parts; files - * are expected to be temp files with reasonable chunk sizes)*/ - if (MAP_FAILED == c->file.mmap.start - || c->offset < c->file.mmap.offset - || c->offset+len > (off_t)(c->file.mmap.offset + c->file.mmap.length)) { - - if (MAP_FAILED != c->file.mmap.start) { - munmap(c->file.mmap.start, c->file.mmap.length); - /*c->file.mmap.start = MAP_FAILED;*//*(assigned below)*/ - } - - c->file.mmap.offset = mmap_align_offset(c->offset); - c->file.mmap.length = c->file.length - c->file.mmap.offset; - c->file.mmap.start = - mmap(NULL, c->file.mmap.length, PROT_READ, MAP_PRIVATE, - c->file.fd, c->file.mmap.offset); - if (MAP_FAILED == c->file.mmap.start) return NULL; - - #if 0 /*(review callers before changing; some expect open file)*/ - /* close() fd as soon as fully mmap() rather than when done w/ chunk - * (possibly worthwhile to keep active fd count lower) */ - if (c->file.is_temp && !c->file.refchg) { - close(c->file.fd); - c->file.fd = -1; - } - #endif - } - return c->file.mmap.start + c->offset - c->file.mmap.offset; -} - -#endif - - #if defined HAVE_SYS_SENDFILE_H && defined HAVE_SENDFILE \ && (!defined _LARGEFILE_SOURCE || defined HAVE_SENDFILE64) \ && defined(__linux__) && !defined HAVE_SENDFILE_BROKEN @@ -1396,13 +1419,16 @@ chunkqueue_write_chunk_file (const int fd, chunk * const restrict c, log_error_s sendfile(fd, c->file.fd, &offset, len < INT32_MAX ? len : INT32_MAX); if (__builtin_expect( (wr >= 0), 1) || (errno != EINVAL && errno != ENOSYS)) return wr; - #endif - - #if defined(HAVE_MMAP) || defined(_WIN32) /*(see local sys-mmap.h)*/ + /*(could fallback to mmap, but if sendfile fails on linux, mmap may, too)*/ + #elif defined(HAVE_MMAP) /*(chunkqueue_write_chunk() caller must protect against SIGBUS, if needed)*/ - const char * const data = chunkqueue_mmap_chunk_len(c, len); - if (NULL != data) - return chunkqueue_write_data(fd, data, len); + const chunk_file_view * const restrict cfv = + chunkqueue_chunk_file_view(c, len, errh); + if (NULL != cfv) { + const off_t mmap_avail = chunk_file_view_dlen(cfv, c->offset); + return chunkqueue_write_data(fd, chunk_file_view_dptr(cfv, c->offset), + len <= mmap_avail ? len : mmap_avail); + } #endif return chunkqueue_write_chunk_file_intermed(fd, c, errh); @@ -1517,8 +1543,7 @@ chunkqueue_small_resp_optim (chunkqueue * const restrict cq) #if 0 -#if defined(_LP64) || defined(__LP64__) || defined(_WIN64) -#if defined(HAVE_MMAP) || defined(_WIN32) /*see local sys-mmap.h*/ +#ifdef HAVE_MMAP __attribute_noinline__ static off_t chunk_setjmp_memcpy_cb (void *dst, const void *src, off_t len) @@ -1529,7 +1554,6 @@ chunk_setjmp_memcpy_cb (void *dst, const void *src, off_t len) } #endif #endif -#endif int @@ -1561,7 +1585,6 @@ chunkqueue_peek_data (chunkqueue * const cq, case FILE_CHUNK: if (c->file.fd >= 0 || 0 == chunk_open_file_chunk(c, errh)) { - off_t offset = c->offset; off_t len = c->file.length - c->offset; if (len > (off_t)space) len = (off_t)space; @@ -1569,27 +1592,26 @@ chunkqueue_peek_data (chunkqueue * const cq, break; #if 0 /* XXX: might improve performance on some system workloads */ - #if defined(_LP64) || defined(__LP64__) || defined(_WIN64) - #if defined(HAVE_MMAP) || defined(_WIN32) /*see local sys-mmap.h*/ + #ifdef HAVE_MMAP /* mmap file to access data - * (For now, also limit to 64-bit to avoid address space issues) - * (chunkqueue_mmap_chunk_len() uses MAP_PRIVATE, even though we - * might use MAP_SHARED, if supported, and !c->file.is_temp) * fd need not be kept open for the mmap once * the mmap has been created, but is currently kept open for * other pre-existing logic which checks fd and opens file, * such as the condition for entering this code block above. */ + /* Note: current use is with somewhat large buffers, e.g. 128k. + * If larger buffers are used, then upper limit, e.g. 512k, + * should be set for 32-bit to avoid address space issues) */ /* Note: under heavy load (or microbenchmark), system-reported * memory use for RSS can be very, very large, due to presence * of lots and lots of temp file read-only memory maps. - * pmap -X and exclude lighttpd temporary files to get a better + * pmap -X and exclude lighttpd mmap files to get a better * view of memory use */ - /* Data should be large enough that mmap is worthwhile. - * mmap if file chunk > 16k or if already mapped */ - char *mdata; - if ((c->file.mmap.start != MAP_FAILED - || c->file.length - c->offset > 16384) - && (mdata = chunkqueue_mmap_chunk_len(c, len))) { + const chunk_file_view * const restrict cfv = (!c->file.is_temp) + ? chunkqueue_chunk_file_view(c, len, errh) + : NULL; + if (cfv && chunk_file_view_dlen(cfv, c->offset) >= len) { + /*(check (above) that mapped chunk length >= requested len)*/ + char * const mdata = chunk_file_view_dptr(cfv, c->offset); if (!c->file.is_temp) {/*(might be changed to policy flag)*/ if (sys_setjmp_eval3(chunk_setjmp_memcpy_cb, data_in+*dlen, mdata, len) < 0) { @@ -1606,11 +1628,10 @@ chunkqueue_peek_data (chunkqueue * const cq, break; } #endif - #endif #endif #ifndef HAVE_PREAD - if (-1 == lseek(c->file.fd, offset, SEEK_SET)) { + if (-1 == lseek(c->file.fd, c->offset, SEEK_SET)) { log_perror(errh, __FILE__, __LINE__, "lseek(\"%s\")", c->mem->ptr); return -1; @@ -1619,9 +1640,9 @@ chunkqueue_peek_data (chunkqueue * const cq, ssize_t rd; do { #ifdef HAVE_PREAD - rd =pread(c->file.fd, data_in + *dlen, (size_t)len, offset); + rd =pread(c->file.fd,data_in+*dlen,(size_t)len,c->offset); #else - rd = read(c->file.fd, data_in + *dlen, (size_t)len); + rd = read(c->file.fd,data_in+*dlen,(size_t)len); #endif } while (-1 == rd && errno == EINTR); if (rd <= 0) { /* -1 error; 0 EOF (unexpected) */ @@ -1697,3 +1718,91 @@ chunkqueue_read_squash (chunkqueue * const restrict cq, log_error_st * const res chunkqueue_append_chunk(cq, c); return c->mem; } + + +#ifdef HAVE_MMAP + +const chunk_file_view * +chunkqueue_chunk_file_viewadj (chunk * const c, off_t n, log_error_st * restrict errh) +{ + /*assert(c->type == FILE_CHUNK);*/ + chunk_file_view * restrict cfv = c->file.view; + + if (NULL == cfv) { + /* XXX: might add global config check to enable/disable mmap use here */ + cfv = c->file.view = chunk_file_view_init(); + } + else if (MAP_FAILED != cfv->mptr) + munmap(cfv->mptr, (size_t)cfv->mlen); + /*cfv->mptr= MAP_FAILED;*//*(assigned below)*/ + + if (c->file.fd < 0 && 0 != chunk_open_file_chunk(c, errh)) { + c->file.view = chunk_file_view_failed(cfv); + return NULL; + } + + cfv->foff = mmap_align_offset(c->offset); + + if (0 != n) { + cfv->mlen = c->offset - cfv->foff + n; + #if !(defined(_LP64) || defined(__LP64__) || defined(_WIN64)) + /*(consider 512k blocks if this func is used more generically)*/ + const off_t mmap_chunk_size = 8 * 1024 * 1024; + if (cfv->mlen > mmap_chunk_size) + cfv->mlen = mmap_chunk_size; + #endif + } + else + cfv->mlen = MMAP_CHUNK_SIZE; + /* XXX: 64-bit might consider larger min block size, or even entire file */ + if (cfv->mlen < MMAP_CHUNK_SIZE) + cfv->mlen = MMAP_CHUNK_SIZE; + if (cfv->mlen > c->file.length - cfv->foff) + cfv->mlen = c->file.length - cfv->foff; + + cfv->mptr = mmap(NULL, (size_t)cfv->mlen, PROT_READ, + c->file.is_temp ? MAP_PRIVATE : chunk_mmap_flags, + c->file.fd, cfv->foff); + + if (__builtin_expect( (MAP_FAILED == cfv->mptr), 0)) { + if (__builtin_expect( (errno == EINVAL), 0)) { + chunk_mmap_flags &= ~MAP_SHARED; + chunk_mmap_flags |= MAP_PRIVATE; + cfv->mptr = mmap(NULL, (size_t)cfv->mlen, PROT_READ, + MAP_PRIVATE, c->file.fd, cfv->foff); + } + if (__builtin_expect( (MAP_FAILED == cfv->mptr), 0)) { + c->file.view = chunk_file_view_failed(cfv); + return NULL; + } + } + + #if 0 /*(review callers before changing; some expect open file)*/ + /* close() fd as soon as fully mmap() rather than when done w/ chunk + * (possibly worthwhile to keep active fd count lower) + * (probably only reasonable if entire file is mapped) */ + if (c->file.is_temp && !c->file.refchg) { + close(c->file.fd); + c->file.fd = -1; + } + #endif + + #if 0 + /* disable madvise unless we find common cases where there is a benefit + * (??? madvise for full mmap length or only for original requested n ???) + * (??? might additional flags param to func to indicate madvise pref ???) + * (??? might experiment with Linux mmap flags MAP_POPULATE|MAP_PRIVATE) + * (??? might experiment with madvise MADV_POPULATE_READ (since Linux 5.14)) + * note: caller might be in better position to know if starting an mmap + * which will be flushed in entirety, and perform madvise at that point, + * perhaps with MADV_SEQUENTIAL */ + #ifdef HAVE_MADVISE + if (cfv->mlen > 65536) /*(skip syscall if size <= 64KB)*/ + (void)madvise(cfv->mptr, (size_t)cfv->mlen, MADV_WILLNEED); + #endif + #endif + + return cfv; +} + +#endif /* HAVE_MMAP */ diff --git a/src/chunk.h b/src/chunk.h index be16fd88..6deaf6d4 100644 --- a/src/chunk.h +++ b/src/chunk.h @@ -2,10 +2,6 @@ #define _CHUNK_H_ #include "first.h" -#ifdef _AIX /*(AIX might #define mmap mmap64)*/ -#include "sys-mmap.h" -#endif - #include "buffer.h" #include "array.h" #include "fdlog.h" @@ -14,6 +10,13 @@ #define MAX_READ_LIMIT (256*1024) #define MAX_WRITE_LIMIT (256*1024) +typedef struct chunk_file_view { + char *mptr; /* base pointer of mmap'ed area */ + off_t mlen; /* length of mmap'ed area */ + off_t foff; /* offset from the start of the file */ + int refcnt; +} chunk_file_view; + typedef struct chunk { struct chunk *next; enum { MEM_CHUNK, FILE_CHUNK } type; @@ -32,11 +35,9 @@ typedef struct chunk { int fd; int is_temp; /* file is temporary and will be deleted if on cleanup */ - struct { - char *start; /* the start pointer of the mmap'ed area */ - size_t length; /* size of the mmap'ed area */ - off_t offset; /* start is octet away from the start of the file */ - } mmap; + #if defined(HAVE_MMAP) || defined(_WIN32) /*(see local sys-mmap.h)*/ + chunk_file_view *view; + #endif void *ref; void(*refchg)(void *, int); } file; @@ -162,6 +163,7 @@ int chunkqueue_read_data (chunkqueue *cq, char *data, uint32_t dlen, log_error_s buffer * chunkqueue_read_squash (chunkqueue * restrict cq, log_error_st * restrict errh); __attribute_pure__ +__attribute_nonnull__() static inline off_t chunkqueue_length(const chunkqueue *cq); static inline off_t chunkqueue_length(const chunkqueue *cq) { return cq->bytes_in - cq->bytes_out; @@ -173,9 +175,55 @@ void chunkqueue_free(chunkqueue *cq); void chunkqueue_reset(chunkqueue *cq); __attribute_pure__ +__attribute_nonnull__() static inline int chunkqueue_is_empty(const chunkqueue *cq); static inline int chunkqueue_is_empty(const chunkqueue *cq) { return NULL == cq->first; } +const chunk_file_view * chunkqueue_chunk_file_viewadj (chunk *c, off_t n, log_error_st * restrict errh); + +__attribute_pure__ +__attribute_nonnull__() +static inline char * +chunk_file_view_dptr (const chunk_file_view * const cfv, off_t offset); +static inline char * +chunk_file_view_dptr (const chunk_file_view * const cfv, off_t offset) +{ + return cfv->mptr - cfv->foff + offset; +} + +__attribute_pure__ +__attribute_nonnull__() +static inline off_t +chunk_file_view_dlen (const chunk_file_view * const cfv, off_t offset); +static inline off_t +chunk_file_view_dlen (const chunk_file_view * const cfv, off_t offset) +{ + return cfv->mlen + cfv->foff - offset; +} + +static inline const chunk_file_view * +chunkqueue_chunk_file_view (chunk * const c, const off_t n, log_error_st * const restrict errh); +static inline const chunk_file_view * +chunkqueue_chunk_file_view (chunk * const c, const off_t n, log_error_st * const restrict errh) +{ + /*assert(c->type == FILE_CHUNK);*/ + #if defined(HAVE_MMAP) || defined(_WIN32) /*(see local sys-mmap.h)*/ + /* mmap buffer if offset is outside old mmap area or not mapped at all */ + const chunk_file_view * const restrict cfv = c->file.view; + if (NULL == cfv + ? c->file.length - c->offset >= 131072 /* TBD: min chunk size to mmap */ + : (c->offset - cfv->foff < 0 + || chunk_file_view_dlen(cfv, c->offset) < (n ? n : 1))) + return chunkqueue_chunk_file_viewadj(c, n, errh); + return cfv; + #else + UNUSED(c); + UNUSED(n); + UNUSED(errh); + return NULL; + #endif +} + #endif diff --git a/src/mod_deflate.c b/src/mod_deflate.c index 89ca4a95..02202057 100644 --- a/src/mod_deflate.c +++ b/src/mod_deflate.c @@ -101,6 +101,9 @@ #include #include #include "sys-mmap.h" +#ifdef HAVE_MMAP +#include "sys-setjmp.h" +#endif #include "sys-time.h" #include @@ -154,12 +157,6 @@ #undef HAVE_LIBDEFLATE #endif -#if defined HAVE_SYS_MMAN_H && defined HAVE_MMAP && defined ENABLE_MMAP -#define USE_MMAP -#include "sys-mmap.h" -#include "sys-setjmp.h" -#endif - /* request: accept-encoding */ #define HTTP_ACCEPT_ENCODING_IDENTITY BV(0) #define HTTP_ACCEPT_ENCODING_GZIP BV(1) @@ -1427,48 +1424,6 @@ static int deflate_compress_cleanup(request_st * const r, handler_ctx * const hc } -#ifdef USE_MMAP -static int mod_deflate_file_chunk_remap(chunk * const c, const off_t n) -{ - if (c->file.mmap.start == MAP_FAILED - || c->offset < c->file.mmap.offset - || c->offset >= c->file.mmap.offset + (off_t)c->file.mmap.length) { - - /* use large chunks since server blocks while compressing - * (mod_deflate is not recommended for large files) */ - - if (c->file.mmap.start != MAP_FAILED) /*(munmap, remap at new offset)*/ - munmap(c->file.mmap.start, c->file.mmap.length); - c->file.mmap.offset = c->offset & ~(16384-1); /* pagesize multiple */ - c->file.mmap.length = (size_t)(c->offset - c->file.mmap.offset + n); - #if !(defined(_LP64) || defined(__LP64__) || defined(_WIN64)) - /*(consider 512k blocks if this func is used more generically)*/ - const off_t mmap_chunk_size = 8 * 1024 * 1024; - if (c->file.mmap.length > (size_t)mmap_chunk_size) - c->file.mmap.length = (size_t)mmap_chunk_size; - #endif - c->file.mmap.start = - mmap(0, c->file.mmap.length, PROT_READ, - c->file.is_temp ? MAP_PRIVATE : MAP_SHARED, - c->file.fd, c->file.mmap.offset); - if (MAP_FAILED == c->file.mmap.start) { - if (errno != EINVAL) return 0; - c->file.mmap.start = - mmap(0, c->file.mmap.length, PROT_READ, MAP_PRIVATE, - c->file.fd, c->file.mmap.offset); - if (MAP_FAILED == c->file.mmap.start) return 0; - } - #ifdef HAVE_MADVISE - /*(consider func param for madvise if func is used more generically)*/ - if (c->file.mmap.length > 65536) /*(skip syscall if size <= 64KB)*/ - (void)madvise(c->file.mmap.start,c->file.mmap.length,MADV_WILLNEED); - #endif - } - return 1; -} -#endif - - #ifdef HAVE_LIBDEFLATE #include @@ -1533,7 +1488,7 @@ static int mod_deflate_using_libdeflate_sm (handler_ctx * const hctx, const plug } -#ifdef USE_MMAP +#ifdef HAVE_MMAP #if defined(_LP64) || defined(__LP64__) || defined(_WIN64) struct mod_deflate_setjmp_params { @@ -1547,7 +1502,7 @@ static off_t mod_deflate_using_libdeflate_setjmp_cb (void *dst, const void *src, const struct mod_deflate_setjmp_params * const params = dst; const handler_ctx * const hctx = src; const chunk * const c = hctx->r->write_queue.first; - const char *in = c->file.mmap.start + c->offset - c->file.mmap.offset; + const char *in = chunk_file_view_dptr(c->file.view, c->offset); return (off_t)((hctx->compression_type == HTTP_ACCEPT_ENCODING_GZIP) ? libdeflate_gzip_compress(params->compressor, in, (size_t)len, params->out, params->outsz) @@ -1641,7 +1596,7 @@ static int mod_deflate_using_libdeflate (handler_ctx * const hctx, const plugin_ } #endif /* defined(_LP64) || defined(__LP64__) || defined(_WIN64) */ -#endif /* USE_MMAP */ +#endif /* HAVE_MMAP */ #endif /* HAVE_LIBDEFLATE */ @@ -1666,7 +1621,7 @@ static off_t mod_deflate_file_chunk_no_mmap(request_st * const r, handler_ctx * } -#ifdef USE_MMAP +#ifdef ENABLE_MMAP static off_t mod_deflate_file_chunk_setjmp_cb (void *dst, const void *src, off_t len) { @@ -1676,12 +1631,18 @@ static off_t mod_deflate_file_chunk_setjmp_cb (void *dst, const void *src, off_t static off_t mod_deflate_file_chunk_mmap(request_st * const r, handler_ctx * const hctx, chunk * const c, off_t n) { - if (!mod_deflate_file_chunk_remap(c, n)) + /* n is length of entire file since server blocks while compressing + * (mod_deflate is not recommended for large files; + * mod_deflate default upper limit is 128MB; deflate.max-compress-size) */ + + const chunk_file_view * const restrict cfv = (!c->file.is_temp) + ? chunkqueue_chunk_file_view(c, n, r->conf.errh) + : NULL; + if (NULL == cfv) return mod_deflate_file_chunk_no_mmap(r, hctx, c, n); - const off_t moffset = c->offset - c->file.mmap.offset; - char * const p = c->file.mmap.start + moffset; - off_t len = c->file.mmap.length - moffset; + const char * const p = chunk_file_view_dptr(cfv, c->offset); + off_t len = chunk_file_view_dlen(cfv, c->offset); if (len > n) len = n; off_t rc = sys_setjmp_eval3(mod_deflate_file_chunk_setjmp_cb, hctx, p, len); if (__builtin_expect( (rc < 0), 0)) { @@ -1705,7 +1666,7 @@ static off_t mod_deflate_file_chunk(request_st * const r, handler_ctx * const hc return -1; } } - #ifdef USE_MMAP + #ifdef ENABLE_MMAP return mod_deflate_file_chunk_mmap(r, hctx, c, n); #else return mod_deflate_file_chunk_no_mmap(r, hctx, c, n); @@ -2097,16 +2058,18 @@ REQUEST_FUNC(mod_deflate_handle_response_start) { #ifdef HAVE_LIBDEFLATE chunk * const c = r->write_queue.first; /*(invalid after compression)*/ - #ifdef USE_MMAP + #ifdef HAVE_MMAP #if defined(_LP64) || defined(__LP64__) || defined(_WIN64) /* optimization to compress single file in one-shot to writeable mmap */ /*(future: might extend to other compression types)*/ - if (len > 65536 /* XXX: TBD what should min size be for this optimization?*/ + /*(chunkqueue_chunk_file_view() current min size for mmap is 128k)*/ + if (len > 131072 /* XXX: TBD what should min size be for optimization?*/ && (hctx->compression_type == HTTP_ACCEPT_ENCODING_GZIP || hctx->compression_type == HTTP_ACCEPT_ENCODING_DEFLATE) && c == r->write_queue.last && c->type == FILE_CHUNK - && mod_deflate_file_chunk_remap(c, len)) { + && chunkqueue_chunk_file_view(c, len, r->conf.errh) + && chunk_file_view_dlen(c->file.view, c->offset) >= len) { /*(cfv)*/ rc = HANDLER_GO_ON; hctx->bytes_in = len; if (mod_deflate_using_libdeflate(hctx, p)) { diff --git a/src/mod_webdav.c b/src/mod_webdav.c index 7b80438a..930c166b 100644 --- a/src/mod_webdav.c +++ b/src/mod_webdav.c @@ -739,9 +739,6 @@ typedef struct { #define MOD_WEBDAV_XMLNS_NS0 "xmlns:ns0=\"urn:uuid:c2f41010-65b3-11d1-a29f-00aa00c14882/\"" -static char * -webdav_mmap_file_chunk (chunk * const c); - __attribute_cold__ __attribute_noinline__ static void @@ -761,8 +758,19 @@ webdav_xml_log_response (request_st * const r) break; case FILE_CHUNK: /*(safe to mmap tempfiles from response XML)*/ - s = webdav_mmap_file_chunk(c); - len = (uint32_t)c->file.length; + /*(response body provided in temporary file, so ok to mmap(). + * Otherwise, must access through sys_setjmp_eval3()) */ + /*(tempfiles (and xml response) should easily fit in uint32_t + * and are not expected to already be mmap'd. Avoid >= 128k + * requirement of chunkqueue_chunk_file_view() by using viewadj)*/ + len = (uint32_t)(c->file.length - c->offset); + { + const chunk_file_view * const restrict cfv = + chunkqueue_chunk_file_viewadj(c, (off_t)len, r->conf.errh); + s = (cfv && chunk_file_view_dlen(cfv, c->offset) >= (off_t)len) + ? chunk_file_view_dptr(cfv, c->offset) + : NULL; + } if (s == NULL) continue; break; default: @@ -3725,73 +3733,29 @@ webdav_propfind_dir (webdav_propfind_bufs * const restrict pb) } -static int -webdav_open_chunk_file_rd (chunk * const c) -{ - if (c->file.fd < 0) /* open file if not already open *//*permit symlink*/ - c->file.fd = fdevent_open_cloexec(c->mem->ptr, 1, O_RDONLY, 0); - return c->file.fd; -} - - -static int -webdav_mmap_file_rd (void ** const addr, const size_t length, - const int fd, const off_t offset) -{ - /*(caller must ensure offset is properly aligned to mmap requirements)*/ - - if (0 == length) { - *addr = NULL; /*(something other than MAP_FAILED)*/ - return 0; - } - - #if defined(HAVE_MMAP) || defined(_WIN32) /*(see local sys-mmap.h)*/ - - *addr = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, offset); - if (*addr == MAP_FAILED && errno == EINVAL) - *addr = mmap(NULL, length, PROT_READ, MAP_PRIVATE, fd, offset); - return (*addr != MAP_FAILED ? 0 : -1); - - #else - - UNUSED(fd); - UNUSED(offset); - return -1; - - #endif -} - +#if defined(USE_PROPPATCH) || defined(USE_LOCKS) static char * -webdav_mmap_file_chunk (chunk * const c) +webdav_mmap_file_chunk (chunk * const c, log_error_st * const errh) { + #ifdef HAVE_MMAP /*(request body provided in temporary file, so ok to mmap(). - * Otherwise, must check defined(ENABLE_MMAP)) */ - /* chunk_reset() or chunk_free() will clean up mmap'd chunk */ - /* close c->file.fd only after mmap() succeeds, since it will not - * be able to be re-opened if it was a tmpfile that was unlinked */ + * Otherwise, must access through sys_setjmp_eval3()) */ /*assert(c->type == FILE_CHUNK);*/ - if (MAP_FAILED != c->file.mmap.start) - return c->file.mmap.start + c->offset - c->file.mmap.offset; - - if (webdav_open_chunk_file_rd(c) < 0) - return NULL; - - webdav_mmap_file_rd((void **)&c->file.mmap.start, (size_t)c->file.length, - c->file.fd, 0); - - if (MAP_FAILED == c->file.mmap.start) - return NULL; - - close(c->file.fd); - c->file.fd = -1; - c->file.mmap.length = c->file.length; - return c->file.mmap.start + c->offset - c->file.mmap.offset; + const off_t len = c->file.length - c->offset; + const chunk_file_view * const restrict cfv = + chunkqueue_chunk_file_view(c, len, errh); + return (cfv && chunk_file_view_dlen(cfv, c->offset) >= len) + ? chunk_file_view_dptr(cfv, c->offset) + : NULL; + #else + UNUSED(c); + UNUSED(errh); + return NULL; + #endif } -#if defined(USE_PROPPATCH) || defined(USE_LOCKS) - __attribute_noinline__ static xmlDoc * webdav_parse_chunkqueue (request_st * const r, @@ -3820,8 +3784,7 @@ webdav_parse_chunkqueue (request_st * const r, weHave = buffer_clen(c->mem) - c->offset; } else if (c->type == FILE_CHUNK) { - xmlstr = webdav_mmap_file_chunk(c); - /*xmlstr = c->file.mmap.start + c->offset - c->file.mmap.offset;*/ + xmlstr = webdav_mmap_file_chunk(c, r->conf.errh); if (NULL != xmlstr) { weHave = c->file.length - c->offset; } diff --git a/src/network_write.c b/src/network_write.c index 5dce2a0b..39bd59eb 100644 --- a/src/network_write.c +++ b/src/network_write.c @@ -63,9 +63,11 @@ # define NETWORK_WRITE_USE_WRITEV #endif -#if defined HAVE_SYS_MMAN_H && defined HAVE_MMAP && defined ENABLE_MMAP +#if defined(HAVE_MMAP) || defined(_WIN32) /*(see local sys-mmap.h)*/ +#ifdef ENABLE_MMAP # define NETWORK_WRITE_USE_MMAP #endif +#endif __attribute_cold__ @@ -182,85 +184,8 @@ static int network_write_file_chunk_no_mmap(const int fd, chunkqueue * const cq, #if defined(NETWORK_WRITE_USE_MMAP) -#include "sys-mmap.h" #include "sys-setjmp.h" -#define MMAP_CHUNK_SIZE (512*1024) - -static off_t mmap_align_offset(off_t start) { - static long pagesize = 0; - if (0 == pagesize) { - pagesize = sysconf(_SC_PAGESIZE); - force_assert(pagesize < MMAP_CHUNK_SIZE); - } - force_assert(start >= (start % pagesize)); - return start - (start % pagesize); -} - -static int -network_write_file_chunk_remap(chunk * const c) -{ - /* mmap buffer if offset is outside old mmap area or not mapped at all */ - if (MAP_FAILED == c->file.mmap.start - || c->offset < c->file.mmap.offset - || c->offset >= (off_t)(c->file.mmap.offset + c->file.mmap.length)) { - - if (MAP_FAILED != c->file.mmap.start) { - munmap(c->file.mmap.start, c->file.mmap.length); - c->file.mmap.start = MAP_FAILED; - } - - /* Optimizations for the future: - * - * adaptive mem-mapping - * the problem: - * we mmap() the whole file. If someone has a lot of large files and - * 32-bit machine the virtual address area will be exhausted and we - * will have a failing mmap() call. - * solution: - * only mmap 16M in one chunk and move the window as soon as we have - * finished the first 8M - * - * read-ahead buffering - * the problem: - * sending out several large files in parallel trashes read-ahead - * of the kernel leading to long wait-for-seek times. - * solutions: (increasing complexity) - * 1. use madvise - * 2. use a internal read-ahead buffer in the chunk-structure - * 3. use non-blocking IO for file-transfers - * */ - - c->file.mmap.offset = mmap_align_offset(c->offset); - - /* all mmap()ed areas are MMAP_CHUNK_SIZE - * except the last which might be smaller */ - c->file.mmap.length = MMAP_CHUNK_SIZE; - if (c->file.mmap.offset > c->file.length - (off_t)c->file.mmap.length) { - c->file.mmap.length = c->file.length - c->file.mmap.offset; - } - - c->file.mmap.start = mmap(NULL, c->file.mmap.length, PROT_READ, - MAP_SHARED, c->file.fd, c->file.mmap.offset); - if (MAP_FAILED == c->file.mmap.start) { - return 0; - } - - #if defined(HAVE_MADVISE) - /* don't advise files < 64Kb */ - if (c->file.mmap.length > (64*1024)) { - /* darwin 7 is returning EINVAL all the time and I don't know how to - * detect this at runtime. - * - * ignore the return value for now */ - madvise(c->file.mmap.start, c->file.mmap.length, MADV_WILLNEED); - } - #endif - } - return 1; -} - - static off_t network_write_setjmp_write_cb (void *fd, const void *data, off_t len) { @@ -269,33 +194,22 @@ network_write_setjmp_write_cb (void *fd, const void *data, off_t len) /* next chunk must be FILE_CHUNK. send mmap()ed file with write() */ static int network_write_file_chunk_mmap(const int fd, chunkqueue * const cq, off_t * const p_max_bytes, log_error_st * const errh) { - chunk* const c = cq->first; + chunk * const restrict c = cq->first; + const chunk_file_view * const restrict cfv = (!c->file.is_temp) + ? chunkqueue_chunk_file_view(cq->first, 0, errh)/*use default 512k block*/ + : NULL; + if (NULL == cfv) + return network_write_file_chunk_no_mmap(fd, cq, p_max_bytes, errh); + off_t toSend = c->file.length - c->offset; if (toSend > *p_max_bytes) toSend = *p_max_bytes; if (toSend <= 0) return network_remove_finished_chunks(cq, toSend); - if (c->file.fd < 0 && 0 != chunkqueue_open_file_chunk(cq, errh)) return -1; - - if (!network_write_file_chunk_remap(c)) - return network_write_file_chunk_no_mmap(fd, cq, p_max_bytes, errh); - - off_t mmap_offset = c->offset - c->file.mmap.offset; - off_t mmap_avail = (off_t)c->file.mmap.length - mmap_offset; - const char * const data = c->file.mmap.start + mmap_offset; + const off_t mmap_avail = chunk_file_view_dlen(cfv, c->offset); + const char * const data = chunk_file_view_dptr(cfv, c->offset); if (toSend > mmap_avail) toSend = mmap_avail; off_t wr = sys_setjmp_eval3(network_write_setjmp_write_cb, (void *)(uintptr_t)fd, data, toSend); - #if 0 - /*(future: might writev() with multiple maps, - * so will not know which chunk failed)*/ - if (wr < 0 && errno == EFAULT) { /*(see sys-setjmp.c)*/ - log_error(errh, __FILE__, __LINE__, - "SIGBUS in mmap: %s %d", c->mem->ptr, c->file.fd); - munmap(c->file.mmap.start, c->file.mmap.length); - c->file.mmap.start = MAP_FAILED; - return -1; - } - #endif return network_write_accounting(fd,cq,p_max_bytes,errh,(ssize_t)wr,toSend); } diff --git a/src/sys-mmap.h b/src/sys-mmap.h index f5edfdc2..bf8e815d 100644 --- a/src/sys-mmap.h +++ b/src/sys-mmap.h @@ -26,6 +26,8 @@ #define PROT_READ PAGE_READONLY #endif +#define HAVE_MMAP 1 + #define munmap(addr, length) UnmapViewOfFile((LPCVOID)(addr)) static inline void * -- cgit v1.2.1