diff options
author | Jim Jagielski <jim@apache.org> | 2017-04-13 11:45:31 +0000 |
---|---|---|
committer | Jim Jagielski <jim@apache.org> | 2017-04-13 11:45:31 +0000 |
commit | 193f939684e4bc5c0eed1607b379738d6ef878e1 (patch) | |
tree | 36e4a1aad7ed66af296366b4e81a3e8969ab8726 | |
parent | 59b99d12dde7c28f2280754e9e6a3e29ab26d5d0 (diff) | |
download | httpd-193f939684e4bc5c0eed1607b379738d6ef878e1.tar.gz |
Merge r1790852, r1790853, r1790860 from trunk:
Merge r1761714, r1762512, r1762515, r1771791, r1779077, r1779091, r1779699, r1790852, r1790853, r1790860 from trunk:
mod_brotli: Add initial implementation.
This new module supports dynamic Brotli (RFC 7932) compression. Existing
mod_deflate installations can benefit from better compression ratio by
sending Brotli-compressed data to the clients that support it:
SetOutputFilter BROTLI_COMPRESS;DEFLATE
The module features zero-copy processing, which is only possible with the
new API from the upcoming 1.0.x series of brotli [1]. The Linux makefile
works against libbrotli [2], as currently the core brotli repository doesn't
offer a way to build a library [3]. Apart from that, only the CMake build
is now supported.
[1] https://github.com/google/brotli
[2] https://github.com/bagder/libbrotli
[3] https://github.com/google/brotli/pull/332
mod_brotli: Allow compression ratio logging with new BrotliFilterNote
directive.
mod_brotli: Handle new 'no-brotli' internal environment variable that
disables Brotli compression for a particular request.
This mimicks how mod_deflate handles the 'no-gzip' env variable, and
should allow seamless migration for configurations that use it.
mod_brotli: Explicitly cast 'const uint8_t *' to 'const char *' when using
the data received from Brotli to create a bucket.
This fixes a /W4 warning in my environment, and should also allow building
mod_brotli on NetWare.
Submitted by: NormW <normw gknw.net>
unused variable error could mistakenly note that brotli isn't available.
1st draft
Be more consitent in the layout, and fix the display of a multi lines <highlight> section
mod_brotli: Update makefile to allow using Brotli library >= 0.6.0.
The 0.6.0 version has just been released [1], and it contains the
necessary API required for mod_brotli.
[1] https://github.com/google/brotli/releases/tag/v0.6.0
mod_brotli: Fix a minor typo in the description of BrotliAlterETag
that has been referring to httpd 2.2.x.
There's no mod_brotli in 2.2.x.
mod_brotli: Comment on the default choice (0) for BROTLI_PARAM_LGBLOCK.
Submitted by: kotkov, jim, jim, jailletc36, kotkov, kotkov, kotkov
Reviewed by: jim, jorton, icing
mod_brotli: Update makefile to allow using Brotli library >= 0.6.0.
The 0.6.0 version has just been released [1], and it contains the
necessary API required for mod_brotli.
[1] https://github.com/google/brotli/releases/tag/v0.6.0
mod_brotli: Fix a minor typo in the description of BrotliAlterETag
that has been referring to httpd 2.2.x.
There's no mod_brotli in 2.2.x.
mod_brotli: Comment on the default choice (0) for BROTLI_PARAM_LGBLOCK.
Submitted by: kotkov
Reviewed by: jim, jorton, icing
git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/branches/2.4.x@1791231 13f79535-47bb-0310-9956-ffa450edef68
-rw-r--r-- | CHANGES | 3 | ||||
-rw-r--r-- | CMakeLists.txt | 29 | ||||
-rw-r--r-- | STATUS | 16 | ||||
-rw-r--r-- | modules/filters/config.m4 | 122 | ||||
-rw-r--r-- | modules/filters/mod_brotli.c | 592 |
5 files changed, 746 insertions, 16 deletions
@@ -2,6 +2,9 @@ Changes with Apache 2.4.26 + *) mod_brotli: Add a new module for dynamic Brotli (RFC 7932) compression. + [Evgeny Kotkov] + *) mod_proxy_http2: Fixed bug in re-attempting proxy requests after connection error. Reliability of reconnect handling improved. [Stefan Eissing] diff --git a/CMakeLists.txt b/CMakeLists.txt index 6e8a6a1e21..373bd708fe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -58,6 +58,12 @@ ELSE() SET(default_nghttp2_libraries "${CMAKE_INSTALL_PREFIX}/lib/nghttp2.lib") ENDIF() +IF(EXISTS "${CMAKE_INSTALL_PREFIX}/lib/brotli_enc.lib") + SET(default_brotli_libraries "${CMAKE_INSTALL_PREFIX}/lib/brotli_enc.lib" "${CMAKE_INSTALL_PREFIX}/lib/brotli_common.lib") +ELSE() + SET(default_brotli_libraries) +ENDIF() + SET(APR_INCLUDE_DIR "${CMAKE_INSTALL_PREFIX}/include" CACHE STRING "Directory with APR[-Util] include files") SET(APR_LIBRARIES ${default_apr_libraries} CACHE STRING "APR libraries to link with") SET(NGHTTP2_INCLUDE_DIR "${CMAKE_INSTALL_PREFIX}/include" CACHE STRING "Directory with NGHTTP2 include files within nghttp2 subdirectory") @@ -66,6 +72,8 @@ SET(PCRE_INCLUDE_DIR "${CMAKE_INSTALL_PREFIX}/include" CACHE STRING "Direct SET(PCRE_LIBRARIES ${default_pcre_libraries} CACHE STRING "PCRE libraries to link with") SET(LIBXML2_ICONV_INCLUDE_DIR "" CACHE STRING "Directory with iconv include files for libxml2") SET(LIBXML2_ICONV_LIBRARIES "" CACHE STRING "iconv libraries to link with for libxml2") +SET(BROTLI_INCLUDE_DIR "${CMAKE_INSTALL_PREFIX}/include" CACHE STRING "Directory with include files for Brotli") +SET(BROTLI_LIBRARIES ${default_brotli_libraries} CACHE STRING "Brotli libraries to link with") # end support library configuration # Misc. options @@ -179,6 +187,18 @@ ELSE() SET(NGHTTP2_FOUND FALSE) ENDIF() +# See if we have Brotli +SET(BROTLI_FOUND TRUE) +IF(EXISTS "${BROTLI_INCLUDE_DIR}/brotli/encode.h") + FOREACH(onelib ${BROTLI_LIBRARIES}) + IF(NOT EXISTS ${onelib}) + SET(BROTLI_FOUND FALSE) + ENDIF() + ENDFOREACH() +ELSE() + SET(BROTLI_FOUND FALSE) +ENDIF() + MESSAGE(STATUS "") MESSAGE(STATUS "Summary of feature detection:") MESSAGE(STATUS "") @@ -187,6 +207,7 @@ MESSAGE(STATUS "LUA51_FOUND .............. : ${LUA51_FOUND}") MESSAGE(STATUS "NGHTTP2_FOUND ............ : ${NGHTTP2_FOUND}") MESSAGE(STATUS "OPENSSL_FOUND ............ : ${OPENSSL_FOUND}") MESSAGE(STATUS "ZLIB_FOUND ............... : ${ZLIB_FOUND}") +MESSAGE(STATUS "BROTLI_FOUND ............. : ${BROTLI_FOUND}") MESSAGE(STATUS "APR_HAS_LDAP ............. : ${APR_HAS_LDAP}") MESSAGE(STATUS "APR_HAS_XLATE ............ : ${APR_HAS_XLATE}") MESSAGE(STATUS "APU_HAVE_CRYPTO .......... : ${APU_HAVE_CRYPTO}") @@ -258,6 +279,7 @@ SET(MODULE_LIST "modules/examples/mod_case_filter_in+O+Example uppercase conversion input filter" "modules/examples/mod_example_hooks+O+Example hook callback handler module" "modules/examples/mod_example_ipc+O+Example of shared memory and mutex usage" + "modules/filters/mod_brotli+i+Brotli compression support" "modules/filters/mod_buffer+I+Filter Buffering" "modules/filters/mod_charset_lite+i+character set translation" "modules/filters/mod_data+O+RFC2397 data encoder" @@ -376,6 +398,11 @@ IF(ZLIB_FOUND) SET(mod_deflate_extra_includes ${ZLIB_INCLUDE_DIR}) SET(mod_deflate_extra_libs ${ZLIB_LIBRARIES}) ENDIF() +SET(mod_brotli_requires BROTLI_FOUND) +IF(BROTLI_FOUND) + SET(mod_brotli_extra_includes ${BROTLI_INCLUDE_DIR}) + SET(mod_brotli_extra_libs ${BROTLI_LIBRARIES}) +ENDIF() SET(mod_firehose_requires SOMEONE_TO_MAKE_IT_COMPILE_ON_WINDOWS) SET(mod_heartbeat_extra_libs mod_watchdog) SET(mod_http2_requires NGHTTP2_FOUND) @@ -922,6 +949,8 @@ MESSAGE(STATUS " PCRE include directory .......... : ${PCRE_INCLUDE_DIR}") MESSAGE(STATUS " PCRE libraries .................. : ${PCRE_LIBRARIES}") MESSAGE(STATUS " libxml2 iconv prereq include dir. : ${LIBXML2_ICONV_INCLUDE_DIR}") MESSAGE(STATUS " libxml2 iconv prereq libraries .. : ${LIBXML2_ICONV_LIBRARIES}") +MESSAGE(STATUS " Brotli include directory......... : ${BROTLI_INCLUDE_DIR}") +MESSAGE(STATUS " Brotli libraries ................ : ${BROTLI_LIBRARIES}") MESSAGE(STATUS " Extra include directories ....... : ${EXTRA_INCLUDES}") MESSAGE(STATUS " Extra compile flags ............. : ${EXTRA_COMPILE_FLAGS}") MESSAGE(STATUS " Extra libraries ................. : ${EXTRA_LIBS}") @@ -138,22 +138,6 @@ RELEASE SHOWSTOPPERS: PATCHES ACCEPTED TO BACKPORT FROM TRUNK: [ start all new proposals below, under PATCHES PROPOSED. ] - *) mod_brotli: Backport of mod_brotli filter - trunk patch: http://svn.apache.org/r1761714 - http://svn.apache.org/r1762512 - http://svn.apache.org/r1762515 - http://svn.apache.org/r1771791 - http://svn.apache.org/r1779077 - http://svn.apache.org/r1779091 - http://svn.apache.org/r1779699 - http://svn.apache.org/r1790852 - http://svn.apache.org/r1790853 - http://svn.apache.org/r1790860 - 2.4.x patch: http://home.apache.org/~jim/patches/brotli-2.4.patch - http://svn.apache.org/r1790852 - http://svn.apache.org/r1790853 - http://svn.apache.org/r1790860 - +1: jim, jorton, icing PATCHES PROPOSED TO BACKPORT FROM TRUNK: [ New proposals should be added at the end of the list ] diff --git a/modules/filters/config.m4 b/modules/filters/config.m4 index 60917edadb..3b57b5a0b6 100644 --- a/modules/filters/config.m4 +++ b/modules/filters/config.m4 @@ -140,6 +140,128 @@ APACHE_MODULE(proxy_html, Fix HTML Links in a Reverse Proxy, , , , [ ] ) +dnl +dnl APACHE_CHECK_BROTLI +dnl +dnl Configure for Brotli, giving preference to +dnl "--with-brotli=<path>" if it was specified. +dnl +AC_DEFUN([APACHE_CHECK_BROTLI],[ + AC_CACHE_CHECK([for Brotli], [ac_cv_brotli], [ + dnl initialise the variables we use + ac_cv_brotli=no + ac_brotli_found="" + ac_brotli_base="" + ac_brotli_libs="" + ac_brotli_mod_cflags="" + ac_brotli_mod_ldflags="" + + dnl Determine the Brotli base directory, if any + AC_MSG_CHECKING([for user-provided Brotli base directory]) + AC_ARG_WITH(brotli, APACHE_HELP_STRING(--with-brotli=PATH,Brotli installation directory), [ + dnl If --with-brotli specifies a directory, we use that directory + if test "x$withval" != "xyes" -a "x$withval" != "x"; then + dnl This ensures $withval is actually a directory and that it is absolute + ac_brotli_base="`cd $withval ; pwd`" + fi + ]) + if test "x$ac_brotli_base" = "x"; then + AC_MSG_RESULT(none) + else + AC_MSG_RESULT($ac_brotli_base) + fi + + dnl Run header and version checks + saved_CPPFLAGS="$CPPFLAGS" + saved_LIBS="$LIBS" + saved_LDFLAGS="$LDFLAGS" + + dnl Before doing anything else, load in pkg-config variables + if test -n "$PKGCONFIG"; then + saved_PKG_CONFIG_PATH="$PKG_CONFIG_PATH" + if test "x$ac_brotli_base" != "x" -a \ + -f "${ac_brotli_base}/lib/pkgconfig/libbrotlienc.pc"; then + dnl Ensure that the given path is used by pkg-config too, otherwise + dnl the system libbrotlienc.pc might be picked up instead. + PKG_CONFIG_PATH="${ac_brotli_base}/lib/pkgconfig${PKG_CONFIG_PATH+:}${PKG_CONFIG_PATH}" + export PKG_CONFIG_PATH + fi + ac_brotli_libs="`$PKGCONFIG --libs-only-l --silence-errors libbrotlienc`" + if test $? -eq 0; then + ac_brotli_found="yes" + pkglookup="`$PKGCONFIG --cflags-only-I libbrotlienc`" + APR_ADDTO(CPPFLAGS, [$pkglookup]) + APR_ADDTO(MOD_CFLAGS, [$pkglookup]) + pkglookup="`$PKGCONFIG --libs-only-L libbrotlienc`" + APR_ADDTO(LDFLAGS, [$pkglookup]) + APR_ADDTO(MOD_LDFLAGS, [$pkglookup]) + pkglookup="`$PKGCONFIG --libs-only-other libbrotlienc`" + APR_ADDTO(LDFLAGS, [$pkglookup]) + APR_ADDTO(MOD_LDFLAGS, [$pkglookup]) + fi + PKG_CONFIG_PATH="$saved_PKG_CONFIG_PATH" + fi + + dnl fall back to the user-supplied directory if not found via pkg-config + if test "x$ac_brotli_base" != "x" -a "x$ac_brotli_found" = "x"; then + APR_ADDTO(CPPFLAGS, [-I$ac_brotli_base/include]) + APR_ADDTO(MOD_CFLAGS, [-I$ac_brotli_base/include]) + APR_ADDTO(LDFLAGS, [-L$ac_brotli_base/lib]) + APR_ADDTO(MOD_LDFLAGS, [-L$ac_brotli_base/lib]) + if test "x$ap_platform_runtime_link_flag" != "x"; then + APR_ADDTO(LDFLAGS, [$ap_platform_runtime_link_flag$ac_brotli_base/lib]) + APR_ADDTO(MOD_LDFLAGS, [$ap_platform_runtime_link_flag$ac_brotli_base/lib]) + fi + fi + + ac_brotli_libs="${ac_brotli_libs:--lbrotlienc `$apr_config --libs`} " + APR_ADDTO(MOD_LDFLAGS, [$ac_brotli_libs]) + APR_ADDTO(LIBS, [$ac_brotli_libs]) + + dnl Run library and function checks + liberrors="" + AC_CHECK_HEADERS([brotli/encode.h]) + AC_MSG_CHECKING([for Brotli version >= 0.6.0]) + AC_TRY_COMPILE([#include <brotli/encode.h>],[ +const uint8_t *o = BrotliEncoderTakeOutput((BrotliEncoderState*)0, (size_t*)0); +if (o) return *o;], + [AC_MSG_RESULT(OK) + ac_cv_brotli="yes"], + [AC_MSG_RESULT(FAILED)]) + + dnl restore + CPPFLAGS="$saved_CPPFLAGS" + LIBS="$saved_LIBS" + LDFLAGS="$saved_LDFLAGS" + + dnl cache MOD_LDFLAGS, MOD_CFLAGS + ac_brotli_mod_cflags=$MOD_CFLAGS + ac_brotli_mod_ldflags=$MOD_LDFLAGS + ]) + if test "x$ac_cv_brotli" = "xyes"; then + APR_ADDTO(MOD_LDFLAGS, [$ac_brotli_mod_ldflags]) + + dnl Ouch! libbrotlienc.1.so doesn't link against libm.so (-lm), + dnl although it should. Workaround that in our LDFLAGS: + + APR_ADDTO(MOD_LDFLAGS, ["-lm"]) + APR_ADDTO(MOD_CFLAGS, [$ac_brotli_mod_cflags]) + fi +]) + +APACHE_MODULE(brotli, Brotli compression support, , , most, [ + APACHE_CHECK_BROTLI + if test "$ac_cv_brotli" = "yes" ; then + if test "x$enable_brotli" = "xshared"; then + # The only symbol which needs to be exported is the module + # structure, so ask libtool to hide everything else: + APR_ADDTO(MOD_BROTLI_LDADD, [-export-symbols-regex brotli_module]) + fi + else + enable_brotli=no + fi +]) + APR_ADDTO(INCLUDES, [-I\$(top_srcdir)/$modpath_current]) APACHE_MODPATH_FINISH diff --git a/modules/filters/mod_brotli.c b/modules/filters/mod_brotli.c new file mode 100644 index 0000000000..b2ab8c6bd0 --- /dev/null +++ b/modules/filters/mod_brotli.c @@ -0,0 +1,592 @@ +/* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "httpd.h" +#include "http_core.h" +#include "http_log.h" +#include "apr_strings.h" + +#include <brotli/encode.h> + +module AP_MODULE_DECLARE_DATA brotli_module; + +typedef enum { + ETAG_MODE_ADDSUFFIX = 0, + ETAG_MODE_NOCHANGE = 1, + ETAG_MODE_REMOVE = 2 +} etag_mode_e; + +typedef struct brotli_server_config_t { + int quality; + int lgwin; + int lgblock; + etag_mode_e etag_mode; + const char *note_ratio_name; + const char *note_input_name; + const char *note_output_name; +} brotli_server_config_t; + +static void *create_server_config(apr_pool_t *p, server_rec *s) +{ + brotli_server_config_t *conf = apr_pcalloc(p, sizeof(*conf)); + + /* These default values allow mod_brotli to behave similarly to + * mod_deflate in terms of compression speed and memory usage. + * + * The idea is that since Brotli (generally) gives better compression + * ratio than Deflate, simply enabling mod_brotli on the server + * will reduce the amount of transferred data while keeping everything + * else unchanged. See https://quixdb.github.io/squash-benchmark/ + */ + conf->quality = 5; + conf->lgwin = 18; + /* Zero is a special value for BROTLI_PARAM_LGBLOCK that allows + * Brotli to automatically select the optimal input block size based + * on other encoder parameters. See enc/quality.h: ComputeLgBlock(). + */ + conf->lgblock = 0; + conf->etag_mode = ETAG_MODE_ADDSUFFIX; + + return conf; +} + +static const char *set_filter_note(cmd_parms *cmd, void *dummy, + const char *arg1, const char *arg2) +{ + brotli_server_config_t *conf = + ap_get_module_config(cmd->server->module_config, &brotli_module); + + if (!arg2) { + conf->note_ratio_name = arg1; + return NULL; + } + + if (ap_cstr_casecmp(arg1, "Ratio") == 0) { + conf->note_ratio_name = arg2; + } + else if (ap_cstr_casecmp(arg1, "Input") == 0) { + conf->note_input_name = arg2; + } + else if (ap_cstr_casecmp(arg1, "Output") == 0) { + conf->note_output_name = arg2; + } + else { + return apr_psprintf(cmd->pool, "Unknown BrotliFilterNote type '%s'", + arg1); + } + + return NULL; +} + +static const char *set_compression_quality(cmd_parms *cmd, void *dummy, + const char *arg) +{ + brotli_server_config_t *conf = + ap_get_module_config(cmd->server->module_config, &brotli_module); + int val = atoi(arg); + + if (val < 0 || val > 11) { + return "BrotliCompressionQuality must be between 0 and 11"; + } + + conf->quality = val; + return NULL; +} + +static const char *set_compression_lgwin(cmd_parms *cmd, void *dummy, + const char *arg) +{ + brotli_server_config_t *conf = + ap_get_module_config(cmd->server->module_config, &brotli_module); + int val = atoi(arg); + + if (val < 10 || val > 24) { + return "BrotliCompressionWindow must be between 10 and 24"; + } + + conf->lgwin = val; + return NULL; +} + +static const char *set_compression_lgblock(cmd_parms *cmd, void *dummy, + const char *arg) +{ + brotli_server_config_t *conf = + ap_get_module_config(cmd->server->module_config, &brotli_module); + int val = atoi(arg); + + if (val < 16 || val > 24) { + return "BrotliCompressionMaxInputBlock must be between 16 and 24"; + } + + conf->lgblock = val; + return NULL; +} + +static const char *set_etag_mode(cmd_parms *cmd, void *dummy, + const char *arg) +{ + brotli_server_config_t *conf = + ap_get_module_config(cmd->server->module_config, &brotli_module); + + if (ap_cstr_casecmp(arg, "AddSuffix") == 0) { + conf->etag_mode = ETAG_MODE_ADDSUFFIX; + } + else if (ap_cstr_casecmp(arg, "NoChange") == 0) { + conf->etag_mode = ETAG_MODE_NOCHANGE; + } + else if (ap_cstr_casecmp(arg, "Remove") == 0) { + conf->etag_mode = ETAG_MODE_REMOVE; + } + else { + return "BrotliAlterETag accepts only 'AddSuffix', 'NoChange' and 'Remove'"; + } + + return NULL; +} + +typedef struct brotli_ctx_t { + BrotliEncoderState *state; + apr_bucket_brigade *bb; + apr_off_t total_in; + apr_off_t total_out; +} brotli_ctx_t; + +static void *alloc_func(void *opaque, size_t size) +{ + return apr_bucket_alloc(size, opaque); +} + +static void free_func(void *opaque, void *block) +{ + if (block) { + apr_bucket_free(block); + } +} + +static apr_status_t cleanup_ctx(void *data) +{ + brotli_ctx_t *ctx = data; + + BrotliEncoderDestroyInstance(ctx->state); + ctx->state = NULL; + return APR_SUCCESS; +} + +static brotli_ctx_t *create_ctx(int quality, + int lgwin, + int lgblock, + apr_bucket_alloc_t *alloc, + apr_pool_t *pool) +{ + brotli_ctx_t *ctx = apr_pcalloc(pool, sizeof(*ctx)); + + ctx->state = BrotliEncoderCreateInstance(alloc_func, free_func, alloc); + BrotliEncoderSetParameter(ctx->state, BROTLI_PARAM_QUALITY, quality); + BrotliEncoderSetParameter(ctx->state, BROTLI_PARAM_LGWIN, lgwin); + BrotliEncoderSetParameter(ctx->state, BROTLI_PARAM_LGBLOCK, lgblock); + apr_pool_cleanup_register(pool, ctx, cleanup_ctx, apr_pool_cleanup_null); + + ctx->bb = apr_brigade_create(pool, alloc); + ctx->total_in = 0; + ctx->total_out = 0; + + return ctx; +} + +static apr_status_t process_chunk(brotli_ctx_t *ctx, + const void *data, + apr_size_t len, + ap_filter_t *f) +{ + const uint8_t *next_in = data; + apr_size_t avail_in = len; + + while (avail_in > 0) { + uint8_t *next_out = NULL; + apr_size_t avail_out = 0; + + if (!BrotliEncoderCompressStream(ctx->state, + BROTLI_OPERATION_PROCESS, + &avail_in, &next_in, + &avail_out, &next_out, NULL)) { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, f->r, APLOGNO(03459) + "Error while compressing data"); + return APR_EGENERAL; + } + + if (BrotliEncoderHasMoreOutput(ctx->state)) { + apr_size_t output_len = 0; + const uint8_t *output; + apr_status_t rv; + apr_bucket *b; + + /* Drain the accumulated output. Avoid copying the data by + * wrapping a pointer to the internal output buffer and passing + * it down to the next filter. The pointer is only valid until + * the next call to BrotliEncoderCompressStream(), but we're okay + * with that, since the brigade is cleaned up right after the + * ap_pass_brigade() call. + */ + output = BrotliEncoderTakeOutput(ctx->state, &output_len); + ctx->total_out += output_len; + + b = apr_bucket_transient_create((const char *)output, output_len, + ctx->bb->bucket_alloc); + APR_BRIGADE_INSERT_TAIL(ctx->bb, b); + + rv = ap_pass_brigade(f->next, ctx->bb); + apr_brigade_cleanup(ctx->bb); + if (rv != APR_SUCCESS) { + return rv; + } + } + } + + ctx->total_in += len; + return APR_SUCCESS; +} + +static apr_status_t flush(brotli_ctx_t *ctx, + BrotliEncoderOperation op, + ap_filter_t *f) +{ + while (1) { + const uint8_t *next_in = NULL; + apr_size_t avail_in = 0; + uint8_t *next_out = NULL; + apr_size_t avail_out = 0; + apr_size_t output_len; + const uint8_t *output; + apr_bucket *b; + + if (!BrotliEncoderCompressStream(ctx->state, op, + &avail_in, &next_in, + &avail_out, &next_out, NULL)) { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, f->r, APLOGNO(03460) + "Error while compressing data"); + return APR_EGENERAL; + } + + if (!BrotliEncoderHasMoreOutput(ctx->state)) { + break; + } + + /* A flush can require several calls to BrotliEncoderCompressStream(), + * so place the data on the heap (otherwise, the pointer will become + * invalid after the next call to BrotliEncoderCompressStream()). + */ + output_len = 0; + output = BrotliEncoderTakeOutput(ctx->state, &output_len); + ctx->total_out += output_len; + + b = apr_bucket_heap_create((const char *)output, output_len, NULL, + ctx->bb->bucket_alloc); + APR_BRIGADE_INSERT_TAIL(ctx->bb, b); + } + + return APR_SUCCESS; +} + +static const char *get_content_encoding(request_rec *r) +{ + const char *encoding; + + encoding = apr_table_get(r->headers_out, "Content-Encoding"); + if (encoding) { + const char *err_enc; + + err_enc = apr_table_get(r->err_headers_out, "Content-Encoding"); + if (err_enc) { + encoding = apr_pstrcat(r->pool, encoding, ",", err_enc, NULL); + } + } + else { + encoding = apr_table_get(r->err_headers_out, "Content-Encoding"); + } + + if (r->content_encoding) { + encoding = encoding ? apr_pstrcat(r->pool, encoding, ",", + r->content_encoding, NULL) + : r->content_encoding; + } + + return encoding; +} + +static apr_status_t compress_filter(ap_filter_t *f, apr_bucket_brigade *bb) +{ + request_rec *r = f->r; + brotli_ctx_t *ctx = f->ctx; + apr_status_t rv; + brotli_server_config_t *conf; + + if (APR_BRIGADE_EMPTY(bb)) { + return APR_SUCCESS; + } + + conf = ap_get_module_config(r->server->module_config, &brotli_module); + + if (!ctx) { + const char *encoding; + const char *token; + const char *accepts; + + /* Only work on main request, not subrequests, that are not + * a 204 response with no content, and are not tagged with the + * no-brotli env variable, and are not a partial response to + * a Range request. + */ + if (r->main || r->status == HTTP_NO_CONTENT + || apr_table_get(r->subprocess_env, "no-brotli") + || apr_table_get(r->headers_out, "Content-Range")) { + ap_remove_output_filter(f); + return ap_pass_brigade(f->next, bb); + } + + /* Let's see what our current Content-Encoding is. */ + encoding = get_content_encoding(r); + + if (encoding) { + const char *tmp = encoding; + + token = ap_get_token(r->pool, &tmp, 0); + while (token && *token) { + if (strcmp(token, "identity") != 0 && + strcmp(token, "7bit") != 0 && + strcmp(token, "8bit") != 0 && + strcmp(token, "binary") != 0) { + /* The data is already encoded, do nothing. */ + ap_remove_output_filter(f); + return ap_pass_brigade(f->next, bb); + } + + if (*tmp) { + ++tmp; + } + token = (*tmp) ? ap_get_token(r->pool, &tmp, 0) : NULL; + } + } + + /* Even if we don't accept this request based on it not having + * the Accept-Encoding, we need to note that we were looking + * for this header and downstream proxies should be aware of + * that. + */ + apr_table_mergen(r->headers_out, "Vary", "Accept-Encoding"); + + accepts = apr_table_get(r->headers_in, "Accept-Encoding"); + if (!accepts) { + ap_remove_output_filter(f); + return ap_pass_brigade(f->next, bb); + } + + /* Do we have Accept-Encoding: br? */ + token = ap_get_token(r->pool, &accepts, 0); + while (token && token[0] && ap_cstr_casecmp(token, "br") != 0) { + while (*accepts == ';') { + ++accepts; + ap_get_token(r->pool, &accepts, 1); + } + + if (*accepts == ',') { + ++accepts; + } + token = (*accepts) ? ap_get_token(r->pool, &accepts, 0) : NULL; + } + + if (!token || token[0] == '\0') { + ap_remove_output_filter(f); + return ap_pass_brigade(f->next, bb); + } + + /* If the entire Content-Encoding is "identity", we can replace it. */ + if (!encoding || ap_cstr_casecmp(encoding, "identity") == 0) { + apr_table_setn(r->headers_out, "Content-Encoding", "br"); + } else { + apr_table_mergen(r->headers_out, "Content-Encoding", "br"); + } + + if (r->content_encoding) { + r->content_encoding = apr_table_get(r->headers_out, + "Content-Encoding"); + } + + apr_table_unset(r->headers_out, "Content-Length"); + apr_table_unset(r->headers_out, "Content-MD5"); + + /* https://bz.apache.org/bugzilla/show_bug.cgi?id=39727 + * https://bz.apache.org/bugzilla/show_bug.cgi?id=45023 + * + * ETag must be unique among the possible representations, so a + * change to content-encoding requires a corresponding change to the + * ETag. We make this behavior configurable, and mimic mod_deflate's + * DeflateAlterETag with BrotliAlterETag to keep the transition from + * mod_deflate seamless. + */ + if (conf->etag_mode == ETAG_MODE_REMOVE) { + apr_table_unset(r->headers_out, "ETag"); + } + else if (conf->etag_mode == ETAG_MODE_ADDSUFFIX) { + const char *etag = apr_table_get(r->headers_out, "ETag"); + + if (etag) { + apr_size_t len = strlen(etag); + + if (len > 2 && etag[len - 1] == '"') { + etag = apr_pstrmemdup(r->pool, etag, len - 1); + etag = apr_pstrcat(r->pool, etag, "-br\"", NULL); + apr_table_setn(r->headers_out, "ETag", etag); + } + } + } + + /* For 304 responses, we only need to send out the headers. */ + if (r->status == HTTP_NOT_MODIFIED) { + ap_remove_output_filter(f); + return ap_pass_brigade(f->next, bb); + } + + ctx = create_ctx(conf->quality, conf->lgwin, conf->lgblock, + f->c->bucket_alloc, r->pool); + f->ctx = ctx; + } + + while (!APR_BRIGADE_EMPTY(bb)) { + apr_bucket *e = APR_BRIGADE_FIRST(bb); + + /* Optimization: If we are a HEAD request and bytes_sent is not zero + * it means that we have passed the content-length filter once and + * have more data to send. This means that the content-length filter + * could not determine our content-length for the response to the + * HEAD request anyway (the associated GET request would deliver the + * body in chunked encoding) and we can stop compressing. + */ + if (r->header_only && r->bytes_sent) { + ap_remove_output_filter(f); + return ap_pass_brigade(f->next, bb); + } + + if (APR_BUCKET_IS_EOS(e)) { + rv = flush(ctx, BROTLI_OPERATION_FINISH, f); + if (rv != APR_SUCCESS) { + return rv; + } + + /* Leave notes for logging. */ + if (conf->note_input_name) { + apr_table_setn(r->notes, conf->note_input_name, + apr_off_t_toa(r->pool, ctx->total_in)); + } + if (conf->note_output_name) { + apr_table_setn(r->notes, conf->note_output_name, + apr_off_t_toa(r->pool, ctx->total_out)); + } + if (conf->note_ratio_name) { + if (ctx->total_in > 0) { + int ratio = (int) (ctx->total_out * 100 / ctx->total_in); + + apr_table_setn(r->notes, conf->note_ratio_name, + apr_itoa(r->pool, ratio)); + } + else { + apr_table_setn(r->notes, conf->note_ratio_name, "-"); + } + } + + APR_BUCKET_REMOVE(e); + APR_BRIGADE_INSERT_TAIL(ctx->bb, e); + + rv = ap_pass_brigade(f->next, ctx->bb); + apr_brigade_cleanup(ctx->bb); + apr_pool_cleanup_run(r->pool, ctx, cleanup_ctx); + return rv; + } + else if (APR_BUCKET_IS_FLUSH(e)) { + rv = flush(ctx, BROTLI_OPERATION_FLUSH, f); + if (rv != APR_SUCCESS) { + return rv; + } + + APR_BUCKET_REMOVE(e); + APR_BRIGADE_INSERT_TAIL(ctx->bb, e); + + rv = ap_pass_brigade(f->next, ctx->bb); + apr_brigade_cleanup(ctx->bb); + if (rv != APR_SUCCESS) { + return rv; + } + } + else if (APR_BUCKET_IS_METADATA(e)) { + APR_BUCKET_REMOVE(e); + APR_BRIGADE_INSERT_TAIL(ctx->bb, e); + } + else { + const char *data; + apr_size_t len; + + rv = apr_bucket_read(e, &data, &len, APR_BLOCK_READ); + if (rv != APR_SUCCESS) { + return rv; + } + rv = process_chunk(ctx, data, len, f); + if (rv != APR_SUCCESS) { + return rv; + } + apr_bucket_delete(e); + } + } + return APR_SUCCESS; +} + +static void register_hooks(apr_pool_t *p) +{ + ap_register_output_filter("BROTLI_COMPRESS", compress_filter, NULL, + AP_FTYPE_CONTENT_SET); +} + +static const command_rec cmds[] = { + AP_INIT_TAKE12("BrotliFilterNote", set_filter_note, + NULL, RSRC_CONF, + "Set a note to report on compression ratio"), + AP_INIT_TAKE1("BrotliCompressionQuality", set_compression_quality, + NULL, RSRC_CONF, + "Compression quality between 0 and 11 (higher quality means " + "slower compression)"), + AP_INIT_TAKE1("BrotliCompressionWindow", set_compression_lgwin, + NULL, RSRC_CONF, + "Sliding window size between 10 and 24 (larger windows can " + "improve compression, but require more memory)"), + AP_INIT_TAKE1("BrotliCompressionMaxInputBlock", set_compression_lgblock, + NULL, RSRC_CONF, + "Maximum input block size between 16 and 24 (larger block " + "sizes require more memory)"), + AP_INIT_TAKE1("BrotliAlterETag", set_etag_mode, + NULL, RSRC_CONF, + "Set how mod_brotli should modify ETag response headers: " + "'AddSuffix' (default), 'NoChange', 'Remove'"), + {NULL} +}; + +AP_DECLARE_MODULE(brotli) = { + STANDARD20_MODULE_STUFF, + NULL, /* create per-directory config structure */ + NULL, /* merge per-directory config structures */ + create_server_config, /* create per-server config structure */ + NULL, /* merge per-server config structures */ + cmds, /* command apr_table_t */ + register_hooks /* register hooks */ +}; |