SERVER-51364 Ubuntu 18.04 Server with OCSP and TLS fails to work

(cherry picked from commit c20e3c5001923d8e8385dab70786da97888b039e)

11 files changed, 84 insertions, 22 deletions

diff --git a/SConstruct b/SConstruct
index 631720209a1..9781f09f29f 100644
--- a/SConstruct
+++ b/SConstruct
@@ -208,6 +208,14 @@ add_option('wiredtiger',
     type='choice',
 )
 
+add_option('ocsp-stapling',
+    choices=['on', 'off'],
+    default='on',
+    help='Enable OCSP Stapling on servers',
+    nargs='?',
+    type='choice',
+)
+
 js_engine_choices = ['mozjs', 'none']
 add_option('js-engine',
     choices=js_engine_choices,
@@ -2154,6 +2162,13 @@ if get_option('wiredtiger') == 'on':
         wiredtiger = True
         env.SetConfigHeaderDefine("MONGO_CONFIG_WIREDTIGER_ENABLED")
 
+if get_option('ocsp-stapling') == 'on':
+    # OCSP Stapling needs to be disabled on ubuntu 18.04 machines because when TLS 1.3 is
+    # enabled on that machine, the status-response message sent contains garbage data. This
+    # is a known bug and needs to be fixed by upstream, but for the time being we need to
+    # disable OCSP Stapling on Ubuntu 18.04 machines. See SERVER-51364 for more details.
+    env.SetConfigHeaderDefine("MONGO_CONFIG_OCSP_STAPLING_ENABLED")
+
 if env['TARGET_ARCH'] == 'i386':
     # If we are using GCC or clang to target 32 bit, set the ISA minimum to 'nocona',
     # and the tuning to 'generic'. The choice of 'nocona' is selected because it
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index a3506e3b3cc..65df2fa3f9d 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -8911,9 +8911,10 @@ buildvariants:
     push_bucket: downloads.mongodb.org
     push_name: linux
     push_arch: x86_64-ubuntu1804
-    compile_flags: --ssl MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
+    compile_flags: --ssl --ocsp-stapling=off MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
     multiversion_platform: ubuntu1804
     multiversion_edition: targeted
+    test_flags: --excludeWithAnyTags=requires_ocsp_stapling
     has_packages: true
     packager_script: packager.py
     packager_arch: x86_64
@@ -8966,9 +8967,10 @@ buildvariants:
     push_bucket: downloads.10gen.com
     push_name: linux
     push_arch: x86_64-enterprise-ubuntu1804
-    compile_flags: --ssl MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
+    compile_flags: --ssl --ocsp-stapling=off MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
     multiversion_platform: ubuntu1804
     multiversion_edition: enterprise
+    test_flags: --excludeWithAnyTags=requires_ocsp_stapling
     has_packages: true
     packager_script: packager_enterprise.py
     packager_arch: x86_64
@@ -9130,7 +9132,8 @@ buildvariants:
     push_bucket: downloads.10gen.com
     push_name: linux
     push_arch: aarch64-enterprise-ubuntu1804
-    compile_flags: --ssl MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo) CCFLAGS="-march=armv8-a+crc -mtune=generic" --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
+    compile_flags: --ssl --ocsp-stapling=off MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo) CCFLAGS="-march=armv8-a+crc -mtune=generic" --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
+    test_flags: --excludeWithAnyTags=requires_ocsp_stapling
     resmoke_jobs_max: 4 # Avoid starting too many mongod's on ARM test servers
     has_packages: true
     packager_script: packager_enterprise.py
@@ -9175,7 +9178,8 @@ buildvariants:
     push_bucket: downloads.mongodb.org
     push_name: linux
     push_arch: aarch64-ubuntu1804
-    compile_flags: --ssl MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo) CCFLAGS="-march=armv8-a+crc -mtune=generic" --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
+    compile_flags: --ssl --ocsp-stapling=off MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo) CCFLAGS="-march=armv8-a+crc -mtune=generic" --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
+    test_flags: --excludeWithAnyTags=requires_ocsp_stapling
     resmoke_jobs_max: 8 # Avoid starting too many mongod's on ARM test servers
     has_packages: true
     packager_script: packager.py
@@ -9209,7 +9213,8 @@ buildvariants:
     push_bucket: downloads.10gen.com
     push_name: linux
     push_arch: ppc64le-enterprise-ubuntu1804
-    compile_flags: --ssl MONGO_DISTMOD=ubuntu1804 -j$(echo "$(grep -c processor /proc/cpuinfo)/2" | bc) CCFLAGS="-mcpu=power8 -mtune=power8 -mcmodel=medium" --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
+    compile_flags: --ssl --ocsp-stapling=off MONGO_DISTMOD=ubuntu1804 -j$(echo "$(grep -c processor /proc/cpuinfo)/2" | bc) CCFLAGS="-mcpu=power8 -mtune=power8 -mcmodel=medium" --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
+    test_flags: --excludeWithAnyTags=requires_ocsp_stapling
     resmoke_jobs_max: 2
     has_packages: true
     packager_script: packager_enterprise.py
@@ -9250,7 +9255,8 @@ buildvariants:
     push_bucket: downloads.mongodb.org
     push_name: linux
     push_arch: s390x-ubuntu1804
-    compile_flags: --ssl MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo) CCFLAGS="-march=z196 -mtune=zEC12" --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
+    compile_flags: --ssl --ocsp-stapling=off MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo) CCFLAGS="-march=z196 -mtune=zEC12" --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
+    test_flags: --excludeWithAnyTags=requires_ocsp_stapling
     multiversion_platform: ubuntu1804
     multiversion_edition: targeted
     multiversion_architecture: s390x
@@ -9283,7 +9289,8 @@ buildvariants:
     push_bucket: downloads.10gen.com
     push_name: linux
     push_arch: s390x-enterprise-ubuntu1804
-    compile_flags: --ssl MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo) CCFLAGS="-march=z196 -mtune=zEC12" --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
+    compile_flags: --ssl --ocsp-stapling=off MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo) CCFLAGS="-march=z196 -mtune=zEC12" --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
+    test_flags: --excludeWithAnyTags=requires_ocsp_stapling
     resmoke_jobs_max: 2
     has_packages: true
     packager_script: packager_enterprise.py
@@ -12331,13 +12338,13 @@ buildvariants:
     # If you add anything to san_options, make sure the appropriate changes are
     # also made to SConstruct.
     san_options: LSAN_OPTIONS="suppressions=etc/lsan.suppressions:report_objects=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" ASAN_OPTIONS="detect_leaks=1:check_initialization_order=true:strict_init_order=true:abort_on_error=1:disable_coredump=0:handle_abort=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer"
-    compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=address --ssl --enable-free-mon=on -j$(grep -c ^processor /proc/cpuinfo) --nostrip
+    compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=address --ssl --ocsp-stapling=off --enable-free-mon=on -j$(grep -c ^processor /proc/cpuinfo) --nostrip
+    test_flags: --excludeWithAnyTags=requires_fast_memory,requires_ocsp_stapling
     multiversion_platform: ubuntu1804
     multiversion_edition: enterprise
     resmoke_jobs_factor: 0.3  # Avoid starting too many mongod's under ASAN build.
     hang_analyzer_dump_core: false
     scons_cache_scope: shared
-    test_flags: --excludeWithAnyTags=requires_fast_memory
   tasks:
   - name: compile_all_run_unittests_TG
   - name: compile_benchmarks
@@ -12396,13 +12403,13 @@ buildvariants:
   expansions:
     lang_environment: LANG=C
     san_options: LSAN_OPTIONS="suppressions=etc/lsan.suppressions:report_objects=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" ASAN_OPTIONS="detect_leaks=1:check_initialization_order=true:strict_init_order=true:abort_on_error=1:disable_coredump=0:handle_abort=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer"
-    compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --opt=on --allocator=system --sanitize=address --ssl -j$(grep -c ^processor /proc/cpuinfo) --nostrip
+    compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --opt=on --allocator=system --sanitize=address --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) --nostrip
     multiversion_platform: ubuntu1804
     multiversion_edition: enterprise
     resmoke_jobs_factor: 0.3  # Avoid starting too many mongod's under ASAN build.
     hang_analyzer_dump_core: false
     scons_cache_scope: shared
-    test_flags: --excludeWithAnyTags=requires_fast_memory
+    test_flags: --excludeWithAnyTags=requires_fast_memory,requires_ocsp_stapling
   tasks:
   - name: compile_all_run_unittests_TG
   - name: .aggfuzzer .common
@@ -12422,7 +12429,8 @@ buildvariants:
     # If you add anything to san_options, make sure the appropriate changes are
     # also made to SConstruct.
     san_options: UBSAN_OPTIONS="print_stacktrace=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer"
-    compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --sanitize=undefined --ssl --enable-free-mon=on -j$(grep -c ^processor /proc/cpuinfo) --nostrip
+    compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --sanitize=undefined --ssl --ocsp-stapling=off --enable-free-mon=on -j$(grep -c ^processor /proc/cpuinfo) --nostrip
+    test_flags: --excludeWithAnyTags=requires_ocsp_stapling
     multiversion_platform: ubuntu1804
     multiversion_edition: enterprise
     resmoke_jobs_factor: 0.3  # Avoid starting too many mongod's under UBSAN build.
@@ -12487,7 +12495,8 @@ buildvariants:
     # If you add anything to san_options, make sure the appropriate changes are
     # also made to SConstruct.
     san_options: UBSAN_OPTIONS="print_stacktrace=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" LSAN_OPTIONS="suppressions=etc/lsan.suppressions:report_objects=1" ASAN_OPTIONS="detect_leaks=1:check_initialization_order=true:strict_init_order=true:abort_on_error=1:disable_coredump=0:handle_abort=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer"
-    compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=undefined,address --ssl -j$(grep -c ^processor /proc/cpuinfo) --nostrip
+    compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=undefined,address --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) --nostrip
+    test_flags: --excludeWithAnyTags=requires_ocsp_stapling
     resmoke_jobs_factor: 0.3  # Avoid starting too many mongod's under {A,UB}SAN build.
     hang_analyzer_dump_core: false
     scons_cache_scope: shared
diff --git a/jstests/ocsp/lib/ocsp_helpers.js b/jstests/ocsp/lib/ocsp_helpers.js
index 073a13e6564..6725a0ae62b 100644
--- a/jstests/ocsp/lib/ocsp_helpers.js
+++ b/jstests/ocsp/lib/ocsp_helpers.js
@@ -54,4 +54,16 @@ var waitForServer = function(conn) {
     } else {
         sleep(15000);
     }
+};
+
+var supportsStapling = function() {
+    if (determineSSLProvider() !== "openssl") {
+        return false;
+    }
+
+    if (isUbuntu1804() === true) {
+        return false;
+    }
+
+    return true;
 };
\ No newline at end of file
diff --git a/jstests/ocsp/ocsp_connection_type_testing.js b/jstests/ocsp/ocsp_connection_type_testing.js
index 32c3c50ff7a..f29987a11c1 100644
--- a/jstests/ocsp/ocsp_connection_type_testing.js
+++ b/jstests/ocsp/ocsp_connection_type_testing.js
@@ -1,12 +1,12 @@
 // Check that OCSP verification works
-// @tags: [requires_http_client]
+// @tags: [requires_http_client, requires_ocsp_stapling]
 
 load("jstests/ocsp/lib/mock_ocsp.js");
 
 (function() {
 "use strict";
 
-if (determineSSLProvider() != "openssl") {
+if (!supportsStapling()) {
     return;
 }
 
diff --git a/jstests/ocsp/ocsp_must_staple.js b/jstests/ocsp/ocsp_must_staple.js
index d6429671dc8..08a6eaba8ab 100644
--- a/jstests/ocsp/ocsp_must_staple.js
+++ b/jstests/ocsp/ocsp_must_staple.js
@@ -1,12 +1,12 @@
 // Check that OCSP verification works
-// @tags: [requires_http_client]
+// @tags: [requires_http_client, requires_ocsp_stapling]
 
 load("jstests/ocsp/lib/mock_ocsp.js");
 
 (function() {
 "use strict";
 
-if (determineSSLProvider() !== "openssl") {
+if (!supportsStapling()) {
     return;
 }
 
diff --git a/jstests/ocsp/ocsp_server_refresh.js b/jstests/ocsp/ocsp_server_refresh.js
index a8870895b5f..8b87089e2bf 100644
--- a/jstests/ocsp/ocsp_server_refresh.js
+++ b/jstests/ocsp/ocsp_server_refresh.js
@@ -1,12 +1,12 @@
 // Check that OCSP verification works
-// @tags: [requires_http_client]
+// @tags: [requires_http_client, requires_ocsp_stapling]
 
 load("jstests/ocsp/lib/mock_ocsp.js");
 
 (function() {
 "use strict";
 
-if (determineSSLProvider() != "openssl") {
+if (!supportsStapling()) {
     return;
 }
 
diff --git a/jstests/ocsp/ocsp_stapling.js b/jstests/ocsp/ocsp_stapling.js
index fb21843ac61..69ac0866a02 100644
--- a/jstests/ocsp/ocsp_stapling.js
+++ b/jstests/ocsp/ocsp_stapling.js
@@ -1,12 +1,12 @@
 // Check that OCSP verification works
-// @tags: [requires_http_client]
+// @tags: [requires_http_client, requires_ocsp_stapling]
 
 load("jstests/ocsp/lib/mock_ocsp.js");
 
 (function() {
 "use strict";
 
-if (determineSSLProvider() !== "openssl") {
+if (!supportsStapling()) {
     return;
 }
 
diff --git a/jstests/ssl/libs/ssl_helpers.js b/jstests/ssl/libs/ssl_helpers.js
index d4b6996308f..77d0c9fd7cd 100644
--- a/jstests/ssl/libs/ssl_helpers.js
+++ b/jstests/ssl/libs/ssl_helpers.js
@@ -318,6 +318,22 @@ function isUbuntu2004() {
     return false;
 }
 
+function isUbuntu1804() {
+    if (_isWindows()) {
+        return false;
+    }
+
+    // Ubuntu 18.04's TLS 1.3 implementation has an issue with OCSP stapling. We have disabled
+    // stapling on this build variant, so we need to ensure that tests that require stapling
+    // do not run on this machine.
+    const grep_result = runProgram('grep', 'bionic', '/etc/os-release');
+    if (grep_result === 0) {
+        return true;
+    }
+
+    return false;
+}
+
 function isDebian10() {
     if (_isWindows()) {
         return false;
diff --git a/src/mongo/SConscript b/src/mongo/SConscript
index 236f32ac3ae..62fbfc02786 100644
--- a/src/mongo/SConscript
+++ b/src/mongo/SConscript
@@ -296,6 +296,7 @@ config_header_substs = (
     ('@mongo_config_have_std_enable_if_t@', 'MONGO_CONFIG_HAVE_STD_ENABLE_IF_T'),
     ('@mongo_config_have_strnlen@', 'MONGO_CONFIG_HAVE_STRNLEN'),
     ('@mongo_config_max_extended_alignment@', 'MONGO_CONFIG_MAX_EXTENDED_ALIGNMENT'),
+    ('@mongo_config_ocsp_stapling_enabled@', 'MONGO_CONFIG_OCSP_STAPLING_ENABLED'),
     ('@mongo_config_optimized_build@', 'MONGO_CONFIG_OPTIMIZED_BUILD'),
     ('@mongo_config_ssl@', 'MONGO_CONFIG_SSL'),
     ('@mongo_config_ssl_has_asn1_any_definitions@', 'MONGO_CONFIG_HAVE_ASN1_ANY_DEFINITIONS'),
diff --git a/src/mongo/config.h.in b/src/mongo/config.h.in
index 91e87b5a9b9..6740524cc8e 100644
--- a/src/mongo/config.h.in
+++ b/src/mongo/config.h.in
@@ -71,6 +71,9 @@
 // A number, if we have some extended alignment ability
 @mongo_config_max_extended_alignment@
 
+// defined if OCSP Stapling is enabled
+@mongo_config_ocsp_stapling_enabled@
+
 // Defined if building an optimized build
 @mongo_config_optimized_build@
 
@@ -96,4 +99,4 @@
 @mongo_config_use_raw_latches@
 
 // Defined if WiredTiger storage engine is enabled
-@mongo_config_wiredtiger_enabled@
+@mongo_config_wiredtiger_enabled@
\ No newline at end of file
diff --git a/src/mongo/util/net/ssl_manager_openssl.cpp b/src/mongo/util/net/ssl_manager_openssl.cpp
index 674717a2a72..5f4a50bba83 100644
--- a/src/mongo/util/net/ssl_manager_openssl.cpp
+++ b/src/mongo/util/net/ssl_manager_openssl.cpp
@@ -1829,6 +1829,7 @@ std::tuple<X509*> getCertificateForContext(SSL_CTX* context) {
 }
 #endif
 
+#ifdef MONGO_CONFIG_OCSP_STAPLING_ENABLED
 Status SSLManagerOpenSSL::stapleOCSPResponse(SSL_CTX* context) {
     if (MONGO_unlikely(disableStapling.shouldFail()) || !tlsOCSPEnabled) {
         return Status::OK();
@@ -1836,6 +1837,11 @@ Status SSLManagerOpenSSL::stapleOCSPResponse(SSL_CTX* context) {
 
     return _fetcher.start(context, true);
 }
+#else
+Status SSLManagerOpenSSL::stapleOCSPResponse(SSL_CTX* context) {
+    return Status::OK();
+}
+#endif  // MONGO_CONFIG_OCSP_STAPLING_ENABLED
 
 Status OCSPFetcher::start(SSL_CTX* context, bool asyncOCSPStaple) {
     // Increment the ref count on SSL_CTX by creating a SSL object so that our context lives with