diff options
author | Ian Boros <puppyofkosh@gmail.com> | 2019-07-19 13:37:05 -0400 |
---|---|---|
committer | Ian Boros <puppyofkosh@gmail.com> | 2019-07-19 13:37:05 -0400 |
commit | 72f18524e2ea1296568e5b5763ae15368392415a (patch) | |
tree | 6da5a0868499d60f199da860b8671ea14bbc4dc9 | |
parent | 8fcef35e07a166746ba68cbe2454ec1cc85230d6 (diff) | |
download | mongo-72f18524e2ea1296568e5b5763ae15368392415a.tar.gz |
Revert "SERVER-41796 Create Evergreen variant for libfuzzer targets"
This reverts commit 51a218fb13e02e59afe203c5efd665c9399bfde2.
-rw-r--r-- | SConstruct | 3 | ||||
-rwxr-xr-x | buildscripts/merge_corpus.sh | 26 | ||||
-rw-r--r-- | buildscripts/resmokeconfig/suites/libfuzzer.yml | 7 | ||||
-rw-r--r-- | buildscripts/resmokelib/config.py | 3 | ||||
-rw-r--r-- | buildscripts/resmokelib/core/process.py | 4 | ||||
-rw-r--r-- | buildscripts/resmokelib/selector.py | 1 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/executor.py | 1 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/testcases/cpp_libfuzzer_test.py | 51 | ||||
-rw-r--r-- | etc/evergreen.yml | 111 | ||||
-rw-r--r-- | src/mongo/db/storage/key_string_to_bson_fuzzer.cpp | 27 |
10 files changed, 18 insertions, 216 deletions
diff --git a/SConstruct b/SConstruct index 7fb8c50bae6..0d1a5fd5096 100644 --- a/SConstruct +++ b/SConstruct @@ -2672,9 +2672,6 @@ def doConfigure(myenv): # to fail sanitizer_list.remove('fuzzer') sanitizer_list.append('fuzzer-no-link') - # These flags are needed to generate a coverage report - myenv.Append(LINKFLAGS=['-fprofile-instr-generate','-fcoverage-mapping']) - myenv.Append(CCFLAGS=['-fprofile-instr-generate','-fcoverage-mapping']) sanitizer_option = '-fsanitize=' + ','.join(sanitizer_list) diff --git a/buildscripts/merge_corpus.sh b/buildscripts/merge_corpus.sh deleted file mode 100755 index 162f7037af5..00000000000 --- a/buildscripts/merge_corpus.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# merge_corpus.sh -# -# Merges the corpus of each libfuzzer tests - -set -o verbose -set -o errexit - -input="build/libfuzzer_tests.txt" -corpus_dir="corpus" - -# We need to merge the corpus once it has been tested -while IFS= read -r line -do - mkdir "$corpus_dir"/corpus-"${line##*/}"-new - ./"$line" "$corpus_dir"/corpus-"${line##*/}"-new "$corpus_dir"/corpus-"${line##*/}" -merge=1 -done < "$input" - -# Delete old corpus -find corpus/* -not -name '*-new' -type d -exec rm -rv {} + - -# Rename new corpus to old corpus -for f in ./corpus/* -do - mv "$f" "${f%-new}" -done
\ No newline at end of file diff --git a/buildscripts/resmokeconfig/suites/libfuzzer.yml b/buildscripts/resmokeconfig/suites/libfuzzer.yml deleted file mode 100644 index b5edb77583a..00000000000 --- a/buildscripts/resmokeconfig/suites/libfuzzer.yml +++ /dev/null @@ -1,7 +0,0 @@ -test_kind: cpp_libfuzzer_test - -selector: - root: build/libfuzzer_tests.txt - -executor: - config: {}
\ No newline at end of file diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index 5733e4789a0..729be502a8f 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -435,10 +435,9 @@ ORDER_TESTS_BY_NAME = True DEFAULT_BENCHMARK_TEST_LIST = "build/benchmarks.txt" DEFAULT_UNIT_TEST_LIST = "build/unittests.txt" DEFAULT_INTEGRATION_TEST_LIST = "build/integration_tests.txt" -DEFAULT_LIBFUZZER_TEST_LIST = "build/libfuzzer_tests.txt" # External files or executables, used as suite selectors, that are created during the build and # therefore might not be available when creating a test membership map. EXTERNAL_SUITE_SELECTORS = (DEFAULT_BENCHMARK_TEST_LIST, DEFAULT_UNIT_TEST_LIST, DEFAULT_INTEGRATION_TEST_LIST, DEFAULT_DBTEST_EXECUTABLE, - DEFAULT_MONGOEBENCH_EXECUTABLE, DEFAULT_LIBFUZZER_TEST_LIST) + DEFAULT_MONGOEBENCH_EXECUTABLE) diff --git a/buildscripts/resmokelib/core/process.py b/buildscripts/resmokelib/core/process.py index 2c49eba1e12..de241ef57ce 100644 --- a/buildscripts/resmokelib/core/process.py +++ b/buildscripts/resmokelib/core/process.py @@ -193,10 +193,10 @@ class Process(object): """Poll.""" return self._process.poll() - def wait(self, timeout=None): + def wait(self): """Wait until process has terminated and all output has been consumed by the logger pipes.""" - return_code = self._process.wait(timeout) + return_code = self._process.wait() if self._stdout_pipe: self._stdout_pipe.wait_until_finished() diff --git a/buildscripts/resmokelib/selector.py b/buildscripts/resmokelib/selector.py index 5880e2b3c9f..36f2e608889 100644 --- a/buildscripts/resmokelib/selector.py +++ b/buildscripts/resmokelib/selector.py @@ -706,7 +706,6 @@ _SELECTOR_REGISTRY = { "sleep_test": (_SleepTestCaseSelectorConfig, _SleepTestCaseSelector), "genny_test": (_FileBasedSelectorConfig, _Selector), "gennylib_test": (_GennylibTestCaseSelectorConfig, _GennylibTestCaseSelector), - "cpp_libfuzzer_test": (_CppTestSelectorConfig, _CppTestSelector), } diff --git a/buildscripts/resmokelib/testing/executor.py b/buildscripts/resmokelib/testing/executor.py index 234abed1d38..21a1d12e128 100644 --- a/buildscripts/resmokelib/testing/executor.py +++ b/buildscripts/resmokelib/testing/executor.py @@ -238,6 +238,7 @@ class TestSuiteExecutor(object): # pylint: disable=too-many-instance-attributes fixture_config = {} fixture_class = fixtures.NOOP_FIXTURE_CLASS + if self.fixture_config is not None: fixture_config = self.fixture_config.copy() fixture_class = fixture_config.pop("class") diff --git a/buildscripts/resmokelib/testing/testcases/cpp_libfuzzer_test.py b/buildscripts/resmokelib/testing/testcases/cpp_libfuzzer_test.py deleted file mode 100644 index 2ac09dd5ab9..00000000000 --- a/buildscripts/resmokelib/testing/testcases/cpp_libfuzzer_test.py +++ /dev/null @@ -1,51 +0,0 @@ -"""The libfuzzertest.TestCase for C++ libfuzzer tests.""" - -import subprocess -import os -import datetime - -from . import interface -from ... import core -from ... import utils - -class CPPLibfuzzerTestCase(interface.ProcessTestCase): - """A C++ libfuzzer test to execute.""" - - REGISTERED_NAME = "cpp_libfuzzer_test" - DEFAULT_TIMEOUT = datetime.timedelta(hours=1) - - def __init__(self, logger, program_executable, program_options=None): - """Initialize the CPPLibfuzzerTestCase with the executable to run.""" - - interface.ProcessTestCase.__init__(self, logger, "C++ libfuzzer test", program_executable) - - self.program_executable = program_executable - self.program_options = utils.default_if_none(program_options, {}).copy() - self.corpus_directory = "corpus/corpus-" + self.short_name() - - os.makedirs(self.corpus_directory, exist_ok=True) - - def _make_process(self): - default_args = [self.program_executable, self.corpus_directory, "-max_len=100000", "-rss_limit_mb=5000"] - return core.programs.make_process(self.logger, default_args, - **self.program_options) - - def _execute(self, process): - """Run the specified process.""" - self.logger.info("Starting Libfuzzer Test %s...\n%s", self.short_description(), process.as_command()) - process.start() - self.logger.info("%s started with pid %s.", self.short_description(), process.pid) - try: - self.return_code = process.wait(self.DEFAULT_TIMEOUT.total_seconds()) - except subprocess.TimeoutExpired: - # If the test timeout, then no errors were detected. Thus, the return code should be 0. - process.stop(kill=True) - process.wait() - self.logger.info("%s timed out. No errors were found.", self.short_description()) - self.return_code = 0 - - if self.return_code != 0: - self.logger.info("Failed %s", self.return_code) - raise self.failureException("%s failed" % (self.short_description())) - - self.logger.info("%s finished.", self.short_description()) diff --git a/etc/evergreen.yml b/etc/evergreen.yml index b345a2c76a1..b0551714084 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -200,12 +200,6 @@ variables: - compile_unittests - unittests -- &libfuzzertests - name: libfuzzertests! - execution_tasks: - - compile_libfuzzertests - - libfuzzertests - - &dbtest name: dbtest! execution_tasks: @@ -406,7 +400,6 @@ variables: - ubuntu1804-debug-asan - ubuntu1804-debug-ubsan - ubuntu1804-debug-aubsan-lite - - ubuntu1804-debug-aubsan-lite_fuzzer - ubuntu1804-debug-aubsan-async @@ -536,51 +529,6 @@ functions: bucket: mciuploads extract_to: src/benchrun_embedded/testcases - "fetch corpus": &fetch_corpus - command: s3.get - params: - aws_key: ${s3_access_key_id} - aws_secret: ${s3_secret_access_key} - remote_file: ${project}/corpus/mongo-${build_variant}-latest.tgz - bucket: fuzzer-artifacts - local_file: src/corpus.tgz - - "extract corpus": &extract_corpus - command: shell.exec - type: test - params: - shell: bash - script: | - set -o verbose - set -o errexit - - target_dir="src/corpus" - mkdir -p $target_dir - mv src/corpus.tgz $target_dir - - cd $target_dir - tar xzf corpus.tgz - - "archive new corpus": &archive_new_corpus - command: archive.targz_pack - params: - target: corpus.tgz - source_dir: src/corpus - include: - - "**" - - "upload new corpus": &upload_new_corpus - command: s3.put - params: - aws_key: ${s3_access_key_id} - aws_secret: ${s3_secret_access_key} - local_file: corpus.tgz - remote_file: ${project}/corpus/mongo-${build_variant}-latest.tgz - bucket: fuzzer-artifacts - permissions: private - content_type: ${content_type|application/gzip} - display_name: "Fuzzer Tests Corpus Tar Archive" - "get buildnumber": &get_buildnumber command: keyval.inc params: @@ -3813,36 +3761,6 @@ tasks: vars: resmoke_args: --suites=unittests -##compile_libfuzzertests - build libfuzzertests ## -- name: compile_libfuzzertests - commands: - - func: "scons compile" - vars: - targets: libfuzzer_tests - task_compile_flags: >- - --detect-odr-violations - -## libfuzzertests - run libfuzzertests ## -- name: libfuzzertests - commands: - - func: "fetch corpus" - - func: "extract corpus" - - func: "run tests" - vars: - resmoke_args: --suites=libfuzzer - - command: shell.exec - type: setup - params: - working_dir: src - shell: bash - script: | - set -o verbose - set -o errexit - - ./buildscripts/merge_corpus.sh - - func: "archive new corpus" - - func: "upload new corpus" - ## compile_dbtest ## - name: compile_dbtest commands: @@ -8302,11 +8220,6 @@ task_groups: - compile_dbtest - dbtest - <<: *compile_task_group_template - name: libfuzzertests_TG - tasks: - - compile_libfuzzertests - - libfuzzertests -- <<: *compile_task_group_template name: compile_all_run_unittests_TG tasks: - compile @@ -12377,30 +12290,6 @@ buildvariants: - name: .logical_session_cache .one_sec - name: watchdog_wiredtiger -- name: ubuntu1804-debug-aubsan-lite_fuzzer - display_name: "{A,UB}SAN Enterprise SSL Ubuntu 18.04 FUZZER" - modules: - - enterprise - run_on: - - ubuntu1804-build - stepback: false - batchtime: 1440 # 1 day - expansions: - # We need llvm-symbolizer in the PATH for ASAN for clang-3.7 or later. - variant_path_suffix: /opt/mongodbtoolchain/v3/bin - lang_environment: LANG=C - san_options: UBSAN_OPTIONS="print_stacktrace=1:halt_on_error=1" LSAN_OPTIONS="suppressions=etc/lsan.suppressions:report_objects=1" ASAN_OPTIONS=detect_leaks=1:check_initialization_order=true:strict_init_order=true:abort_on_error=1:disable_coredump=0:handle_abort=1 - compile_flags: LINKFLAGS=-nostdlib++ LIBS=stdc++ --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=undefined,address,fuzzer --ssl -j$(grep -c ^processor /proc/cpuinfo) --nostrip --disable-warnings-as-errors - resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under {A,UB}SAN build. - tooltags: "ssl" - build_mongoreplay: true - hang_analyzer_dump_core: false - scons_cache_scope: shared - display_tasks: - - *libfuzzertests - tasks: - - name: libfuzzertests_TG - - name: ubuntu1804-debug-aubsan-async display_name: "~ {A,UB}SAN Enterprise SSL Ubuntu 18.04 async" modules: diff --git a/src/mongo/db/storage/key_string_to_bson_fuzzer.cpp b/src/mongo/db/storage/key_string_to_bson_fuzzer.cpp index 7208a589c54..1b34c79790d 100644 --- a/src/mongo/db/storage/key_string_to_bson_fuzzer.cpp +++ b/src/mongo/db/storage/key_string_to_bson_fuzzer.cpp @@ -36,27 +36,28 @@ const auto kV1 = mongo::KeyString::Version::V1; const auto kV0 = mongo::KeyString::Version::V0; uint8_t getZeroType(char val) { + using mongo::KeyString; switch (val % 10) { case 0: - return mongo::KeyString::TypeBits::kInt; + return KeyString::TypeBits::kInt; case 1: - return mongo::KeyString::TypeBits::kDouble; + return KeyString::TypeBits::kDouble; case 2: - return mongo::KeyString::TypeBits::kLong; + return KeyString::TypeBits::kLong; case 3: - return mongo::KeyString::TypeBits::kNegativeDoubleZero; + return KeyString::TypeBits::kNegativeDoubleZero; case 4: - return mongo::KeyString::TypeBits::kDecimalZero0xxx; + return KeyString::TypeBits::kDecimalZero0xxx; case 5: - return mongo::KeyString::TypeBits::kDecimalZero1xxx; + return KeyString::TypeBits::kDecimalZero1xxx; case 6: - return mongo::KeyString::TypeBits::kDecimalZero2xxx; + return KeyString::TypeBits::kDecimalZero2xxx; case 7: - return mongo::KeyString::TypeBits::kDecimalZero3xxx; + return KeyString::TypeBits::kDecimalZero3xxx; case 8: - return mongo::KeyString::TypeBits::kDecimalZero4xxx; + return KeyString::TypeBits::kDecimalZero4xxx; case 9: - return mongo::KeyString::TypeBits::kDecimalZero5xxx; + return KeyString::TypeBits::kDecimalZero5xxx; default: return 0x00; } @@ -71,12 +72,12 @@ extern "C" int LLVMFuzzerTestOneInput(const char* Data, size_t Size) { mongo::KeyString::TypeBits tb(version); - const size_t len = Data[2]; - if (len > static_cast<size_t>(Size - 3)) + const auto len = Data[2]; + if (len > Size - 3) return 0; // Data[2] defines the number of types to append to the TypeBits // Data[3 + i] defines which types have to be added - for (size_t i = 0; i < len; i++) { + for (int i = 0; i < len; i++) { char randomType = Data[3 + i] & 0xf; char randomZeroType = (Data[3 + i] & 0xf0) >> 4; switch (randomType % 9) { |