/** * Copyright (C) 2022-present MongoDB, Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the Server Side Public License, version 1, * as published by MongoDB, Inc. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * Server Side Public License for more details. * * You should have received a copy of the Server Side Public License * along with this program. If not, see * . * * As a special exception, the copyright holders give permission to link the * code of portions of this program with the OpenSSL library under certain * conditions as described in each individual source file and distribute * linked combinations including the program with the OpenSSL library. You * must comply with the Server Side Public License in all respects for * all of the code used other than as permitted herein. If you modify file(s) * with this exception, you may extend this exception to your version of the * file(s), but you are not obligated to do so. If you do not wish to do so, * delete this exception statement from your version. If you delete this * exception statement from all source files in the program, then also delete * it in the license file. */ #pragma once #include #include #include #include #include #include #include "mongo/base/string_data.h" #include "mongo/bson/bsonobj.h" #include "mongo/unittest/test_info.h" namespace mongo::unittest { /** * Allows executing golden data tests. That is, tests that produce a text output which is compared * against checked-in expected results (a.k.a "golden data".) * * The test fails if its output doesn't match the golden file's contents, or if * the golden data file doesn't exist. * if expected output doesnt exist. When this happens, the actual and expected outputs will be * stored in the configured output location. This allows: * - bulk comparison to determine if further code changes are needed or if the new results are * acceptable. * - bulk update of expected outputs if the new outputs are acceptable. * * Usage: * GoldenTestConfig myConfig("src/mongo/my_expected_output"); * TEST(MySuite, MyTest) { * GoldenTestContext ctx(myConfig) * ctx.outStream() << "print something here" << std::endl; * ctx.outStream() << "print something else" << std::endl; * } * * For complete usage guide and documentation see: docs/golden_data_test_framework.md */ /** * A configuration specific to each golden test suite. */ struct GoldenTestConfig { /** * A relative path to the golden data files. The path is relative to root of the repo. * This path can be shared by multiple suites. * * It is recommended to keep golden data is a separate subfolder from other source code files. * Good: * src/mongo/unittest/expected_output * src/mongo/my_module/my_sub_module/expected_output * * Bad: * src/mongo/my_module/ * src * ../.. * /etc * C:\Windows */ std::string relativePath; static GoldenTestConfig parseFromBson(const BSONObj&); }; /** * Global environment shared across all golden test suites. * Specifically, output directory is shared across all suites to allow simple directory diffing, * even if multiple suites were executed. */ class GoldenTestEnvironment { private: GoldenTestEnvironment(); public: GoldenTestEnvironment(const GoldenTestEnvironment&) = delete; GoldenTestEnvironment& operator=(const GoldenTestEnvironment&) = delete; static GoldenTestEnvironment* getInstance(); boost::filesystem::path actualOutputRoot() const { return _actualOutputRoot; } boost::filesystem::path expectedOutputRoot() const { return _expectedOutputRoot; } boost::filesystem::path goldenDataRoot() const { return _goldenDataRoot; } std::string diffCmd(const std::string& expectedOutputFile, const std::string& actualOutputFile) const; private: boost::filesystem::path _goldenDataRoot; boost::filesystem::path _actualOutputRoot; boost::filesystem::path _expectedOutputRoot; std::string _diffCmd; }; /** * Context for each golden test that can be used to accumulate, verify and optionally overwrite test * output data. Format of the output data is left to the test implementation. However it is * recommended that the output: 1) Is in text format. 2) Can be udated incrementally. Incremental * changes to the production or test code should result in incremental changes to the test output. * This reduces the size the side of diffs and reduces chances of conflicts. 3) Includes both input * and output. This helps with inspecting the changes, without need to pattern match across files. */ class GoldenTestContextBase { public: explicit GoldenTestContextBase( const GoldenTestConfig* config, boost::filesystem::path testPath, bool validateOnClose, std::function& expectedStr)> onError) : _env(GoldenTestEnvironment::getInstance()), _config(config), _testPath(testPath), _validateOnClose(validateOnClose), _onError(onError) {} ~GoldenTestContextBase() noexcept(false) { if (_validateOnClose && !std::uncaught_exceptions()) { verifyOutput(); } } /** * Returns the output stream that a test should write its output to. * The output that is written here will be compared against expected golden data. If the output * that test produced differs from the expected output, the test will fail. */ std::ostream& outStream() { return _outStream; } /** * Verifies that output accumulated in this context matches the expected output golden data. * If output does not match, the test fails with TestAssertionFailureException. * * Additionally, in case of mismatch: * - a file with the actual test output is created. * - a file with the expected output is created: * this preserves the snapshot of the golden data that was used for verification, as the * files in the source repo can subsequently change. Output files are written to a temp files * location unless configured otherwise. */ void verifyOutput(); /** * Returns the path where the actual test output will be written. */ boost::filesystem::path getActualOutputPath() const; /** * Returns the path where the expected test output will be written. */ boost::filesystem::path getExpectedOutputPath() const; /** * Returns the path to the golden data used for verification. */ boost::filesystem::path getGoldenDataPath() const; /** * Returns relative test path. Typically composed of suite and test names. */ boost::filesystem::path getTestPath() const; protected: static std::string sanitizeName(const std::string& str); const GoldenTestEnvironment* getEnv() const { return _env; } private: static std::string toSnakeCase(const std::string& str); static boost::filesystem::path toTestPath(const std::string& suiteName, const std::string& testName); void failResultMismatch(const std::string& actualStr, const boost::optional& expectedStr, const std::string& messsage); GoldenTestEnvironment* _env; const GoldenTestConfig* _config; const boost::filesystem::path _testPath; bool _validateOnClose; std::ostringstream _outStream; // Use a callback instead of a virtual function, because calling virtual functions doesn't work // properly in a destructor, even a virtual destructor. const std::function& expectedStr)> _onError; }; /** * Represents configuration variables used by golden tests. */ struct GoldenTestOptions { /** * Parses the options from environment variables that start with GOLDEN_TEST_ prefix. * Supported options: * - GOLDEN_TEST_CONFIG_PATH: (optional) specifies the yaml config file. * See config file reference: * docs/golden_data_test_framework.md#appendix---config-file-reference */ static GoldenTestOptions parseEnvironment(); /** * Root path patten that will be used to write expected and actual test outputs for all tests in * the test run. If not specified a temporary folder location will be used. Path pattern string * may use '%' characters in the last part of the path. '%' characters will be replaced with * random lowercase hexadecimal digits. */ boost::optional outputRootPattern; /** * Shell command to diff a single golden test run output. * {{expected}} and {{actual}} variables should be used and will be replaced with expected and * actual output paths respectively. */ boost::optional diffCmd; }; } // namespace mongo::unittest