From 7967db9bdb5766afc20e2e6a0adf7276f3babf3d Mon Sep 17 00:00:00 2001 From: Mark Benvenuto Date: Thu, 17 Feb 2022 10:36:13 -0500 Subject: SERVER-63738 Add AES support for no-ssl builds --- src/mongo/crypto/SConscript | 102 +++--- src/mongo/crypto/aead_encryption.h | 1 - src/mongo/crypto/fle_crypto_test.cpp | 9 +- src/mongo/crypto/fle_data_frames.h | 1 - src/mongo/crypto/symmetric_crypto_tom.cpp | 371 +++++++++++++++++++++ .../db/storage/wiredtiger/wiredtiger_kv_engine.cpp | 4 +- 6 files changed, 428 insertions(+), 60 deletions(-) create mode 100644 src/mongo/crypto/symmetric_crypto_tom.cpp diff --git a/src/mongo/crypto/SConscript b/src/mongo/crypto/SConscript index 4b558132a7f..c4ef4f52973 100644 --- a/src/mongo/crypto/SConscript +++ b/src/mongo/crypto/SConscript @@ -25,64 +25,56 @@ env.Library('sha256_block', '$BUILD_DIR/mongo/util/secure_compare_memory', ]) +cryptoEnv = env.Clone() + if "tom" in env["MONGO_CRYPTO"]: - tomEnv = env.Clone() - tomEnv.InjectThirdParty(libraries=['tomcrypt']) - tomEnv.Append( + cryptoEnv.InjectThirdParty(libraries=['tomcrypt']) + cryptoEnv.Append( CPPDEFINES=[ 'LTC_NO_PROTOTYPES', ] ) - tomEnv.Library( - target='sha_block_tom', - source=[ - 'sha_block_tom.cpp' - ], - LIBDEPS=[ - '$BUILD_DIR/mongo/base', - 'sha1_block', - 'sha256_block', - ], - LIBDEPS_PRIVATE=[ - '$BUILD_DIR/third_party/shim_tomcrypt', - ] - ) - -else: - env.Library('sha_block_${MONGO_CRYPTO}', - source=[ - 'sha_block_${MONGO_CRYPTO}.cpp' - ], - LIBDEPS=[ - '$BUILD_DIR/mongo/base', - 'sha1_block', - 'sha256_block', - ]) +cryptoEnv.Library( + target='sha_block_${MONGO_CRYPTO}', + source=[ + 'sha_block_${MONGO_CRYPTO}.cpp' + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + 'sha1_block', + 'sha256_block', + ], + LIBDEPS_PRIVATE=[ + '$BUILD_DIR/third_party/shim_tomcrypt' if "tom" in env["MONGO_CRYPTO"] else [], + ] +) -if get_option('ssl') == 'on': - env.Library(target='symmetric_crypto', - source=[ - 'symmetric_crypto.cpp', - 'symmetric_crypto_${MONGO_CRYPTO}.cpp', - 'symmetric_key.cpp', - ], - LIBDEPS=[ - '$BUILD_DIR/mongo/base/secure_allocator', - '$BUILD_DIR/mongo/util/secure_zero_memory', - ], - ) +cryptoEnv.Library(target='symmetric_crypto', + source=[ + 'symmetric_crypto.cpp', + 'symmetric_crypto_${MONGO_CRYPTO}.cpp', + 'symmetric_key.cpp', + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base/secure_allocator', + '$BUILD_DIR/mongo/util/secure_zero_memory', + ], + LIBDEPS_PRIVATE=[ + '$BUILD_DIR/third_party/shim_tomcrypt' if "tom" in env["MONGO_CRYPTO"] else [], + ] +) - env.Library( - target="aead_encryption", - source=[ - "aead_encryption.cpp", - ], - LIBDEPS=[ - '$BUILD_DIR/mongo/db/matcher/expressions', - 'symmetric_crypto', - ], - ) +env.Library( + target="aead_encryption", + source=[ + "aead_encryption.cpp", + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/db/matcher/expressions', + 'symmetric_crypto', + ], +) env.Library( target="encrypted_field_config", @@ -114,19 +106,19 @@ env.Library( env.CppUnitTest( target='crypto_test', source=[ - 'aead_encryption_test.cpp' if "tom" not in env["MONGO_CRYPTO"] else [], + 'aead_encryption_test.cpp', 'mechanism_scram_test.cpp', 'sha1_block_test.cpp', - 'fle_crypto_test.cpp' if "tom" not in env["MONGO_CRYPTO"] else [], + 'fle_crypto_test.cpp', 'sha256_block_test.cpp', 'sha512_block_test.cpp', - 'symmetric_crypto_test.cpp' if "tom" not in env["MONGO_CRYPTO"] else [], + 'symmetric_crypto_test.cpp', ], LIBDEPS=[ '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/base/secure_allocator', - 'aead_encryption' if "tom" not in env["MONGO_CRYPTO"] else [], - 'fle_crypto' if "tom" not in env["MONGO_CRYPTO"] else [], + 'aead_encryption', + 'fle_crypto', 'sha_block_${MONGO_CRYPTO}', ], ) diff --git a/src/mongo/crypto/aead_encryption.h b/src/mongo/crypto/aead_encryption.h index ce474c4d666..d6b95b27aa7 100644 --- a/src/mongo/crypto/aead_encryption.h +++ b/src/mongo/crypto/aead_encryption.h @@ -33,7 +33,6 @@ #include #include "mongo/crypto/fle_data_frames.h" -#include "mongo/shell/kms_gen.h" #include "mongo/base/data_view.h" #include "mongo/base/status.h" diff --git a/src/mongo/crypto/fle_crypto_test.cpp b/src/mongo/crypto/fle_crypto_test.cpp index e9da2a86c44..6cf4513f186 100644 --- a/src/mongo/crypto/fle_crypto_test.cpp +++ b/src/mongo/crypto/fle_crypto_test.cpp @@ -48,9 +48,11 @@ #include "mongo/bson/json.h" #include "mongo/bson/oid.h" #include "mongo/bson/timestamp.h" +#include "mongo/config.h" #include "mongo/db/matcher/schema/encrypt_schema_gen.h" #include "mongo/idl/idl_parser.h" #include "mongo/platform/decimal128.h" +#include "mongo/rpc/object_check.h" #include "mongo/unittest/bson_test_util.h" #include "mongo/unittest/unittest.h" #include "mongo/util/assert_util.h" @@ -58,6 +60,10 @@ #include "mongo/util/time_support.h" +// TODO (SERVER-63780) - remove once we stop using AEAD for ECC crypto with empty associated data +// Tomcrypt SHA implementation cannot accept it so we skip these tests for now +#ifdef MONGO_CONFIG_SSL + namespace mongo { template @@ -313,7 +319,6 @@ TEST(FLE_ESC, RoundTrip) { } } - TEST(FLE_ECC, RoundTrip) { ConstDataRange value(testValue); @@ -891,3 +896,5 @@ TEST(FLE_EDC, DuplicateSafeContent_IncompatibleType) { } } // namespace mongo + +#endif diff --git a/src/mongo/crypto/fle_data_frames.h b/src/mongo/crypto/fle_data_frames.h index a72b31dceb1..c2668454b43 100644 --- a/src/mongo/crypto/fle_data_frames.h +++ b/src/mongo/crypto/fle_data_frames.h @@ -32,7 +32,6 @@ #include "mongo/base/data_range.h" #include "mongo/crypto/symmetric_crypto.h" #include "mongo/db/matcher/schema/encrypt_schema_gen.h" -#include "mongo/shell/kms_gen.h" #include "mongo/util/uuid.h" namespace mongo { diff --git a/src/mongo/crypto/symmetric_crypto_tom.cpp b/src/mongo/crypto/symmetric_crypto_tom.cpp new file mode 100644 index 00000000000..174bcb8e0b7 --- /dev/null +++ b/src/mongo/crypto/symmetric_crypto_tom.cpp @@ -0,0 +1,371 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kStorage + +#include "mongo/platform/basic.h" + +#include +#include +#include +#include +#include +#include + +#include "mongo/base/data_cursor.h" +#include "mongo/base/data_range.h" +#include "mongo/base/error_codes.h" +#include "mongo/base/init.h" +#include "mongo/base/status.h" +#include "mongo/config.h" +#include "mongo/crypto/symmetric_crypto.h" +#include "mongo/crypto/symmetric_key.h" +#include "mongo/platform/random.h" +#include "mongo/util/assert_util.h" +#include "mongo/util/net/ssl_manager.h" +#include "mongo/util/str.h" + +#include + +#ifdef MONGO_CONFIG_SSL +#error This file should not be included if compiling with SSL support +#endif + +namespace mongo { +namespace crypto { + +namespace { + +/** + * Since tomcrypt only does block aligned crypto, we have to buffer bytes into blocks. + * + * This is implementation is simple but inefficient. It only needs to support the non-ssl build + * which is not shipped to customers. + * + * Also, tomcrypt does not support padding in 1.8.2 so we implement PKCS#7 padding. + */ +class BufferedCryptoStream { +public: + virtual ~BufferedCryptoStream() = default; + + virtual StatusWith doBlockAlignedOperation(DataRange buf, size_t size) = 0; + + StatusWith addData(ConstDataRange cdr, DataRange out, bool encrypt) { + // Add all the data in + std::copy(cdr.data(), cdr.data() + cdr.length(), std::back_inserter(_buffer)); + + // Encrypt/decrypt all the blocks we can + size_t blocks = _buffer.size() / aesBlockSize; + size_t payloadSize = blocks * aesBlockSize; + + if (payloadSize > 0) { + auto swOp = doBlockAlignedOperation(_buffer, payloadSize); + if (!swOp.isOK()) { + return swOp; + } + + // For decryption, save the last block as it is supposed a block of pads + if (!encrypt) { + payloadSize -= aesBlockSize; + } + + memcpy(out.data(), _buffer.data(), payloadSize); + + // Remove everything that we just encrypted/decrypted + _buffer.erase(_buffer.begin(), _buffer.begin() + payloadSize); + + return payloadSize; + } else { + return 0; + } + } + + StatusWith finalizeBufferForDecryption(DataRange out) { + if (_buffer.size() != aesBlockSize) { + return Status(ErrorCodes::BadValue, "invalid final block buffer"); + } + + uint8_t pad = _buffer[_buffer.size() - 1]; + if (pad == 0 || pad > aesBlockSize) { + return Status(ErrorCodes::BadValue, "wrong pad length"); + } + + // Validate pad + for (size_t i = 0; i < pad; i++) { + if (_buffer[aesBlockSize - i - 1] != pad) { + return Status(ErrorCodes::BadValue, "wrong pad byte"); + } + } + + size_t final_size = aesBlockSize - pad; + if (final_size == 0) { + return 0; + } + + memcpy(out.data(), _buffer.data(), final_size); + + return {final_size}; + } + + StatusWith finalizeBufferForEncryption(DataRange out) { + std::array buffer; + + if (_buffer.size() == 0) { + buffer.fill(aesBlockSize); + } else { + invariant(_buffer.size() < aesBlockSize); + + // Pad + std::copy(_buffer.data(), _buffer.data() + _buffer.size(), buffer.begin()); + + uint8_t padding_pkcs7 = aesBlockSize - _buffer.size(); + uint8_t start = _buffer.size(); + for (std::size_t i = 0; i < padding_pkcs7; i++) { + buffer[start + i] = padding_pkcs7; + } + } + + auto swOp = doBlockAlignedOperation(buffer, aesBlockSize); + if (!swOp.isOK()) { + return swOp; + } + + memcpy(out.data(), buffer.data(), aesBlockSize); + + return {buffer.size()}; + } + +private: + std::vector _buffer; +}; + +class TomCryptSetup { +public: + TomCryptSetup() { + invariant(register_cipher(&aes_desc) != -1); + _cipher = find_cipher("aes"); + } + + int getCipher() { + return _cipher; + } + +private: + int _cipher{0}; +}; + +static TomCryptSetup& getTomCryptSetup() { + static TomCryptSetup loader; + return loader; +} + +class SymmetricEncryptorTomCrypt : public SymmetricEncryptor, BufferedCryptoStream { +public: + SymmetricEncryptorTomCrypt(const SymmetricKey& key, aesMode mode, ConstDataRange iv) + : _mode(mode) { + if (_mode == crypto::aesMode::cbc) { + int ret = cbc_start(getTomCryptSetup().getCipher(), + iv.data(), + key.getKey(), + key.getKeySize(), + 0, + &_contextCBC); + uassert(6373801, "cbc encrypt init failed", ret == CRYPT_OK); + + } else if (_mode == crypto::aesMode::ctr) { + int ret = ctr_start(getTomCryptSetup().getCipher(), + iv.data(), + key.getKey(), + key.getKeySize(), + 0, + CTR_COUNTER_BIG_ENDIAN, + &_contextCTR); + uassert(6373802, "ctr decrypt init failed", ret == CRYPT_OK); + } else { + MONGO_UNREACHABLE; + } + } + + StatusWith doBlockAlignedOperation(DataRange buf, size_t size) final { + if (_mode == crypto::aesMode::cbc) { + int ret = cbc_encrypt(buf.data(), buf.data(), size, &_contextCBC); + uassert(6373803, "cbc encrypt failed", ret == CRYPT_OK); + } else { + uasserted(6373804, "not supported"); + } + + return size; + } + + StatusWith update(ConstDataRange in, DataRange out) final { + if (_mode == crypto::aesMode::cbc) { + return addData(in, out, true); + } else if (_mode == crypto::aesMode::ctr) { + int ret = + ctr_encrypt(in.data(), out.data(), in.length(), &_contextCTR); + uassert(6373805, "ctr encrypt failed", ret == CRYPT_OK); + } + + return in.length(); + } + + Status addAuthenticatedData(ConstDataRange in) final { + fassert(6373806, _mode == crypto::aesMode::gcm); + + return Status::OK(); + } + + StatusWith finalize(DataRange out) final { + if (_mode == crypto::aesMode::cbc) { + auto sw = finalizeBufferForEncryption(out); + cbc_done(&_contextCBC); + return sw; + } + + return 0; + } + + StatusWith finalizeTag(DataRange out) final { + + return Status(ErrorCodes::UnsupportedFormat, "GCM support is not available"); + } + +private: + const aesMode _mode; + symmetric_CBC _contextCBC; + symmetric_CTR _contextCTR; +}; + +class SymmetricDecryptorTomCrypt : public SymmetricDecryptor, BufferedCryptoStream { +public: + SymmetricDecryptorTomCrypt(const SymmetricKey& key, aesMode mode, ConstDataRange iv) + : _mode(mode) { + if (_mode == crypto::aesMode::cbc) { + int ret = cbc_start(getTomCryptSetup().getCipher(), + iv.data(), + key.getKey(), + key.getKeySize(), + 0, + &_contextCBC); + uassert(6373807, "cbc decrypt init failed", ret == CRYPT_OK); + } else if (_mode == crypto::aesMode::ctr) { + int ret = ctr_start(getTomCryptSetup().getCipher(), + iv.data(), + key.getKey(), + key.getKeySize(), + 0, + CTR_COUNTER_BIG_ENDIAN, + &_contextCTR); + uassert(6373808, "ctr decrypt init failed", ret == CRYPT_OK); + } else { + MONGO_UNREACHABLE; + } + } + + StatusWith doBlockAlignedOperation(DataRange buf, size_t size) final { + if (_mode == crypto::aesMode::cbc) { + int ret = cbc_decrypt(buf.data(), buf.data(), size, &_contextCBC); + uassert(6373809, "cbc decrypt failed", ret == CRYPT_OK); + } else { + uasserted(6373810, "not supported"); + } + + return size; + } + + StatusWith update(ConstDataRange in, DataRange out) final { + if (_mode == crypto::aesMode::cbc) { + return addData(in, out, false); + } else if (_mode == crypto::aesMode::ctr) { + int ret = + ctr_decrypt(in.data(), out.data(), in.length(), &_contextCTR); + uassert(6373811, "ctr decrypt failed", ret == CRYPT_OK); + } + + return in.length(); + } + + Status addAuthenticatedData(ConstDataRange authData) final { + fassert(6373812, _mode == crypto::aesMode::gcm); + + return Status::OK(); + } + + StatusWith finalize(DataRange out) final { + if (_mode == crypto::aesMode::cbc) { + auto sw = finalizeBufferForDecryption(out); + cbc_done(&_contextCBC); + return sw; + } + return 0; + } + + Status updateTag(ConstDataRange tag) final { + return {ErrorCodes::BadValue, "Unexpected tag for non-gcm cipher"}; + } + +private: + const aesMode _mode; + symmetric_CBC _contextCBC; + symmetric_CTR _contextCTR; +}; + +} // namespace + +std::set getSupportedSymmetricAlgorithms() { + return {aes256CBCName, aes256CTRName}; +} + +Status engineRandBytes(DataRange buffer) { + SecureRandom().fill(buffer.data(), buffer.length()); + return Status::OK(); +} + +StatusWith> SymmetricEncryptor::create(const SymmetricKey& key, + aesMode mode, + ConstDataRange iv) try { + std::unique_ptr encryptor = + std::make_unique(key, mode, iv); + return std::move(encryptor); +} catch (const DBException& e) { + return e.toStatus(); +} + +StatusWith> SymmetricDecryptor::create(const SymmetricKey& key, + aesMode mode, + ConstDataRange iv) try { + std::unique_ptr decryptor = + std::make_unique(key, mode, iv); + return std::move(decryptor); +} catch (const DBException& e) { + return e.toStatus(); +} + +} // namespace crypto +} // namespace mongo diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp index 111acf1411f..32fa2099c19 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp @@ -2548,8 +2548,8 @@ void WiredTigerKVEngine::unpinOldestTimestamp(const std::string& requestingServi stdx::lock_guard lock(_oldestTimestampPinRequestsMutex); auto it = _oldestTimestampPinRequests.find(requestingServiceName); if (it == _oldestTimestampPinRequests.end()) { - LOGV2_DEBUG(2, - 5380105, + LOGV2_DEBUG(5380105, + 2, "The requested service had nothing to unpin", "service"_attr = requestingServiceName); return; -- cgit v1.2.1