diff options
author | Robert Newson <rnewson@apache.org> | 2020-04-08 16:40:26 +0100 |
---|---|---|
committer | Robert Newson <rnewson@apache.org> | 2020-04-27 14:28:03 +0100 |
commit | a5fded8c5d47bcc14699429a46dfb0cec6bd9e76 (patch) | |
tree | d799decd325e40fad50912340f1cf98a0e63e893 | |
parent | 232e1d51fd95c111dd81a19112de56a59e3f2a74 (diff) | |
download | couchdb-a5fded8c5d47bcc14699429a46dfb0cec6bd9e76.tar.gz |
Add native encryption support
A new application, aegis, is introduced to provide strong at-rest
protection of CouchDB data (where possible).
Currently we encrypt the following values (if enabled):
1. Document content
2. Attachment content
3. Index values
Things not encrypted:
1. _all_docs
2. _changes
3. doc id
4. doc rev
5. Index keys
6. All other metadata
Co-Authored-By: Eric Avdey <eiri@apache.org>
Co-Authored-By: Robert Samuel Newson <rnewson@apache.org>
-rwxr-xr-x | configure | 19 | ||||
-rw-r--r-- | rebar.config.script | 1 | ||||
-rw-r--r-- | rel/reltool.config | 2 | ||||
-rw-r--r-- | src/aegis/rebar.config.script | 35 | ||||
-rw-r--r-- | src/aegis/src/aegis.app.src | 34 | ||||
-rw-r--r-- | src/aegis/src/aegis.erl | 72 | ||||
-rw-r--r-- | src/aegis/src/aegis.hrl | 57 | ||||
-rw-r--r-- | src/aegis/src/aegis_app.erl | 26 | ||||
-rw-r--r-- | src/aegis/src/aegis_key_manager.erl | 22 | ||||
-rw-r--r-- | src/aegis/src/aegis_keywrap.erl (renamed from src/couch/src/couch_keywrap.erl) | 34 | ||||
-rw-r--r-- | src/aegis/src/aegis_noop_key_manager.erl | 31 | ||||
-rw-r--r-- | src/aegis/src/aegis_server.erl | 275 | ||||
-rw-r--r-- | src/aegis/src/aegis_sup.erl | 46 | ||||
-rw-r--r-- | src/aegis/test/aegis_basic_test.erl | 17 | ||||
-rw-r--r-- | src/aegis/test/aegis_server_test.erl | 165 | ||||
-rw-r--r-- | src/chttpd/src/chttpd.erl | 8 | ||||
-rw-r--r-- | src/couch_views/src/couch_views_fdb.erl | 8 | ||||
-rw-r--r-- | src/fabric/include/fabric2.hrl | 1 | ||||
-rw-r--r-- | src/fabric/src/fabric.app.src | 3 | ||||
-rw-r--r-- | src/fabric/src/fabric2_fdb.erl | 31 |
20 files changed, 849 insertions, 38 deletions
@@ -96,6 +96,24 @@ parse_opts() { continue ;; + --key-manager) + if [ -n "$2" ]; then + eval AEGIS_KEY_MANAGER=$2 + shift 2 + continue + else + printf 'ERROR: "--key-manager" requires a non-empty argument.\n' >&2 + exit 1 + fi + ;; + --key-manager=?*) + eval AEGIS_KEY_MANAGER=${1#*=} + ;; + --key-manager=) + printf 'ERROR: "--key-manager" requires a non-empty argument.\n' >&2 + exit 1 + ;; + --dev) WITH_DOCS=0 WITH_FAUXTON=0 @@ -241,6 +259,7 @@ cat > $rootdir/config.erl << EOF {with_curl, $WITH_CURL}. {with_proper, $WITH_PROPER}. {erlang_md5, $ERLANG_MD5}. +{aegis_key_manager, "$AEGIS_KEY_MANAGER"}. {spidermonkey_version, "$SM_VSN"}. EOF diff --git a/rebar.config.script b/rebar.config.script index b3ea2c933..2badaba2d 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -135,6 +135,7 @@ SubDirs = [ "src/ddoc_cache", "src/dreyfus", "src/fabric", + "src/aegis", "src/couch_jobs", "src/couch_expiring_cache", "src/global_changes", diff --git a/rel/reltool.config b/rel/reltool.config index 9fbf28544..b59c95f55 100644 --- a/rel/reltool.config +++ b/rel/reltool.config @@ -27,6 +27,7 @@ syntax_tools, xmerl, %% couchdb + aegis, b64url, bear, chttpd, @@ -90,6 +91,7 @@ {app, xmerl, [{incl_cond, include}]}, %% couchdb + {app, aegis, [{incl_cond, include}]}, {app, b64url, [{incl_cond, include}]}, {app, bear, [{incl_cond, include}]}, {app, chttpd, [{incl_cond, include}]}, diff --git a/src/aegis/rebar.config.script b/src/aegis/rebar.config.script new file mode 100644 index 000000000..ef148bfbe --- /dev/null +++ b/src/aegis/rebar.config.script @@ -0,0 +1,35 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + + +CouchConfig = case filelib:is_file(os:getenv("COUCHDB_CONFIG")) of + true -> + {ok, Result} = file:consult(os:getenv("COUCHDB_CONFIG")), + Result; + false -> + [] +end. + +AegisKeyManager = case lists:keyfind(aegis_key_manager, 1, CouchConfig) of + {aegis_key_manager, Module} when Module /= "" -> + list_to_atom(Module); + _ -> + aegis_noop_key_manager +end, + +CurrentOpts = case lists:keyfind(erl_opts, 1, CONFIG) of + {erl_opts, Opts} -> Opts; + false -> [] +end, + +AegisOpts = {d, 'AEGIS_KEY_MANAGER', AegisKeyManager}, +lists:keystore(erl_opts, 1, CONFIG, {erl_opts, [AegisOpts | CurrentOpts]}). diff --git a/src/aegis/src/aegis.app.src b/src/aegis/src/aegis.app.src new file mode 100644 index 000000000..deb152674 --- /dev/null +++ b/src/aegis/src/aegis.app.src @@ -0,0 +1,34 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +{application, aegis, + [ + {description, "If it's good enough for Zeus, it's good enough for CouchDB"}, + {vsn, git}, + {mod, {aegis_app, []}}, + {registered, [ + aegis_server + ]}, + {applications, + [kernel, + stdlib, + crypto, + couch_log, + erlfdb + ]}, + {env,[]}, + {modules, []}, + {maintainers, []}, + {licenses, []}, + {links, []} + ] +}. diff --git a/src/aegis/src/aegis.erl b/src/aegis/src/aegis.erl new file mode 100644 index 000000000..e8a0b4bfb --- /dev/null +++ b/src/aegis/src/aegis.erl @@ -0,0 +1,72 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(aegis). +-include_lib("fabric/include/fabric2.hrl"). + + +-define(WRAPPED_KEY, {?DB_AEGIS, 1}). + + +-export([ + init_db/2, + open_db/1, + + decrypt/2, + decrypt/3, + encrypt/3, + wrap_fold_fun/2 +]). + +init_db(#{} = Db, Options) -> + Db#{ + is_encrypted => aegis_server:init_db(Db, Options) + }. + + +open_db(#{} = Db) -> + Db#{ + is_encrypted => aegis_server:open_db(Db) + }. + + +encrypt(#{} = _Db, _Key, <<>>) -> + <<>>; + +encrypt(#{is_encrypted := false}, _Key, Value) when is_binary(Value) -> + Value; + +encrypt(#{is_encrypted := true} = Db, Key, Value) + when is_binary(Key), is_binary(Value) -> + aegis_server:encrypt(Db, Key, Value). + + +decrypt(#{} = Db, Rows) when is_list(Rows) -> + lists:map(fun({Key, Value}) -> + {Key, decrypt(Db, Key, Value)} + end, Rows). + +decrypt(#{} = _Db, _Key, <<>>) -> + <<>>; + +decrypt(#{is_encrypted := false}, _Key, Value) when is_binary(Value) -> + Value; + +decrypt(#{is_encrypted := true} = Db, Key, Value) + when is_binary(Key), is_binary(Value) -> + aegis_server:decrypt(Db, Key, Value). + + +wrap_fold_fun(Db, Fun) when is_function(Fun, 2) -> + fun({Key, Value}, Acc) -> + Fun({Key, decrypt(Db, Key, Value)}, Acc) + end. diff --git a/src/aegis/src/aegis.hrl b/src/aegis/src/aegis.hrl new file mode 100644 index 000000000..2a2a2dcde --- /dev/null +++ b/src/aegis/src/aegis.hrl @@ -0,0 +1,57 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +%% Assume old crypto api + +-define(sha256_hmac(Key, PlainText), crypto:hmac(sha256, Key, PlainText)). + +-define(aes_gcm_encrypt(Key, IV, AAD, Data), + crypto:block_encrypt(aes_gcm, Key, IV, {AAD, Data, 16})). + +-define(aes_gcm_decrypt(Key, IV, AAD, CipherText, CipherTag), + crypto:block_decrypt(aes_gcm, Key, IV, {AAD, CipherText, CipherTag})). + +-define(aes_ecb_encrypt(Key, Data), + crypto:block_encrypt(aes_ecb, Key, Data)). + +-define(aes_ecb_decrypt(Key, Data), + crypto:block_decrypt(aes_ecb, Key, Data)). + +%% Replace macros if new crypto api is available +-ifdef(OTP_RELEASE). +-if(?OTP_RELEASE >= 22). + +-undef(sha256_hmac). +-define(sha256_hmac(Key, PlainText), crypto:mac(hmac, sha256, Key, PlainText)). + +-undef(aes_gcm_encrypt). +-define(aes_gcm_encrypt(Key, IV, AAD, Data), + crypto:crypto_one_time_aead(aes_256_gcm, Key, IV, Data, AAD, 16, true)). + +-undef(aes_gcm_decrypt). +-define(aes_gcm_decrypt(Key, IV, AAD, CipherText, CipherTag), + crypto:crypto_one_time_aead(aes_256_gcm, Key, IV, CipherText, + AAD, CipherTag, false)). + +-define(key_alg(Key), case bit_size(Key) of + 128 -> aes_128_ecb; 192 -> aes_192_ecb; 256 -> aes_256_ecb end). + +-undef(aes_ecb_encrypt). +-define(aes_ecb_encrypt(Key, Data), + crypto:crypto_one_time(?key_alg(Key), Key, Data, true)). + +-undef(aes_ecb_decrypt). +-define(aes_ecb_decrypt(Key, Data), + crypto:crypto_one_time(?key_alg(Key), Key, Data, false)). + +-endif. +-endif.
\ No newline at end of file diff --git a/src/aegis/src/aegis_app.erl b/src/aegis/src/aegis_app.erl new file mode 100644 index 000000000..4a5a11f0c --- /dev/null +++ b/src/aegis/src/aegis_app.erl @@ -0,0 +1,26 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(aegis_app). + +-behaviour(application). + + +-export([start/2, stop/1]). + + +start(_StartType, _StartArgs) -> + aegis_sup:start_link(). + + +stop(_State) -> + ok. diff --git a/src/aegis/src/aegis_key_manager.erl b/src/aegis/src/aegis_key_manager.erl new file mode 100644 index 000000000..aa9e3429a --- /dev/null +++ b/src/aegis/src/aegis_key_manager.erl @@ -0,0 +1,22 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(aegis_key_manager). + + + +-callback init_db( + Db :: #{}, + DbOptions :: list()) -> {ok, binary()} | false. + + +-callback open_db(Db :: #{}) -> {ok, binary()} | false. diff --git a/src/couch/src/couch_keywrap.erl b/src/aegis/src/aegis_keywrap.erl index 0d1e3f59d..58c7668e8 100644 --- a/src/couch/src/couch_keywrap.erl +++ b/src/aegis/src/aegis_keywrap.erl @@ -1,4 +1,17 @@ --module(couch_keywrap). +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(aegis_keywrap). +-include("aegis.hrl"). %% Implementation of NIST Special Publication 800-38F %% For wrapping and unwrapping keys with AES. @@ -7,25 +20,6 @@ -define(ICV1, 16#A6A6A6A6A6A6A6A6). -%% Assume old crypto api --define(aes_ecb_encrypt(Key, Data), - crypto:block_encrypt(aes_ecb, Key, Data)). --define(aes_ecb_decrypt(Key, Data), - crypto:block_decrypt(aes_ecb, Key, Data)). - -%% Replace macros if new crypto api is available --ifdef(OTP_RELEASE). --if(?OTP_RELEASE >= 22). --define(key_alg(Key), case bit_size(Key) of 128 -> aes_128_ecb; 192 -> aes_192_ecb; 256 -> aes_256_ecb end). --undef(aes_ecb_encrypt). --define(aes_ecb_encrypt(Key, Data), - crypto:crypto_one_time(?key_alg(Key), Key, Data, true)). --undef(aes_ecb_decrypt). --define(aes_ecb_decrypt(Key, Data), - crypto:crypto_one_time(?key_alg(Key), Key, Data, false)). --endif. --endif. - -spec key_wrap(WrappingKey :: binary(), KeyToWrap :: binary()) -> binary(). key_wrap(WrappingKey, KeyToWrap) when is_binary(WrappingKey), bit_size(KeyToWrap) rem 64 == 0 -> diff --git a/src/aegis/src/aegis_noop_key_manager.erl b/src/aegis/src/aegis_noop_key_manager.erl new file mode 100644 index 000000000..2b61f1d29 --- /dev/null +++ b/src/aegis/src/aegis_noop_key_manager.erl @@ -0,0 +1,31 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(aegis_noop_key_manager). + + +-behaviour(aegis_key_manager). + + +-export([ + init_db/2, + open_db/1 +]). + + + +init_db(#{} = _Db, _Options) -> + false. + + +open_db(#{} = _Db) -> + false. diff --git a/src/aegis/src/aegis_server.erl b/src/aegis/src/aegis_server.erl new file mode 100644 index 000000000..be8202ced --- /dev/null +++ b/src/aegis/src/aegis_server.erl @@ -0,0 +1,275 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(aegis_server). + +-behaviour(gen_server). + +-vsn(1). + + +-include("aegis.hrl"). + + +%% aegis_server API +-export([ + start_link/0, + init_db/2, + open_db/1, + encrypt/3, + decrypt/3 +]). + +%% gen_server callbacks +-export([ + init/1, + terminate/2, + handle_call/3, + handle_cast/2, + handle_info/2, + code_change/3 +]). + + + +-define(KEY_CHECK, aegis_key_check). +-define(INIT_TIMEOUT, 60000). +-define(TIMEOUT, 10000). + + +-record(entry, {uuid, encryption_key}). + + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + + +-spec init_db(Db :: #{}, Options :: list()) -> boolean(). +init_db(#{uuid := UUID} = Db, Options) -> + process_flag(sensitive, true), + + case ?AEGIS_KEY_MANAGER:init_db(Db, Options) of + {ok, DbKey} -> + gen_server:call(?MODULE, {insert_key, UUID, DbKey}), + true; + false -> + false + end. + + +-spec open_db(Db :: #{}) -> boolean(). +open_db(#{} = Db) -> + process_flag(sensitive, true), + + case do_open_db(Db) of + {ok, _DbKey} -> + true; + false -> + false + end. + + +-spec encrypt(Db :: #{}, Key :: binary(), Value :: binary()) -> binary(). +encrypt(#{} = Db, Key, Value) when is_binary(Key), is_binary(Value) -> + #{ + uuid := UUID + } = Db, + + case ets:member(?KEY_CHECK, UUID) of + true -> + case gen_server:call(?MODULE, {encrypt, Db, Key, Value}) of + CipherText when is_binary(CipherText) -> + CipherText; + {error, {_Tag, {_C_FileName,_LineNumber}, _Desc} = Reason} -> + couch_log:error("aegis encryption failure: ~p ", [Reason]), + erlang:error(decryption_failed); + {error, Reason} -> + erlang:error(Reason) + end; + false -> + process_flag(sensitive, true), + + {ok, DbKey} = do_open_db(Db), + do_encrypt(DbKey, Db, Key, Value) + end. + + +-spec decrypt(Db :: #{}, Key :: binary(), Value :: binary()) -> binary(). +decrypt(#{} = Db, Key, Value) when is_binary(Key), is_binary(Value) -> + #{ + uuid := UUID + } = Db, + + case ets:member(?KEY_CHECK, UUID) of + true -> + case gen_server:call(?MODULE, {decrypt, Db, Key, Value}) of + PlainText when is_binary(PlainText) -> + PlainText; + {error, {_Tag, {_C_FileName,_LineNumber}, _Desc} = Reason} -> + couch_log:error("aegis decryption failure: ~p ", [Reason]), + erlang:error(decryption_failed); + {error, Reason} -> + erlang:error(Reason) + end; + false -> + process_flag(sensitive, true), + + {ok, DbKey} = do_open_db(Db), + do_decrypt(DbKey, Db, Key, Value) + end. + + +%% gen_server functions + +init([]) -> + process_flag(sensitive, true), + Cache = ets:new(?MODULE, [set, private, {keypos, #entry.uuid}]), + ets:new(?KEY_CHECK, [named_table, protected, {read_concurrency, true}]), + + St = #{ + cache => Cache + }, + {ok, St, ?INIT_TIMEOUT}. + + +terminate(_Reason, _St) -> + ok. + + +handle_call({insert_key, UUID, DbKey}, _From, #{cache := Cache} = St) -> + ok = insert(Cache, UUID, DbKey), + {reply, ok, St, ?TIMEOUT}; + +handle_call({encrypt, #{uuid := UUID} = Db, Key, Value}, From, St) -> + #{ + cache := Cache + } = St, + + {ok, DbKey} = lookup(Cache, UUID), + + erlang:spawn(fun() -> + process_flag(sensitive, true), + try + do_encrypt(DbKey, Db, Key, Value) + of + Resp -> + gen_server:reply(From, Resp) + catch + _:Error -> + gen_server:reply(From, {error, Error}) + end + end), + + {noreply, St, ?TIMEOUT}; + +handle_call({decrypt, #{uuid := UUID} = Db, Key, Value}, From, St) -> + #{ + cache := Cache + } = St, + + {ok, DbKey} = lookup(Cache, UUID), + + erlang:spawn(fun() -> + process_flag(sensitive, true), + try + do_decrypt(DbKey, Db, Key, Value) + of + Resp -> + gen_server:reply(From, Resp) + catch + _:Error -> + gen_server:reply(From, {error, Error}) + end + end), + + {noreply, St, ?TIMEOUT}; + +handle_call(_Msg, _From, St) -> + {noreply, St}. + + +handle_cast(_Msg, St) -> + {noreply, St}. + + +handle_info(_Msg, St) -> + {noreply, St}. + + +code_change(_OldVsn, St, _Extra) -> + {ok, St}. + + +%% private functions + +do_open_db(#{uuid := UUID} = Db) -> + case ?AEGIS_KEY_MANAGER:open_db(Db) of + {ok, DbKey} -> + gen_server:call(?MODULE, {insert_key, UUID, DbKey}), + {ok, DbKey}; + false -> + false + end. + + +do_encrypt(DbKey, #{uuid := UUID}, Key, Value) -> + EncryptionKey = crypto:strong_rand_bytes(32), + <<WrappedKey:320>> = aegis_keywrap:key_wrap(DbKey, EncryptionKey), + + {CipherText, <<CipherTag:128>>} = + ?aes_gcm_encrypt( + EncryptionKey, + <<0:96>>, + <<UUID/binary, 0:8, Key/binary>>, + Value), + <<1:8, WrappedKey:320, CipherTag:128, CipherText/binary>>. + + +do_decrypt(DbKey, #{uuid := UUID}, Key, Value) -> + case Value of + <<1:8, WrappedKey:320, CipherTag:128, CipherText/binary>> -> + case aegis_keywrap:key_unwrap(DbKey, <<WrappedKey:320>>) of + fail -> + erlang:error(decryption_failed); + DecryptionKey -> + Decrypted = + ?aes_gcm_decrypt( + DecryptionKey, + <<0:96>>, + <<UUID/binary, 0:8, Key/binary>>, + CipherText, + <<CipherTag:128>>), + if Decrypted /= error -> Decrypted; true -> + erlang:error(decryption_failed) + end + end; + _ -> + erlang:error(not_ciphertext) + end. + + +%% cache functions + +insert(Cache, UUID, DbKey) -> + Entry = #entry{uuid = UUID, encryption_key = DbKey}, + true = ets:insert(Cache, Entry), + true = ets:insert(?KEY_CHECK, {UUID, true}), + ok. + + +lookup(Cache, UUID) -> + case ets:lookup(Cache, UUID) of + [#entry{uuid = UUID, encryption_key = DbKey}] -> + {ok, DbKey}; + [] -> + {error, not_found} + end. diff --git a/src/aegis/src/aegis_sup.erl b/src/aegis/src/aegis_sup.erl new file mode 100644 index 000000000..6d3ee83d8 --- /dev/null +++ b/src/aegis/src/aegis_sup.erl @@ -0,0 +1,46 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(aegis_sup). + +-behaviour(supervisor). + +-vsn(1). + + +-export([ + start_link/0 +]). + +-export([ + init/1 +]). + + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + + +init([]) -> + Flags = #{ + strategy => one_for_one, + intensity => 5, + period => 10 + }, + Children = [ + #{ + id => aegis_server, + start => {aegis_server, start_link, []}, + shutdown => 5000 + } + ], + {ok, {Flags, Children}}. diff --git a/src/aegis/test/aegis_basic_test.erl b/src/aegis/test/aegis_basic_test.erl new file mode 100644 index 000000000..61d9737dd --- /dev/null +++ b/src/aegis/test/aegis_basic_test.erl @@ -0,0 +1,17 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(aegis_basic_test). + +-include_lib("eunit/include/eunit.hrl"). + +-define(DB, #{uuid => <<"foo">>}). diff --git a/src/aegis/test/aegis_server_test.erl b/src/aegis/test/aegis_server_test.erl new file mode 100644 index 000000000..0f23a3fd9 --- /dev/null +++ b/src/aegis/test/aegis_server_test.erl @@ -0,0 +1,165 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(aegis_server_test). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("couch/include/couch_eunit.hrl"). + +-define(DB, #{uuid => <<0:64>>}). +-define(VALUE, <<0:8>>). +-define(ENCRYPTED, <<1,155,242,89,190,54,112,151,18,145,25,251,217, + 49,147,125,14,162,146,201,189,100,232,38,239,111,163,84,25,60, + 147,167,237,107,24,204,171,232,227,16,72,203,101,118,150,252, + 204,80,245,66,98,213,223,63,111,105,101,154>>). +-define(TIMEOUT, 10000). + + + +basic_test_() -> + { + foreach, + fun setup/0, + fun teardown/1, + [ + {"init_db returns true when encryption enabled", + {timeout, ?TIMEOUT, fun test_init_db/0}}, + {"open_db returns true when encryption enabled", + {timeout, ?TIMEOUT, fun test_open_db/0}}, + {"init_db caches key", + {timeout, ?TIMEOUT, fun test_init_db_cache/0}}, + {"open_db caches key", + {timeout, ?TIMEOUT, fun test_open_db_cache/0}}, + {"encrypt fetches and caches key when it's missing", + {timeout, ?TIMEOUT, fun test_encrypt_cache/0}}, + {"decrypt fetches and caches key when it's missing", + {timeout, ?TIMEOUT, fun test_decrypt_cache/0}} + ] + }. + + +setup() -> + Ctx = test_util:start_couch([fabric]), + meck:new([?AEGIS_KEY_MANAGER], [passthrough]), + ok = meck:expect(?AEGIS_KEY_MANAGER, init_db, 2, {ok, <<0:256>>}), + ok = meck:expect(?AEGIS_KEY_MANAGER, open_db, 1, {ok, <<0:256>>}), + Ctx. + + +teardown(Ctx) -> + meck:unload(), + test_util:stop_couch(Ctx). + + +test_init_db() -> + ?assert(aegis_server:init_db(?DB, [])), + ?assertEqual(1, meck:num_calls(?AEGIS_KEY_MANAGER, init_db, 2)). + + +test_open_db() -> + ?assert(aegis_server:open_db(?DB)), + ?assertEqual(1, meck:num_calls(?AEGIS_KEY_MANAGER, open_db, 1)). + + +test_init_db_cache() -> + ?assertEqual(0, meck:num_calls(?AEGIS_KEY_MANAGER, init_db, 2)), + + ?assert(aegis_server:init_db(?DB, [])), + + lists:foreach(fun(I) -> + Encrypted = aegis_server:encrypt(?DB, <<I:64>>, ?VALUE), + ?assertNotEqual(?VALUE, Encrypted), + ?assertMatch(<<1:8, _/binary>>, Encrypted) + end, lists:seq(1, 12)), + + ?assertEqual(1, meck:num_calls(?AEGIS_KEY_MANAGER, init_db, 2)). + + +test_open_db_cache() -> + ?assertEqual(0, meck:num_calls(?AEGIS_KEY_MANAGER, open_db, 1)), + + ?assert(aegis_server:open_db(?DB)), + + lists:foreach(fun(I) -> + Encrypted = aegis_server:encrypt(?DB, <<I:64>>, ?VALUE), + ?assertNotEqual(?VALUE, Encrypted), + ?assertMatch(<<1:8, _/binary>>, Encrypted) + end, lists:seq(1, 12)), + + ?assertEqual(1, meck:num_calls(?AEGIS_KEY_MANAGER, open_db, 1)). + + +test_encrypt_cache() -> + ?assertEqual(0, meck:num_calls(?AEGIS_KEY_MANAGER, open_db, 1)), + + Encrypted = aegis_server:encrypt(?DB, <<1:64>>, ?VALUE), + ?assertNotEqual(?VALUE, Encrypted), + ?assertMatch(<<1:8, _/binary>>, Encrypted), + + ?assertEqual(1, meck:num_calls(?AEGIS_KEY_MANAGER, open_db, 1)). + + +test_decrypt_cache() -> + ?assertEqual(0, meck:num_calls(?AEGIS_KEY_MANAGER, open_db, 1)), + + Decrypted = aegis_server:decrypt(?DB, <<1:64>>, ?ENCRYPTED), + ?assertEqual(<<0>>, Decrypted), + + ?assertEqual(1, meck:num_calls(?AEGIS_KEY_MANAGER, open_db, 1)). + + + +disabled_test_() -> + { + foreach, + fun() -> + Ctx = setup(), + ok = meck:delete(?AEGIS_KEY_MANAGER, init_db, 2), + ok = meck:expect(?AEGIS_KEY_MANAGER, init_db, 2, false), + ok = meck:delete(?AEGIS_KEY_MANAGER, open_db, 1), + ok = meck:expect(?AEGIS_KEY_MANAGER, open_db, 1, false), + Ctx + end, + fun teardown/1, + [ + {"init_db returns false when encryptions disabled", + {timeout, ?TIMEOUT, fun test_disabled_init_db/0}}, + {"open_db returns false when encryptions disabled", + {timeout, ?TIMEOUT, fun test_disabled_open_db/0}}, + {"pass through on encrypt when encryption disabled", + {timeout, ?TIMEOUT, fun test_disabled_encrypt/0}}, + {"pass through on decrypt when encryption disabled", + {timeout, ?TIMEOUT, fun test_disabled_decrypt/0}} + ] + }. + + +test_disabled_init_db() -> + ?assertNot(aegis_server:init_db(?DB, [])), + ?assertEqual(1, meck:num_calls(?AEGIS_KEY_MANAGER, init_db, 2)). + + +test_disabled_open_db() -> + ?assertNot(aegis_server:open_db(?DB)), + ?assertEqual(1, meck:num_calls(?AEGIS_KEY_MANAGER, open_db, 1)). + + +test_disabled_encrypt() -> + Db = ?DB#{is_encrypted => aegis_server:open_db(?DB)}, + Encrypted = aegis:encrypt(Db, <<1:64>>, ?VALUE), + ?assertEqual(?VALUE, Encrypted). + + +test_disabled_decrypt() -> + Db = ?DB#{is_encrypted => aegis_server:open_db(?DB)}, + Decrypted = aegis:decrypt(Db, <<1:64>>, ?ENCRYPTED), + ?assertEqual(?ENCRYPTED, Decrypted). diff --git a/src/chttpd/src/chttpd.erl b/src/chttpd/src/chttpd.erl index 4640258a8..699601c0e 100644 --- a/src/chttpd/src/chttpd.erl +++ b/src/chttpd/src/chttpd.erl @@ -357,6 +357,10 @@ catch_error(HttpReq, throw, Error) -> send_error(HttpReq, Error); catch_error(HttpReq, error, database_does_not_exist) -> send_error(HttpReq, database_does_not_exist); +catch_error(HttpReq, error, decryption_failed) -> + send_error(HttpReq, decryption_failed); +catch_error(HttpReq, error, not_ciphertext) -> + send_error(HttpReq, not_ciphertext); catch_error(HttpReq, Tag, Error) -> Stack = erlang:get_stacktrace(), % TODO improve logging and metrics collection for client disconnects @@ -965,6 +969,10 @@ error_info(not_implemented) -> error_info(timeout) -> {500, <<"timeout">>, <<"The request could not be processed in a reasonable" " amount of time.">>}; +error_info(decryption_failed) -> + {500, <<"decryption_failed">>, <<"Decryption failed">>}; +error_info(not_ciphertext) -> + {500, <<"not_ciphertext">>, <<"Not Ciphertext">>}; error_info({service_unavailable, Reason}) -> {503, <<"service unavailable">>, Reason}; error_info({timeout, _Reason}) -> diff --git a/src/couch_views/src/couch_views_fdb.erl b/src/couch_views/src/couch_views_fdb.erl index 2181e5373..c95722230 100644 --- a/src/couch_views/src/couch_views_fdb.erl +++ b/src/couch_views/src/couch_views_fdb.erl @@ -161,7 +161,7 @@ fold_map_idx(TxDb, Sig, ViewId, Options, Callback, Acc0) -> callback => Callback, acc => Acc0 }, - Fun = fun fold_fwd/2, + Fun = aegis:wrap_fold_fun(TxDb, fun fold_fwd/2), #{ acc := Acc1 @@ -321,7 +321,7 @@ update_id_idx(TxDb, Sig, ViewId, DocId, NewRows, KVSize) -> Key = id_idx_key(DbPrefix, Sig, DocId, ViewId), Val = couch_views_encoding:encode([length(NewRows), KVSize, Unique]), - ok = erlfdb:set(Tx, Key, Val). + ok = erlfdb:set(Tx, Key, aegis:encrypt(TxDb, Key, Val)). update_map_idx(TxDb, Sig, ViewId, DocId, ExistingKeys, NewRows) -> @@ -341,7 +341,7 @@ update_map_idx(TxDb, Sig, ViewId, DocId, ExistingKeys, NewRows) -> lists:foreach(fun({DupeId, Key1, Key2, EV}) -> KK = map_idx_key(MapIdxPrefix, {Key1, DocId}, DupeId), Val = erlfdb_tuple:pack({Key2, EV}), - ok = erlfdb:set(Tx, KK, Val) + ok = erlfdb:set(Tx, KK, aegis:encrypt(TxDb, KK, Val)) end, KVsToAdd). @@ -356,7 +356,7 @@ get_view_keys(TxDb, Sig, DocId) -> erlfdb_tuple:unpack(K, DbPrefix), [TotalKeys, TotalSize, UniqueKeys] = couch_views_encoding:decode(V), {ViewId, TotalKeys, TotalSize, UniqueKeys} - end, erlfdb:get_range(Tx, Start, End, [])). + end, aegis:decrypt(TxDb, erlfdb:get_range(Tx, Start, End, []))). update_row_count(TxDb, Sig, ViewId, Increment) -> diff --git a/src/fabric/include/fabric2.hrl b/src/fabric/include/fabric2.hrl index bf3e2aa03..234c5291e 100644 --- a/src/fabric/include/fabric2.hrl +++ b/src/fabric/include/fabric2.hrl @@ -42,6 +42,7 @@ -define(DB_LOCAL_DOC_BODIES, 25). -define(DB_ATT_NAMES, 26). -define(DB_SEARCH, 27). +-define(DB_AEGIS, 28). % Versions diff --git a/src/fabric/src/fabric.app.src b/src/fabric/src/fabric.app.src index 0538b19b4..a7059fd10 100644 --- a/src/fabric/src/fabric.app.src +++ b/src/fabric/src/fabric.app.src @@ -28,6 +28,7 @@ mem3, couch_log, couch_stats, - erlfdb + erlfdb, + aegis ]} ]}. diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl index b1ada52fc..ba57e646d 100644 --- a/src/fabric/src/fabric2_fdb.erl +++ b/src/fabric/src/fabric2_fdb.erl @@ -181,7 +181,7 @@ create(#{} = Db0, Options) -> name := DbName, tx := Tx, layer_prefix := LayerPrefix - } = Db = ensure_current(Db0, false), + } = Db1 = ensure_current(Db0, false), DbKey = erlfdb_tuple:pack({?ALL_DBS, DbName}, LayerPrefix), HCA = erlfdb_hca:create(erlfdb_tuple:pack({?DB_HCA}, LayerPrefix)), @@ -224,7 +224,7 @@ create(#{} = Db0, Options) -> UserCtx = fabric2_util:get_value(user_ctx, Options, #user_ctx{}), Options1 = lists:keydelete(user_ctx, 1, Options), - Db#{ + Db2 = Db1#{ uuid => UUID, db_prefix => DbPrefix, db_version => DbVersion, @@ -240,7 +240,8 @@ create(#{} = Db0, Options) -> % All other db things as we add features, db_options => Options1 - }. + }, + aegis:init_db(Db2, Options). open(#{} = Db0, Options) -> @@ -286,14 +287,15 @@ open(#{} = Db0, Options) -> }, Db3 = load_config(Db2), + Db4 = aegis:open_db(Db3), - case {UUID, Db3} of + case {UUID, Db4} of {undefined, _} -> ok; {<<_/binary>>, #{uuid := UUID}} -> ok; {<<_/binary>>, #{uuid := _}} -> erlang:error(database_does_not_exist) end, - load_validate_doc_funs(Db3). + load_validate_doc_funs(Db4). % Match on `name` in the function head since some non-fabric2 db @@ -701,9 +703,10 @@ get_doc_body_wait(#{} = Db0, DocId, RevInfo, Future) -> rev_path := RevPath } = RevInfo, - RevBodyRows = erlfdb:fold_range_wait(Tx, Future, fun({_K, V}, Acc) -> + FoldFun = aegis:wrap_fold_fun(Db, fun({_K, V}, Acc) -> [V | Acc] - end, []), + end), + RevBodyRows = erlfdb:fold_range_wait(Tx, Future, FoldFun, []), BodyRows = lists:reverse(RevBodyRows), fdb_to_doc(Db, DocId, RevPos, [Rev | RevPath], BodyRows). @@ -720,7 +723,7 @@ get_local_doc(#{} = Db0, <<?LOCAL_DOC_PREFIX, _/binary>> = DocId) -> Prefix = erlfdb_tuple:pack({?DB_LOCAL_DOC_BODIES, DocId}, DbPrefix), Future = erlfdb:get_range_startswith(Tx, Prefix), - Chunks = lists:map(fun({_K, V}) -> V end, erlfdb:wait(Future)), + {_, Chunks} = lists:unzip(aegis:decrypt(Db, erlfdb:wait(Future))), fdb_to_local_doc(Db, DocId, Rev, Chunks). @@ -949,7 +952,9 @@ write_local_doc(#{} = Db0, Doc) -> % Make sure to clear the whole range, in case there was a larger % document body there before. erlfdb:clear_range_startswith(Tx, BPrefix), - lists:foreach(fun({K, V}) -> erlfdb:set(Tx, K, V) end, Rows) + lists:foreach(fun({K, V}) -> + erlfdb:set(Tx, K, aegis:encrypt(Db, K, V)) + end, Rows) end, case {WasDeleted, Doc#doc.deleted} of @@ -977,8 +982,8 @@ read_attachment(#{} = Db, DocId, AttId) -> not_found -> throw({not_found, missing}); KVs -> - Vs = [V || {_K, V} <- KVs], - iolist_to_binary(Vs) + {_, Chunks} = lists:unzip(aegis:decrypt(Db, KVs)), + iolist_to_binary(Chunks) end, IdKey = erlfdb_tuple:pack({?DB_ATT_NAMES, DocId, AttId}, DbPrefix), @@ -1023,7 +1028,7 @@ write_attachment(#{} = Db, DocId, Data, Encoding) lists:foldl(fun(Chunk, ChunkId) -> AttKey = erlfdb_tuple:pack({?DB_ATTS, DocId, AttId, ChunkId}, DbPrefix), - ok = erlfdb:set(Tx, AttKey, Chunk), + ok = erlfdb:set(Tx, AttKey, aegis:encrypt(Db, AttKey, Chunk)), ChunkId + 1 end, 0, Chunks), {ok, AttId}. @@ -1332,7 +1337,7 @@ write_doc_body(#{} = Db0, #doc{} = Doc) -> Rows = doc_to_fdb(Db, Doc), lists:foreach(fun({Key, Value}) -> - ok = erlfdb:set(Tx, Key, Value) + ok = erlfdb:set(Tx, Key, aegis:encrypt(Db, Key, Value)) end, Rows). |