Add a generic 'props' list at db creation time

This props list is recorded in each database shard as well as the
shard document in the special _dbs database.

Co-authored-by: Garren Smith <garren.smith@gmail.com>
Co-authored-by: Robert Newson <rnewson@apache.org>

10 files changed, 185 insertions, 11 deletions

diff --git a/src/couch/src/couch_bt_engine.erl b/src/couch/src/couch_bt_engine.erl
index ee0d6d864..6748e988a 100644
--- a/src/couch/src/couch_bt_engine.erl
+++ b/src/couch/src/couch_bt_engine.erl
@@ -39,12 +39,16 @@
     get_purge_seq/1,
     get_revs_limit/1,
     get_security/1,
+    get_props/1,
+    get_prop/2,
+    get_prop/3,
     get_size_info/1,
     get_update_seq/1,
     get_uuid/1,
 
     set_revs_limit/2,
     set_security/2,
+    set_prop/3,
 
     open_docs/2,
     open_local_docs/2,
@@ -93,7 +97,8 @@
 -export([
     set_update_seq/2,
     update_header/2,
-    copy_security/2
+    copy_security/2,
+    copy_props/2
 ]).
 
 
@@ -132,8 +137,9 @@ init(FilePath, Options) ->
         true ->
             delete_compaction_files(FilePath),
             Header0 = couch_bt_engine_header:new(),
-            ok = couch_file:write_header(Fd, Header0),
-            Header0;
+            Header1 = set_initial_props(Fd, Header0, Options),
+            ok = couch_file:write_header(Fd, Header1),
+            Header1;
         false ->
             case couch_file:read_header(Fd) of
                 {ok, Header0} ->
@@ -266,6 +272,31 @@ get_security(#st{header = Header} = St) ->
     end.
 
 
+get_props(#st{header = Header} = St) ->
+    case couch_bt_engine_header:get(Header, props_ptr) of
+        undefined ->
+            [];
+        Pointer ->
+            {ok, Props} = couch_file:pread_term(St#st.fd, Pointer),
+            Props
+    end.
+
+
+get_prop(St, Key) ->
+    Props = get_props(St),
+    case lists:keyfind(Key, 1, Props) of
+        false -> {error, no_value};
+        {Key, Value} -> {ok, Value}
+    end.
+
+
+get_prop(St, Key, DefaultValue) ->
+    case get_prop(St, Key) of
+        {error, no_value} -> DefaultValue;
+        Value -> Value
+    end.
+
+
 get_update_seq(#st{header = Header}) ->
     couch_bt_engine_header:get(Header, update_seq).
 
@@ -296,6 +327,20 @@ set_security(#st{header = Header} = St, NewSecurity) ->
     {ok, increment_update_seq(NewSt)}.
 
 
+set_prop(#st{header = Header} = St, Key, Value) ->
+    OldProps = get_props(St),
+    NewProps = lists:ukeymerge(1, [{Key, Value}], OldProps),
+    Options = [{compression, St#st.compression}],
+    {ok, Ptr, _} = couch_file:append_term(St#st.fd, NewProps, Options),
+    NewSt = St#st{
+        header = couch_bt_engine_header:set(Header, [
+            {props_ptr, Ptr}
+        ]),
+        needs_commit = true
+    },
+    {ok, increment_update_seq(NewSt)}.
+
+
 open_docs(#st{} = St, DocIds) ->
     Results = couch_btree:lookup(St#st.id_tree, DocIds),
     lists:map(fun
@@ -646,6 +691,16 @@ copy_security(#st{header = Header} = St, SecProps) ->
         needs_commit = true
     }}.
 
+copy_props(#st{header = Header} = St, Props) ->
+    Options = [{compression, St#st.compression}],
+    {ok, Ptr, _} = couch_file:append_term(St#st.fd, Props, Options),
+    {ok, St#st{
+        header = couch_bt_engine_header:set(Header, [
+            {props_ptr, Ptr}
+        ]),
+        needs_commit = true
+    }}.
+
 
 open_db_file(FilePath, Options) ->
     case couch_file:open(FilePath, Options) of
@@ -763,6 +818,18 @@ set_default_security_object(Fd, Header, Compression, Options) ->
     end.
 
 
+set_initial_props(Fd, Header, Options) ->
+    case couch_util:get_value(initial_props, Options) of
+        undefined ->
+            Header;
+        InitialProps ->
+            Compression = couch_compress:get_compression_method(),
+            AppendOpts = [{compression, Compression}],
+            {ok, Ptr, _} = couch_file:append_term(Fd, InitialProps, AppendOpts),
+            couch_bt_engine_header:set(Header, props_ptr, Ptr)
+    end.
+
+
 delete_compaction_files(FilePath) ->
     RootDir = config:get("couchdb", "database_dir", "."),
     DelOpts = [{context, compaction}],
diff --git a/src/couch/src/couch_bt_engine_compactor.erl b/src/couch/src/couch_bt_engine_compactor.erl
index 4e52064b4..83c9bbc84 100644
--- a/src/couch/src/couch_bt_engine_compactor.erl
+++ b/src/couch/src/couch_bt_engine_compactor.erl
@@ -186,9 +186,13 @@ copy_compact(DbName, St, NewSt0, Retry) ->
     SecProps = couch_bt_engine:get_security(St),
     {ok, NewSt4} = couch_bt_engine:copy_security(NewSt3, SecProps),
 
+    % Copy general properties over
+    Props = couch_bt_engine:get_props(St),
+    {ok, NewSt5} = couch_bt_engine:copy_props(NewSt4, Props),
+
     FinalUpdateSeq = couch_bt_engine:get_update_seq(St),
-    {ok, NewSt5} = couch_bt_engine:set_update_seq(NewSt4, FinalUpdateSeq),
-    commit_compaction_data(NewSt5).
+    {ok, NewSt6} = couch_bt_engine:set_update_seq(NewSt5, FinalUpdateSeq),
+    commit_compaction_data(NewSt6).
 
 
 copy_docs(St, #st{} = NewSt, MixedInfos, Retry) ->
diff --git a/src/couch/src/couch_bt_engine_header.erl b/src/couch/src/couch_bt_engine_header.erl
index 3d24f3189..779bf26cf 100644
--- a/src/couch/src/couch_bt_engine_header.erl
+++ b/src/couch/src/couch_bt_engine_header.erl
@@ -66,7 +66,8 @@
     revs_limit = 1000,
     uuid,
     epochs,
-    compacted_seq
+    compacted_seq,
+    props_ptr
 }).
 
 
diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl
index 65ca54a59..9bb68f878 100644
--- a/src/couch/src/couch_db.erl
+++ b/src/couch/src/couch_db.erl
@@ -47,6 +47,7 @@
     get_pid/1,
     get_revs_limit/1,
     get_security/1,
+    get_props/1,
     get_update_seq/1,
     get_user_ctx/1,
     get_uuid/1,
@@ -58,6 +59,7 @@
 
     set_revs_limit/2,
     set_security/2,
+    set_prop/3,
     set_user_ctx/2,
 
     ensure_full_commit/1,
@@ -578,6 +580,15 @@ set_security(#db{main_pid=Pid}=Db, {NewSecProps}) when is_list(NewSecProps) ->
 set_security(_, _) ->
     throw(bad_request).
 
+get_props(#db{props=Props}) ->
+    Props.
+
+set_prop(#db{main_pid=Pid}=Db, Key, Value) ->
+    check_is_admin(Db),
+    ok = gen_server:call(Pid, {set_prop, Key, Value}, infinity),
+    {ok, _} = ensure_full_commit(Db),
+    ok.
+
 set_user_ctx(#db{} = Db, UserCtx) ->
     {ok, Db#db{user_ctx = UserCtx}}.
 
diff --git a/src/couch/src/couch_db_engine.erl b/src/couch/src/couch_db_engine.erl
index 502faa7ee..9b288127a 100644
--- a/src/couch/src/couch_db_engine.erl
+++ b/src/couch/src/couch_db_engine.erl
@@ -224,6 +224,19 @@
 % the last value that was passed to set_security/2.
 -callback get_security(DbHandle::db_handle()) -> SecProps::any().
 
+% Get the current properties.
+-callback get_props(DbHandle::db_handle()) -> Props::any().
+
+% Get the current properties. This should just return
+% the last value that was passed to set_prop/2.
+-callback get_prop(DbHandle::db_handle(), Prop::atom()) ->
+    {ok, SecProps::json()} | {error, no_value}.
+
+% Get the current properties. If the value isn't set it will return the set default value.
+% This should just return the last value that was passed to set_prop/2.
+-callback get_prop(DbHandle::db_handle(), Prop::atom(), DefaultValue::any()) ->
+    {ok, SecProps::json()}.
+
 
 % This information is displayed in the database info poperties. It
 % should just be a list of {Name::atom(), Size::non_neg_integer()}
@@ -265,6 +278,15 @@
         {ok, NewDbHandle::db_handle()}.
 
 
+% This function is only called by couch_db_updater and
+% as such is guaranteed to be single threaded calls. The
+% database should simply store prop key and value somewhere so
+% they can be returned by the corresponding get_prop calls.
+
+-callback set_prop(DbHandle::db_handle(), PropKey::atom(), PropValue::any()) ->
+        {ok, NewDbHandle::db_handle()}.
+
+
 % This function will be called by many processes concurrently.
 % It should return a #full_doc_info{} record or not_found for
 % every provided DocId in the order those DocId's appear in
@@ -601,12 +623,16 @@
     get_purge_seq/1,
     get_revs_limit/1,
     get_security/1,
+    get_props/1,
+    get_prop/2,
+    get_prop/3,
     get_size_info/1,
     get_update_seq/1,
     get_uuid/1,
 
     set_revs_limit/2,
     set_security/2,
+    set_prop/3,
 
     open_docs/2,
     open_local_docs/2,
@@ -757,6 +783,21 @@ get_security(#db{} = Db) ->
     Engine:get_security(EngineState).
 
 
+get_props(#db{} = Db) ->
+    #db{engine = {Engine, EngineState}} = Db,
+    Engine:get_props(EngineState).
+
+
+get_prop(#db{} = Db, Prop) ->
+    #db{engine = {Engine, EngineState}} = Db,
+    Engine:get_prop(EngineState, Prop).
+
+
+get_prop(#db{} = Db, Prop, DefaultValue) ->
+    #db{engine = {Engine, EngineState}} = Db,
+    Engine:get_prop(EngineState, Prop, DefaultValue).
+
+
 get_size_info(#db{} = Db) ->
     #db{engine = {Engine, EngineState}} = Db,
     Engine:get_size_info(EngineState).
@@ -783,6 +824,12 @@ set_security(#db{} = Db, SecProps) ->
     {ok, Db#db{engine = {Engine, NewSt}}}.
 
 
+set_prop(#db{} = Db, Key, Value) ->
+    #db{engine = {Engine, EngineState}} = Db,
+    {ok, NewSt} = Engine:set_prop(EngineState, Key, Value),
+    {ok, Db#db{engine = {Engine, NewSt}}}.
+
+
 open_docs(#db{} = Db, DocIds) ->
     #db{engine = {Engine, EngineState}} = Db,
     Engine:open_docs(EngineState, DocIds).
diff --git a/src/couch/src/couch_db_int.hrl b/src/couch/src/couch_db_int.hrl
index a412b338b..f2a8e17f5 100644
--- a/src/couch/src/couch_db_int.hrl
+++ b/src/couch/src/couch_db_int.hrl
@@ -35,7 +35,8 @@
     waiting_delayed_commit = nil,
 
     options = [],
-    compression
+    compression,
+    props = []
 }).
 
 
diff --git a/src/couch/src/couch_db_updater.erl b/src/couch/src/couch_db_updater.erl
index acb9ec1c9..d8a769b7f 100644
--- a/src/couch/src/couch_db_updater.erl
+++ b/src/couch/src/couch_db_updater.erl
@@ -88,6 +88,14 @@ handle_call({set_security, NewSec}, _From, #db{} = Db) ->
     ok = gen_server:call(couch_server, {db_updated, NewSecDb}, infinity),
     {reply, ok, NewSecDb, idle_limit()};
 
+handle_call({set_prop, Key, Value}, _From, #db{} = Db) ->
+    {ok, NewDb} = couch_db_engine:set_prop(Db, Key, Value),
+    NewPropDb = commit_data(NewDb#db{
+        props = couch_db_engine:get_props(NewDb)
+    }),
+    ok = gen_server:call(couch_server, {db_updated, NewPropDb}, infinity),
+    {reply, ok, NewPropDb, idle_limit()};
+
 handle_call({set_revs_limit, Limit}, _From, Db) ->
     {ok, Db2} = couch_db_engine:set_revs_limit(Db, Limit),
     Db3 = commit_data(Db2),
@@ -375,7 +383,8 @@ init_db(DbName, FilePath, EngineState, Options) ->
 
     InitDb#db{
         committed_update_seq = couch_db_engine:get_update_seq(InitDb),
-        security = couch_db_engine:get_security(InitDb)
+        security = couch_db_engine:get_security(InitDb),
+        props = couch_db_engine:get_props(InitDb)
     }.
 
 
diff --git a/src/couch/src/test_engine_get_set_props.erl b/src/couch/src/test_engine_get_set_props.erl
index 6d2a44779..b0fbff450 100644
--- a/src/couch/src/test_engine_get_set_props.erl
+++ b/src/couch/src/test_engine_get_set_props.erl
@@ -50,6 +50,28 @@ cet_set_revs_limit() ->
     check_prop_set(get_revs_limit, set_revs_limit, 1000, 50).
 
 
+cet_set_prop() ->
+    Engine = test_engine_util:get_engine(),
+    DbPath = test_engine_util:dbpath(),
+
+    {ok, St0} = Engine:init(DbPath, [
+            create,
+            {default_security_object, dso}
+        ]),
+    ?assertEqual({error, no_value}, Engine:get_prop(St0, shardkey)),
+
+    ?assertEqual(false, Engine:get_prop(St0, shardkey, false)),
+
+    {ok, St1} = Engine:set_prop(St0, shardkey, true),
+    ?assertEqual({ok, true}, Engine:get_prop(St1, shardkey)),
+
+    {ok, St2} = Engine:commit_data(St1),
+    Engine:terminate(normal, St2),
+
+    {ok, St3} = Engine:init(DbPath, []),
+    ?assertEqual({ok, true}, Engine:get_prop(St3, shardkey)).
+
+
 check_prop_set(GetFun, SetFun, Default, Value) ->
     Engine = test_engine_util:get_engine(),
     DbPath = test_engine_util:dbpath(),
diff --git a/src/fabric/src/fabric_db_create.erl b/src/fabric/src/fabric_db_create.erl
index 94ffd5643..90ccdab86 100644
--- a/src/fabric/src/fabric_db_create.erl
+++ b/src/fabric/src/fabric_db_create.erl
@@ -164,6 +164,10 @@ make_document([#shard{dbname=DbName}|_] = Shards, Suffix, Options) ->
         {[[<<"add">>, Range, Node] | Raw], orddict:append(Node, Range, ByNode),
             orddict:append(Range, Node, ByRange)}
     end, {[], [], []}, Shards),
+    InitialProps = case couch_util:get_value(initial_props, Options) of
+        I when is_list(I) -> [{<<"options">>, {I}}];
+        _ -> []
+    end,
     EngineProp = case couch_util:get_value(engine, Options) of
         E when is_binary(E) -> [{<<"engine">>, E}];
         _ -> []
@@ -175,7 +179,7 @@ make_document([#shard{dbname=DbName}|_] = Shards, Suffix, Options) ->
             {<<"changelog">>, lists:sort(RawOut)},
             {<<"by_node">>, {[{K,lists:sort(V)} || {K,V} <- ByNodeOut]}},
             {<<"by_range">>, {[{K,lists:sort(V)} || {K,V} <- ByRangeOut]}}
-        ] ++ EngineProp}
+        ] ++ EngineProp ++ InitialProps}
     }.
 
 db_exists(DbName) -> is_list(catch mem3:shards(DbName)).
diff --git a/src/mem3/src/mem3_util.erl b/src/mem3/src/mem3_util.erl
index 0b69d790d..e08d37532 100644
--- a/src/mem3/src/mem3_util.erl
+++ b/src/mem3/src/mem3_util.erl
@@ -162,7 +162,7 @@ build_shards_by_node(DbName, DocProps) ->
                 dbname = DbName,
                 node = to_atom(Node),
                 range = [Beg, End],
-                opts = get_engine_opt(DocProps)
+                opts = get_engine_opt(DocProps) ++ get_opts(DocProps)
             }, Suffix)
         end, Ranges)
     end, ByNode).
@@ -180,7 +180,7 @@ build_shards_by_range(DbName, DocProps) ->
                 node = to_atom(Node),
                 range = [Beg, End],
                 order = Order,
-                opts = get_engine_opt(DocProps)
+                opts = get_engine_opt(DocProps) ++ get_opts(DocProps)
             }, Suffix)
         end, lists:zip(Nodes, lists:seq(1, length(Nodes))))
     end, ByRange).
@@ -197,6 +197,14 @@ to_integer(N) when is_binary(N) ->
 to_integer(N) when is_list(N) ->
     list_to_integer(N).
 
+get_opts(DocProps) ->
+    case couch_util:get_value(<<"options">>, DocProps) of
+        {Opts} ->
+            Opts;
+        _ ->
+            []
+    end.
+
 get_engine_opt(DocProps) ->
     case couch_util:get_value(<<"engine">>, DocProps) of
         Engine when is_binary(Engine) ->