Add PSE API to store opaque properties

This allows us to implement features outside of the PSE API without
requiring changes to the API for each bit of data we may want to end up
storing. The use of this opaque object should only be used for features
that don't require a beahvior change from the storage engine API.

Co-authored-by: Garren Smith <garren.smith@gmail.com>
Co-authored-by: Robert Newson <rnewson@apache.org>

6 files changed, 97 insertions, 9 deletions

diff --git a/src/couch/src/couch_bt_engine.erl b/src/couch/src/couch_bt_engine.erl
index 6d858ed49..f52f447b9 100644
--- a/src/couch/src/couch_bt_engine.erl
+++ b/src/couch/src/couch_bt_engine.erl
@@ -40,6 +40,7 @@
     get_purge_infos_limit/1,
     get_revs_limit/1,
     get_security/1,
+    get_props/1,
     get_size_info/1,
     get_update_seq/1,
     get_uuid/1,
@@ -47,6 +48,7 @@
     set_revs_limit/2,
     set_purge_infos_limit/2,
     set_security/2,
+    set_props/2,
 
     open_docs/2,
     open_local_docs/2,
@@ -104,7 +106,8 @@
 -export([
     set_update_seq/2,
     update_header/2,
-    copy_security/2
+    copy_security/2,
+    copy_props/2
 ]).
 
 
@@ -143,8 +146,9 @@ init(FilePath, Options) ->
         true ->
             delete_compaction_files(FilePath),
             Header0 = couch_bt_engine_header:new(),
-            ok = couch_file:write_header(Fd, Header0),
-            Header0;
+            Header1 = init_set_props(Fd, Header0, Options),
+            ok = couch_file:write_header(Fd, Header1),
+            Header1;
         false ->
             case couch_file:read_header(Fd) of
                 {ok, Header0} ->
@@ -283,6 +287,16 @@ get_security(#st{header = Header} = St) ->
     end.
 
 
+get_props(#st{header = Header} = St) ->
+    case couch_bt_engine_header:get(Header, props_ptr) of
+        undefined ->
+            [];
+        Pointer ->
+            {ok, Props} = couch_file:pread_term(St#st.fd, Pointer),
+            Props
+    end.
+
+
 get_update_seq(#st{header = Header}) ->
     couch_bt_engine_header:get(Header, update_seq).
 
@@ -323,6 +337,18 @@ set_security(#st{header = Header} = St, NewSecurity) ->
     {ok, increment_update_seq(NewSt)}.
 
 
+set_props(#st{header = Header} = St, Props) ->
+    Options = [{compression, St#st.compression}],
+    {ok, Ptr, _} = couch_file:append_term(St#st.fd, Props, Options),
+    NewSt = St#st{
+        header = couch_bt_engine_header:set(Header, [
+            {props_ptr, Ptr}
+        ]),
+        needs_commit = true
+    },
+    {ok, increment_update_seq(NewSt)}.
+
+
 open_docs(#st{} = St, DocIds) ->
     Results = couch_btree:lookup(St#st.id_tree, DocIds),
     lists:map(fun
@@ -753,6 +779,17 @@ copy_security(#st{header = Header} = St, SecProps) ->
     }}.
 
 
+copy_props(#st{header = Header} = St, Props) ->
+    Options = [{compression, St#st.compression}],
+    {ok, Ptr, _} = couch_file:append_term(St#st.fd, Props, Options),
+    {ok, St#st{
+        header = couch_bt_engine_header:set(Header, [
+            {props_ptr, Ptr}
+        ]),
+        needs_commit = true
+    }}.
+
+
 open_db_file(FilePath, Options) ->
     case couch_file:open(FilePath, Options) of
         {ok, Fd} ->
@@ -939,6 +976,18 @@ upgrade_purge_info(Fd, Header) ->
     end.
 
 
+init_set_props(Fd, Header, Options) ->
+    case couch_util:get_value(props, Options) of
+        undefined ->
+            Header;
+        InitialProps ->
+            Compression = couch_compress:get_compression_method(),
+            AppendOpts = [{compression, Compression}],
+            {ok, Ptr, _} = couch_file:append_term(Fd, InitialProps, AppendOpts),
+            couch_bt_engine_header:set(Header, props_ptr, Ptr)
+    end.
+
+
 delete_compaction_files(FilePath) ->
     RootDir = config:get("couchdb", "database_dir", "."),
     DelOpts = [{context, compaction}],
diff --git a/src/couch/src/couch_bt_engine_compactor.erl b/src/couch/src/couch_bt_engine_compactor.erl
index 10de68687..737f77245 100644
--- a/src/couch/src/couch_bt_engine_compactor.erl
+++ b/src/couch/src/couch_bt_engine_compactor.erl
@@ -276,9 +276,13 @@ copy_compact(DbName, St, NewSt0, Retry) ->
     SecProps = couch_bt_engine:get_security(St),
     {ok, NewSt4} = couch_bt_engine:copy_security(NewSt3, SecProps),
 
+    % Copy general properties over
+    Props = couch_bt_engine:get_props(St),
+    {ok, NewSt5} = couch_bt_engine:set_props(NewSt4, Props),
+
     FinalUpdateSeq = couch_bt_engine:get_update_seq(St),
-    {ok, NewSt5} = couch_bt_engine:set_update_seq(NewSt4, FinalUpdateSeq),
-    commit_compaction_data(NewSt5).
+    {ok, NewSt6} = couch_bt_engine:set_update_seq(NewSt5, FinalUpdateSeq),
+    commit_compaction_data(NewSt6).
 
 
 copy_docs(St, #st{} = NewSt, MixedInfos, Retry) ->
diff --git a/src/couch/src/couch_bt_engine_header.erl b/src/couch/src/couch_bt_engine_header.erl
index 467bb2ff8..7f701fca2 100644
--- a/src/couch/src/couch_bt_engine_header.erl
+++ b/src/couch/src/couch_bt_engine_header.erl
@@ -68,7 +68,8 @@
     uuid,
     epochs,
     compacted_seq,
-    purge_infos_limit = 1000
+    purge_infos_limit = 1000,
+    props_ptr
 }).
 
 
diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl
index 55664e964..397193ed5 100644
--- a/src/couch/src/couch_db.erl
+++ b/src/couch/src/couch_db.erl
@@ -579,6 +579,10 @@ get_db_info(Db) ->
         undefined -> null;
         Else1 -> Else1
     end,
+    Props = case couch_db_engine:get_props(Db) of
+        undefined -> null;
+        Else2 -> Else2
+    end,
     InfoList = [
         {db_name, Name},
         {engine, couch_db_engine:get_engine(Db)},
@@ -600,6 +604,7 @@ get_db_info(Db) ->
         {disk_format_version, DiskVersion},
         {committed_update_seq, CommittedUpdateSeq},
         {compacted_seq, CompactedSeq},
+        {props, {Props}},
         {uuid, Uuid}
     ],
     {ok, InfoList}.
diff --git a/src/couch/src/couch_db_engine.erl b/src/couch/src/couch_db_engine.erl
index ea30dbc77..eed46e807 100644
--- a/src/couch/src/couch_db_engine.erl
+++ b/src/couch/src/couch_db_engine.erl
@@ -243,6 +243,10 @@
 -callback get_security(DbHandle::db_handle()) -> SecProps::any().
 
 
+% Get the current properties.
+-callback get_props(DbHandle::db_handle()) -> Props::any().
+
+
 % This information is displayed in the database info poperties. It
 % should just be a list of {Name::atom(), Size::non_neg_integer()}
 % tuples that will then be combined across shards. Currently,
@@ -288,6 +292,15 @@
         {ok, NewDbHandle::db_handle()}.
 
 
+% This function is only called by couch_db_updater and
+% as such is guaranteed to be single threaded calls. The
+% database should simply store provided property list
+% unaltered.
+
+-callback set_props(DbHandle::db_handle(), Props::any()) ->
+        {ok, NewDbHandle::db_handle()}.
+
+
 % This function will be called by many processes concurrently.
 % It should return a #full_doc_info{} record or not_found for
 % every provided DocId in the order those DocId's appear in
@@ -670,6 +683,7 @@
     get_purge_infos_limit/1,
     get_revs_limit/1,
     get_security/1,
+    get_props/1,
     get_size_info/1,
     get_update_seq/1,
     get_uuid/1,
@@ -677,6 +691,7 @@
     set_revs_limit/2,
     set_security/2,
     set_purge_infos_limit/2,
+    set_props/2,
 
     open_docs/2,
     open_local_docs/2,
@@ -836,6 +851,11 @@ get_security(#db{} = Db) ->
     Engine:get_security(EngineState).
 
 
+get_props(#db{} = Db) ->
+    #db{engine = {Engine, EngineState}} = Db,
+    Engine:get_props(EngineState).
+
+
 get_size_info(#db{} = Db) ->
     #db{engine = {Engine, EngineState}} = Db,
     Engine:get_size_info(EngineState).
@@ -868,6 +888,12 @@ set_security(#db{} = Db, SecProps) ->
     {ok, Db#db{engine = {Engine, NewSt}}}.
 
 
+set_props(#db{} = Db, Props) ->
+    #db{engine = {Engine, EngineState}} = Db,
+    {ok, NewSt} = Engine:set_props(EngineState, Props),
+    {ok, Db#db{engine = {Engine, NewSt}}}.
+
+
 open_docs(#db{} = Db, DocIds) ->
     #db{engine = {Engine, EngineState}} = Db,
     Engine:open_docs(EngineState, DocIds).
diff --git a/src/couch/src/couch_db_updater.erl b/src/couch/src/couch_db_updater.erl
index 52a4d2f1b..b0b5e2c52 100644
--- a/src/couch/src/couch_db_updater.erl
+++ b/src/couch/src/couch_db_updater.erl
@@ -310,21 +310,24 @@ init_db(DbName, FilePath, EngineState, Options) ->
     BDU = couch_util:get_value(before_doc_update, Options, nil),
     ADR = couch_util:get_value(after_doc_read, Options, nil),
 
-    CleanedOpts = [Opt || Opt <- Options, Opt /= create],
+    NonCreateOpts = [Opt || Opt <- Options, Opt /= create],
 
     InitDb = #db{
         name = DbName,
         filepath = FilePath,
         engine = EngineState,
         instance_start_time = StartTime,
-        options = CleanedOpts,
+        options = NonCreateOpts,
         before_doc_update = BDU,
         after_doc_read = ADR
     },
 
+    DbProps = couch_db_engine:get_props(InitDb),
+
     InitDb#db{
         committed_update_seq = couch_db_engine:get_update_seq(InitDb),
-        security = couch_db_engine:get_security(InitDb)
+        security = couch_db_engine:get_security(InitDb),
+        options = lists:keystore(props, 1, NonCreateOpts, {props, DbProps})
     }.