Add PSE API to store opaque properties

This allows us to implement features outside of the PSE API without
requiring changes to the API for each bit of data we may want to end up
storing. The use of this opaque object should only be used for features
that don't require a beahvior change from the storage engine API.

Co-authored-by: Garren Smith <garren.smith@gmail.com>
Co-authored-by: Robert Newson <rnewson@apache.org>

7 files changed, 180 insertions, 9 deletions

diff --git a/src/couch/src/couch_bt_engine.erl b/src/couch/src/couch_bt_engine.erl
index f856bde8f..946b74d0c 100644
--- a/src/couch/src/couch_bt_engine.erl
+++ b/src/couch/src/couch_bt_engine.erl
@@ -40,6 +40,7 @@
     get_purge_infos_limit/1,
     get_revs_limit/1,
     get_security/1,
+    get_props/1,
     get_size_info/1,
     get_update_seq/1,
     get_uuid/1,
@@ -47,6 +48,7 @@
     set_revs_limit/2,
     set_purge_infos_limit/2,
     set_security/2,
+    set_props/2,
 
     open_docs/2,
     open_local_docs/2,
@@ -104,7 +106,8 @@
 -export([
     set_update_seq/2,
     update_header/2,
-    copy_security/2
+    copy_security/2,
+    copy_props/2
 ]).
 
 
@@ -143,8 +146,9 @@ init(FilePath, Options) ->
         true ->
             delete_compaction_files(FilePath),
             Header0 = couch_bt_engine_header:new(),
-            ok = couch_file:write_header(Fd, Header0),
-            Header0;
+            Header1 = init_set_props(Fd, Header0, Options),
+            ok = couch_file:write_header(Fd, Header1),
+            Header1;
         false ->
             case couch_file:read_header(Fd) of
                 {ok, Header0} ->
@@ -283,6 +287,16 @@ get_security(#st{header = Header} = St) ->
     end.
 
 
+get_props(#st{header = Header} = St) ->
+    case couch_bt_engine_header:get(Header, props_ptr) of
+        undefined ->
+            [];
+        Pointer ->
+            {ok, Props} = couch_file:pread_term(St#st.fd, Pointer),
+            Props
+    end.
+
+
 get_update_seq(#st{header = Header}) ->
     couch_bt_engine_header:get(Header, update_seq).
 
@@ -323,6 +337,18 @@ set_security(#st{header = Header} = St, NewSecurity) ->
     {ok, increment_update_seq(NewSt)}.
 
 
+set_props(#st{header = Header} = St, Props) ->
+    Options = [{compression, St#st.compression}],
+    {ok, Ptr, _} = couch_file:append_term(St#st.fd, Props, Options),
+    NewSt = St#st{
+        header = couch_bt_engine_header:set(Header, [
+            {props_ptr, Ptr}
+        ]),
+        needs_commit = true
+    },
+    {ok, increment_update_seq(NewSt)}.
+
+
 open_docs(#st{} = St, DocIds) ->
     Results = couch_btree:lookup(St#st.id_tree, DocIds),
     lists:map(fun
@@ -753,6 +779,17 @@ copy_security(#st{header = Header} = St, SecProps) ->
     }}.
 
 
+copy_props(#st{header = Header} = St, Props) ->
+    Options = [{compression, St#st.compression}],
+    {ok, Ptr, _} = couch_file:append_term(St#st.fd, Props, Options),
+    {ok, St#st{
+        header = couch_bt_engine_header:set(Header, [
+            {props_ptr, Ptr}
+        ]),
+        needs_commit = true
+    }}.
+
+
 open_db_file(FilePath, Options) ->
     case couch_file:open(FilePath, Options) of
         {ok, Fd} ->
@@ -939,6 +976,18 @@ upgrade_purge_info(Fd, Header) ->
     end.
 
 
+init_set_props(Fd, Header, Options) ->
+    case couch_util:get_value(props, Options) of
+        undefined ->
+            Header;
+        InitialProps ->
+            Compression = couch_compress:get_compression_method(),
+            AppendOpts = [{compression, Compression}],
+            {ok, Ptr, _} = couch_file:append_term(Fd, InitialProps, AppendOpts),
+            couch_bt_engine_header:set(Header, props_ptr, Ptr)
+    end.
+
+
 delete_compaction_files(FilePath) ->
     RootDir = config:get("couchdb", "database_dir", "."),
     DelOpts = [{context, compaction}],
diff --git a/src/couch/src/couch_bt_engine_compactor.erl b/src/couch/src/couch_bt_engine_compactor.erl
index 10de68687..737f77245 100644
--- a/src/couch/src/couch_bt_engine_compactor.erl
+++ b/src/couch/src/couch_bt_engine_compactor.erl
@@ -276,9 +276,13 @@ copy_compact(DbName, St, NewSt0, Retry) ->
     SecProps = couch_bt_engine:get_security(St),
     {ok, NewSt4} = couch_bt_engine:copy_security(NewSt3, SecProps),
 
+    % Copy general properties over
+    Props = couch_bt_engine:get_props(St),
+    {ok, NewSt5} = couch_bt_engine:set_props(NewSt4, Props),
+
     FinalUpdateSeq = couch_bt_engine:get_update_seq(St),
-    {ok, NewSt5} = couch_bt_engine:set_update_seq(NewSt4, FinalUpdateSeq),
-    commit_compaction_data(NewSt5).
+    {ok, NewSt6} = couch_bt_engine:set_update_seq(NewSt5, FinalUpdateSeq),
+    commit_compaction_data(NewSt6).
 
 
 copy_docs(St, #st{} = NewSt, MixedInfos, Retry) ->
diff --git a/src/couch/src/couch_bt_engine_header.erl b/src/couch/src/couch_bt_engine_header.erl
index 619264a0d..2dafb3e47 100644
--- a/src/couch/src/couch_bt_engine_header.erl
+++ b/src/couch/src/couch_bt_engine_header.erl
@@ -69,7 +69,8 @@
     uuid,
     epochs,
     compacted_seq,
-    purge_infos_limit = 1000
+    purge_infos_limit = 1000,
+    props_ptr
 }).
 
 
diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl
index 2f63fcfe8..9bc0f9263 100644
--- a/src/couch/src/couch_db.erl
+++ b/src/couch/src/couch_db.erl
@@ -584,6 +584,10 @@ get_db_info(Db) ->
         undefined -> null;
         Else1 -> Else1
     end,
+    Props = case couch_db_engine:get_props(Db) of
+        undefined -> null;
+        Else2 -> {Else2}
+    end,
     InfoList = [
         {db_name, Name},
         {engine, couch_db_engine:get_engine(Db)},
@@ -605,6 +609,7 @@ get_db_info(Db) ->
         {disk_format_version, DiskVersion},
         {committed_update_seq, CommittedUpdateSeq},
         {compacted_seq, CompactedSeq},
+        {props, Props},
         {uuid, Uuid}
     ],
     {ok, InfoList}.
diff --git a/src/couch/src/couch_db_engine.erl b/src/couch/src/couch_db_engine.erl
index ea30dbc77..806d352cb 100644
--- a/src/couch/src/couch_db_engine.erl
+++ b/src/couch/src/couch_db_engine.erl
@@ -243,6 +243,10 @@
 -callback get_security(DbHandle::db_handle()) -> SecProps::any().
 
 
+% Get the current properties.
+-callback get_props(DbHandle::db_handle()) -> Props::[any()].
+
+
 % This information is displayed in the database info poperties. It
 % should just be a list of {Name::atom(), Size::non_neg_integer()}
 % tuples that will then be combined across shards. Currently,
@@ -288,6 +292,15 @@
         {ok, NewDbHandle::db_handle()}.
 
 
+% This function is only called by couch_db_updater and
+% as such is guaranteed to be single threaded calls. The
+% database should simply store provided property list
+% unaltered.
+
+-callback set_props(DbHandle::db_handle(), Props::any()) ->
+        {ok, NewDbHandle::db_handle()}.
+
+
 % This function will be called by many processes concurrently.
 % It should return a #full_doc_info{} record or not_found for
 % every provided DocId in the order those DocId's appear in
@@ -670,6 +683,7 @@
     get_purge_infos_limit/1,
     get_revs_limit/1,
     get_security/1,
+    get_props/1,
     get_size_info/1,
     get_update_seq/1,
     get_uuid/1,
@@ -677,6 +691,7 @@
     set_revs_limit/2,
     set_security/2,
     set_purge_infos_limit/2,
+    set_props/2,
 
     open_docs/2,
     open_local_docs/2,
@@ -836,6 +851,11 @@ get_security(#db{} = Db) ->
     Engine:get_security(EngineState).
 
 
+get_props(#db{} = Db) ->
+    #db{engine = {Engine, EngineState}} = Db,
+    Engine:get_props(EngineState).
+
+
 get_size_info(#db{} = Db) ->
     #db{engine = {Engine, EngineState}} = Db,
     Engine:get_size_info(EngineState).
@@ -868,6 +888,12 @@ set_security(#db{} = Db, SecProps) ->
     {ok, Db#db{engine = {Engine, NewSt}}}.
 
 
+set_props(#db{} = Db, Props) ->
+    #db{engine = {Engine, EngineState}} = Db,
+    {ok, NewSt} = Engine:set_props(EngineState, Props),
+    {ok, Db#db{engine = {Engine, NewSt}}}.
+
+
 open_docs(#db{} = Db, DocIds) ->
     #db{engine = {Engine, EngineState}} = Db,
     Engine:open_docs(EngineState, DocIds).
diff --git a/src/couch/src/couch_db_updater.erl b/src/couch/src/couch_db_updater.erl
index 87301d2d8..c0974aa94 100644
--- a/src/couch/src/couch_db_updater.erl
+++ b/src/couch/src/couch_db_updater.erl
@@ -310,21 +310,24 @@ init_db(DbName, FilePath, EngineState, Options) ->
     BDU = couch_util:get_value(before_doc_update, Options, nil),
     ADR = couch_util:get_value(after_doc_read, Options, nil),
 
-    CleanedOpts = [Opt || Opt <- Options, Opt /= create],
+    NonCreateOpts = [Opt || Opt <- Options, Opt /= create],
 
     InitDb = #db{
         name = DbName,
         filepath = FilePath,
         engine = EngineState,
         instance_start_time = StartTime,
-        options = CleanedOpts,
+        options = NonCreateOpts,
         before_doc_update = BDU,
         after_doc_read = ADR
     },
 
+    DbProps = couch_db_engine:get_props(InitDb),
+
     InitDb#db{
         committed_update_seq = couch_db_engine:get_update_seq(InitDb),
-        security = couch_db_engine:get_security(InitDb)
+        security = couch_db_engine:get_security(InitDb),
+        options = lists:keystore(props, 1, NonCreateOpts, {props, DbProps})
     }.
 
 
diff --git a/src/couch/test/couch_db_props_upgrade_tests.erl b/src/couch/test/couch_db_props_upgrade_tests.erl
new file mode 100644
index 000000000..40ad283cf
--- /dev/null
+++ b/src/couch/test/couch_db_props_upgrade_tests.erl
@@ -0,0 +1,83 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_db_props_upgrade_tests).
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch_mrview/include/couch_mrview.hrl").
+
+
+setup() ->
+    DbName = <<"test">>,
+    DbFileName = "test.couch",
+    OldDbFilePath = filename:join([?FIXTURESDIR, DbFileName]),
+
+    DbDir = config:get("couchdb", "database_dir"),
+    NewDbFilePath = filename:join([DbDir, DbFileName]),
+
+    file:delete(NewDbFilePath),
+    {ok, _} = file:copy(OldDbFilePath, NewDbFilePath),
+
+    DbName.
+
+
+teardown(DbName) when is_binary(DbName) ->
+    couch_server:delete(DbName, [?ADMIN_CTX]),
+    ok.
+
+
+old_db_info_test_() ->
+    {
+        "Old database versions work",
+        {
+            setup,
+            fun test_util:start_couch/0,
+            fun test_util:stop_couch/1,
+            {
+                foreach,
+                fun setup/0,
+                fun teardown/1,
+                [
+                    fun can_get_props/1,
+                    fun can_get_db_info/1,
+                    fun can_compact_db/1
+                ]
+            }
+        }
+    }.
+
+
+can_get_props(DbName) ->
+    ?_test(begin
+        {ok, Db} = couch_db:open_int(DbName, []),
+        Props = couch_db_engine:get_props(Db),
+        ?assert(is_list(Props))
+      end).
+
+
+can_get_db_info(DbName) ->
+    ?_test(begin
+        {ok, Db} = couch_db:open_int(DbName, []),
+        {ok, Info} = couch_db:get_db_info(Db),
+        Props = couch_util:get_value(props, Info),
+        ?assertEqual({[]}, Props)
+      end).
+
+
+can_compact_db(DbName) ->
+    ?_test(begin
+        couch_util:with_db(DbName, fun(Db) ->
+            couch_db:start_compact(Db),
+            couch_db:wait_for_compaction(Db)
+        end)
+    end).