Fix engine header upgradesfix-pq-db-header-version

11 files changed, 41 insertions, 165 deletions

diff --git a/src/couch/src/couch_bt_engine_header.erl b/src/couch/src/couch_bt_engine_header.erl
index 2dafb3e47..3f9f51821 100644
--- a/src/couch/src/couch_bt_engine_header.erl
+++ b/src/couch/src/couch_bt_engine_header.erl
@@ -53,7 +53,7 @@
 % if the disk revision is incremented, then new upgrade logic will need to be
 % added to couch_db_updater:init_db.
 
--define(LATEST_DISK_VERSION, 7).
+-define(LATEST_DISK_VERSION, 8).
 
 -record(db_header, {
     disk_version = ?LATEST_DISK_VERSION,
@@ -104,7 +104,6 @@ is_header(Header) ->
 
 upgrade(Header) ->
     Funs = [
-        fun downgrade_partition_header/1,
         fun upgrade_tuple/1,
         fun upgrade_disk_version/1,
         fun upgrade_uuid/1,
@@ -220,50 +219,6 @@ indexes() ->
     lists:zip(Fields, Indexes).
 
 
-downgrade_partition_header(Header) ->
-    DiskVersion = disk_version(Header),
-    Latest = latest_disk_version(),
-    case DiskVersion of
-        N when N =< Latest ->
-            Header;
-        ?PARTITION_DISK_VERSION ->
-            {
-                db_header,
-                _DiskVer,
-                UpSeq,
-                _Unused,
-                IdTreeState,
-                SeqTreeState,
-                LocalTreeState,
-                PurgeTreeState,
-                PurgeSeqTreeState,
-                SecurityPtr,
-                RevsLimit,
-                Uuid,
-                Epochs,
-                CompactedSeq,
-                PurgeInfosLimit,
-                _PropsPtr
-            } = Header,
-            
-            NewHeader = new(),
-            set(NewHeader, [
-                {update_seq, UpSeq},
-                {id_tree_state, IdTreeState},
-                {seq_tree_state, SeqTreeState},
-                {local_tree_state, LocalTreeState},
-                {purge_tree_state, PurgeTreeState},
-                {purge_seq_tree_state, PurgeSeqTreeState},
-                {security_ptr, SecurityPtr},
-                {revs_limit, RevsLimit},
-                {uuid, Uuid},
-                {epochs, Epochs},
-                {compacted_seq, CompactedSeq},
-                {purge_infos_limit, PurgeInfosLimit}
-            ])
-    end.
-
-
 upgrade_tuple(Old) when is_record(Old, db_header) ->
     Old;
 upgrade_tuple(Old) when is_tuple(Old) ->
@@ -290,6 +245,7 @@ upgrade_disk_version(#db_header{}=Header) ->
         4 -> Header#db_header{security_ptr = nil}; % [0.10 - 0.11)
         5 -> Header#db_header{disk_version = ?LATEST_DISK_VERSION}; % pre 1.2
         6 -> Header#db_header{disk_version = ?LATEST_DISK_VERSION}; % pre clustered purge
+        7 -> Header#db_header{disk_version = ?LATEST_DISK_VERSION}; % pre partitioned dbs
         ?LATEST_DISK_VERSION -> Header;
         _ ->
             Reason = "Incorrect disk header version",
@@ -422,12 +378,12 @@ upgrade_v3_test() ->
 
 -endif.
 
-upgrade_v5_to_v7_test() ->
+upgrade_v5_to_v8_test() ->
     Vsn5Header = mk_header(5),
     NewHeader = upgrade_disk_version(upgrade_tuple(Vsn5Header)),
 
     ?assert(is_record(NewHeader, db_header)),
-    ?assertEqual(7, disk_version(NewHeader)),
+    ?assertEqual(8, disk_version(NewHeader)),
 
     % Security ptr isn't changed for v5 headers
     ?assertEqual(bang, security_ptr(NewHeader)).
diff --git a/src/couch/test/couch_bt_engine_partition_downgrade_tests.erl b/src/couch/test/couch_bt_engine_partition_downgrade_tests.erl
deleted file mode 100644
index f0839a411..000000000
--- a/src/couch/test/couch_bt_engine_partition_downgrade_tests.erl
+++ /dev/null
@@ -1,86 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-%   http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--module(couch_bt_engine_partition_downgrade_tests).
-
--include_lib("couch/include/couch_eunit.hrl").
--include_lib("couch/include/couch_db.hrl").
-
-
-setup() ->
-    Ctx = test_util:start_couch(),
-    DbDir = config:get("couchdb", "database_dir"),
-    DbFileNames = [
-        "db_non_partitioned.couch"
-    ],
-    NewPaths = lists:map(fun(DbFileName) ->
-        OldDbFilePath = filename:join([?FIXTURESDIR, DbFileName]),
-        NewDbFilePath = filename:join([DbDir, DbFileName]),
-        ok = filelib:ensure_dir(NewDbFilePath),
-        file:delete(NewDbFilePath),
-        {ok, _} = file:copy(OldDbFilePath, NewDbFilePath),
-        NewDbFilePath
-    end, DbFileNames),
-    {Ctx, NewPaths}.
-
-
-teardown({Ctx, Paths}) ->
-    test_util:stop_couch(Ctx),
-    lists:foreach(fun(Path) ->
-        file:delete(Path)
-    end, Paths).
-
-
-downgrade_test_() ->
-    {
-        "Couch Bt Engine partition downgrade tests",
-        {
-            setup,
-            fun setup/0,
-            fun teardown/1,
-            [
-                t_downgrade_non_partitioned_db()
-            ]
-        }
-    }.
-
-
-t_downgrade_non_partitioned_db() ->
-    ?_test(begin
-        % There are 13 documents in the fixture with 1 conflicted
-        DbName = <<"db_non_partitioned">>,
-        ?assertEqual(8, get_disk_version_from_header(DbName)),
-
-        {ok, _} = save_doc(DbName, {[{<<"_id">>, <<"doc4">>}, {<<"v">>, 1}]}),
-        {ok, _} = save_doc(DbName, {[{<<"_id">>, <<"doc5">>}, {<<"v">>, 2}]}),
-
-        couch_util:with_db(DbName, fun(Db) ->
-            ?assertEqual(7, couch_db_engine:get_disk_version(Db))
-        end)
-    end).
-
-
-get_disk_version_from_header(DbFileName) ->
-    DbDir = config:get("couchdb", "database_dir"),
-    DbFilePath = filename:join([DbDir, ?l2b(?b2l(DbFileName) ++ ".couch")]),
-    {ok, Fd} = couch_file:open(DbFilePath, []),
-    {ok, Header} = couch_file:read_header(Fd),
-    DiskVerison = couch_bt_engine_header:disk_version(Header),
-    couch_file:close(Fd),
-    DiskVerison.
-
-
-save_doc(DbName, Json) ->
-    Doc = couch_doc:from_json_obj(Json),
-    couch_util:with_db(DbName, fun(Db) ->
-        couch_db:update_doc(Db, Doc, [])
-    end).
diff --git a/src/couch/test/couch_bt_engine_upgrade_tests.erl b/src/couch/test/couch_bt_engine_upgrade_tests.erl
index 8025528fb..3a516f8f7 100644
--- a/src/couch/test/couch_bt_engine_upgrade_tests.erl
+++ b/src/couch/test/couch_bt_engine_upgrade_tests.erl
@@ -16,14 +16,18 @@
 -include_lib("couch/include/couch_db.hrl").
 
 
-setup() ->
+setup(_) ->
     Ctx = test_util:start_couch(),
     DbDir = config:get("couchdb", "database_dir"),
     DbFileNames = [
-        "db_without_purge_req.couch",
-        "db_with_1_purge_req.couch",
-        "db_with_2_purge_req.couch",
-        "db_with_1_purge_req_for_2_docs.couch"
+        "db_v6_without_purge_req.couch",
+        "db_v6_with_1_purge_req.couch",
+        "db_v6_with_2_purge_req.couch",
+        "db_v6_with_1_purge_req_for_2_docs.couch",
+        "db_v7_without_purge_req.couch",
+        "db_v7_with_1_purge_req.couch",
+        "db_v7_with_2_purge_req.couch",
+        "db_v7_with_1_purge_req_for_2_docs.couch"
     ],
     NewPaths = lists:map(fun(DbFileName) ->
         OldDbFilePath = filename:join([?FIXTURESDIR, DbFileName]),
@@ -36,7 +40,7 @@ setup() ->
     {Ctx, NewPaths}.
 
 
-teardown({Ctx, Paths}) ->
+teardown(_, {Ctx, Paths}) ->
     test_util:stop_couch(Ctx),
     lists:foreach(fun(Path) ->
         file:delete(Path)
@@ -44,35 +48,34 @@ teardown({Ctx, Paths}) ->
 
 
 upgrade_test_() ->
+    From = [6, 7],
     {
         "Couch Bt Engine Upgrade tests",
         {
-            setup,
-            fun setup/0,
-            fun teardown/1,
-            [
-                t_upgrade_without_purge_req(),
-                t_upgrade_with_1_purge_req(),
-                t_upgrade_with_N_purge_req(),
-                t_upgrade_with_1_purge_req_for_2_docs()
-            ]
+            foreachx,
+            fun setup/1, fun teardown/2,
+            [{F, fun t_upgrade_without_purge_req/2} || F <- From] ++
+            [{F, fun t_upgrade_with_1_purge_req/2} || F <- From] ++
+            [{F, fun t_upgrade_with_N_purge_req/2} || F <- From] ++
+            [{F, fun t_upgrade_with_1_purge_req_for_2_docs/2} || F <- From]
         }
     }.
 
 
-t_upgrade_without_purge_req() ->
+t_upgrade_without_purge_req(VersionFrom, {_Ctx, _NewPaths}) ->
     ?_test(begin
         % There are three documents in the fixture
         % db with zero purge entries
-        DbName = <<"db_without_purge_req">>,
+        DbName = ?l2b("db_v"  ++ integer_to_list(VersionFrom)
+            ++ "_without_purge_req"),
 
-        ?assertEqual(6, get_disk_version_from_header(DbName)),
+        ?assertEqual(VersionFrom, get_disk_version_from_header(DbName)),
         {ok, UpgradedPurged} = couch_util:with_db(DbName, fun(Db) ->
             ?assertEqual(0, couch_db:get_purge_seq(Db)),
             couch_db:fold_purge_infos(Db, 0, fun fold_fun/2, [])
         end),
         ?assertEqual([], UpgradedPurged),
-        ?assertEqual(7, get_disk_version_from_header(DbName)),
+        ?assertEqual(8, get_disk_version_from_header(DbName)),
         {ok, Rev} = save_doc(
             DbName, {[{<<"_id">>, <<"doc4">>}, {<<"v">>, 1}]}
         ),
@@ -99,18 +102,19 @@ t_upgrade_without_purge_req() ->
     end).
 
 
-t_upgrade_with_1_purge_req() ->
+t_upgrade_with_1_purge_req(VersionFrom, {_Ctx, _NewPaths}) ->
     ?_test(begin
         % There are two documents in the fixture database
         % with a single purge entry
-        DbName = <<"db_with_1_purge_req">>,
+        DbName = ?l2b("db_v"  ++ integer_to_list(VersionFrom)
+            ++ "_with_1_purge_req"),
 
-        ?assertEqual(6, get_disk_version_from_header(DbName)),
+        ?assertEqual(VersionFrom, get_disk_version_from_header(DbName)),
         {ok, UpgradedPurged} = couch_util:with_db(DbName, fun(Db) ->
             ?assertEqual(1, couch_db:get_purge_seq(Db)),
             couch_db:fold_purge_infos(Db, 0, fun fold_fun/2, [])
         end),
-        ?assertEqual(7, get_disk_version_from_header(DbName)),
+        ?assertEqual(8, get_disk_version_from_header(DbName)),
         ?assertEqual([{1, <<"doc1">>}], UpgradedPurged),
 
         {ok, Rev} = save_doc(
@@ -139,18 +143,19 @@ t_upgrade_with_1_purge_req() ->
     end).
 
 
-t_upgrade_with_N_purge_req() ->
+t_upgrade_with_N_purge_req(VersionFrom, {_Ctx, _NewPaths}) ->
     ?_test(begin
         % There is one document in the fixture database
         % with two docs that have been purged
-        DbName = <<"db_with_2_purge_req">>,
+        DbName = ?l2b("db_v"  ++ integer_to_list(VersionFrom)
+            ++ "_with_2_purge_req"),
 
-        ?assertEqual(6, get_disk_version_from_header(DbName)),
+        ?assertEqual(VersionFrom, get_disk_version_from_header(DbName)),
         {ok, UpgradedPurged} = couch_util:with_db(DbName, fun(Db) ->
             ?assertEqual(2, couch_db:get_purge_seq(Db)),
             couch_db:fold_purge_infos(Db, 1, fun fold_fun/2, [])
         end),
-        ?assertEqual(7, get_disk_version_from_header(DbName)),
+        ?assertEqual(8, get_disk_version_from_header(DbName)),
         ?assertEqual([{2, <<"doc2">>}], UpgradedPurged),
 
         {ok, Rev} = save_doc(DbName, {[{<<"_id">>, <<"doc4">>}, {<<"v">>, 1}]}),
@@ -177,19 +182,20 @@ t_upgrade_with_N_purge_req() ->
     end).
 
 
-t_upgrade_with_1_purge_req_for_2_docs() ->
+t_upgrade_with_1_purge_req_for_2_docs(VersionFrom, {_Ctx, _NewPaths}) ->
     ?_test(begin
         % There are two documents (Doc4 and Doc5) in the fixture database
         % with three docs (Doc1, Doc2 and Doc3) that have been purged, and
         % with one purge req for Doc1 and another purge req for Doc 2 and Doc3
-        DbName = <<"db_with_1_purge_req_for_2_docs">>,
+        DbName = ?l2b("db_v"  ++ integer_to_list(VersionFrom)
+            ++ "_with_1_purge_req_for_2_docs"),
 
-        ?assertEqual(6, get_disk_version_from_header(DbName)),
+        ?assertEqual(VersionFrom, get_disk_version_from_header(DbName)),
         {ok, UpgradedPurged} = couch_util:with_db(DbName, fun(Db) ->
             ?assertEqual(3, couch_db:get_purge_seq(Db)),
             couch_db:fold_purge_infos(Db, 1, fun fold_fun/2, [])
         end),
-        ?assertEqual(7, get_disk_version_from_header(DbName)),
+        ?assertEqual(8, get_disk_version_from_header(DbName)),
         ?assertEqual([{3,<<"doc2">>},{2,<<"doc3">>}], UpgradedPurged),
 
         {ok, Rev} = save_doc(DbName, {[{<<"_id">>, <<"doc6">>}, {<<"v">>, 1}]}),
diff --git a/src/couch/test/fixtures/db_with_1_purge_req.couch b/src/couch/test/fixtures/db_v6_with_1_purge_req.couch
index b0d39c9ec..b0d39c9ec 100644
--- a/src/couch/test/fixtures/db_with_1_purge_req.couch
+++ b/src/couch/test/fixtures/db_v6_with_1_purge_req.couch
diff --git a/src/couch/test/fixtures/db_with_1_purge_req_for_2_docs.couch b/src/couch/test/fixtures/db_v6_with_1_purge_req_for_2_docs.couch
index b584fce31..b584fce31 100644
--- a/src/couch/test/fixtures/db_with_1_purge_req_for_2_docs.couch
+++ b/src/couch/test/fixtures/db_v6_with_1_purge_req_for_2_docs.couch
diff --git a/src/couch/test/fixtures/db_with_2_purge_req.couch b/src/couch/test/fixtures/db_v6_with_2_purge_req.couch
index ee4e11b7f..ee4e11b7f 100644
--- a/src/couch/test/fixtures/db_with_2_purge_req.couch
+++ b/src/couch/test/fixtures/db_v6_with_2_purge_req.couch
diff --git a/src/couch/test/fixtures/db_without_purge_req.couch b/src/couch/test/fixtures/db_v6_without_purge_req.couch
index 814feb8e1..814feb8e1 100644
--- a/src/couch/test/fixtures/db_without_purge_req.couch
+++ b/src/couch/test/fixtures/db_v6_without_purge_req.couch
diff --git a/src/couch/test/fixtures/db_v7_with_1_purge_req.couch b/src/couch/test/fixtures/db_v7_with_1_purge_req.couch
new file mode 100644
index 000000000..cab8331db
--- /dev/null
+++ b/src/couch/test/fixtures/db_v7_with_1_purge_req.couch
diff --git a/src/couch/test/fixtures/db_v7_with_1_purge_req_for_2_docs.couch b/src/couch/test/fixtures/db_v7_with_1_purge_req_for_2_docs.couch
new file mode 100644
index 000000000..b613646b1
--- /dev/null
+++ b/src/couch/test/fixtures/db_v7_with_1_purge_req_for_2_docs.couch
diff --git a/src/couch/test/fixtures/db_v7_with_2_purge_req.couch b/src/couch/test/fixtures/db_v7_with_2_purge_req.couch
new file mode 100644
index 000000000..126fc919e
--- /dev/null
+++ b/src/couch/test/fixtures/db_v7_with_2_purge_req.couch
diff --git a/src/couch/test/fixtures/db_v7_without_purge_req.couch b/src/couch/test/fixtures/db_v7_without_purge_req.couch
new file mode 100644
index 000000000..762dc8dad
--- /dev/null
+++ b/src/couch/test/fixtures/db_v7_without_purge_req.couch