diff options
author | Peng Hui Jiang <jiangphcn@apache.org> | 2019-01-30 20:56:22 +0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-01-30 20:56:22 +0800 |
commit | aba66331f74142e84229aae3c569376375494a36 (patch) | |
tree | 5e75bd76adcaaadf5d3248b1e43eb1a8152e0a0c | |
parent | f30fca21a5d5db136c694501526e4b2e01648977 (diff) | |
parent | 6d317e10e9d984216b4ee0e7d8a879493417e11e (diff) | |
download | couchdb-aba66331f74142e84229aae3c569376375494a36.tar.gz |
Merge pull request #1886 from apache/fix-pq-db-header-version
Fix partitioned query database header version
11 files changed, 41 insertions, 165 deletions
diff --git a/src/couch/src/couch_bt_engine_header.erl b/src/couch/src/couch_bt_engine_header.erl index 2dafb3e47..3f9f51821 100644 --- a/src/couch/src/couch_bt_engine_header.erl +++ b/src/couch/src/couch_bt_engine_header.erl @@ -53,7 +53,7 @@ % if the disk revision is incremented, then new upgrade logic will need to be % added to couch_db_updater:init_db. --define(LATEST_DISK_VERSION, 7). +-define(LATEST_DISK_VERSION, 8). -record(db_header, { disk_version = ?LATEST_DISK_VERSION, @@ -104,7 +104,6 @@ is_header(Header) -> upgrade(Header) -> Funs = [ - fun downgrade_partition_header/1, fun upgrade_tuple/1, fun upgrade_disk_version/1, fun upgrade_uuid/1, @@ -220,50 +219,6 @@ indexes() -> lists:zip(Fields, Indexes). -downgrade_partition_header(Header) -> - DiskVersion = disk_version(Header), - Latest = latest_disk_version(), - case DiskVersion of - N when N =< Latest -> - Header; - ?PARTITION_DISK_VERSION -> - { - db_header, - _DiskVer, - UpSeq, - _Unused, - IdTreeState, - SeqTreeState, - LocalTreeState, - PurgeTreeState, - PurgeSeqTreeState, - SecurityPtr, - RevsLimit, - Uuid, - Epochs, - CompactedSeq, - PurgeInfosLimit, - _PropsPtr - } = Header, - - NewHeader = new(), - set(NewHeader, [ - {update_seq, UpSeq}, - {id_tree_state, IdTreeState}, - {seq_tree_state, SeqTreeState}, - {local_tree_state, LocalTreeState}, - {purge_tree_state, PurgeTreeState}, - {purge_seq_tree_state, PurgeSeqTreeState}, - {security_ptr, SecurityPtr}, - {revs_limit, RevsLimit}, - {uuid, Uuid}, - {epochs, Epochs}, - {compacted_seq, CompactedSeq}, - {purge_infos_limit, PurgeInfosLimit} - ]) - end. - - upgrade_tuple(Old) when is_record(Old, db_header) -> Old; upgrade_tuple(Old) when is_tuple(Old) -> @@ -290,6 +245,7 @@ upgrade_disk_version(#db_header{}=Header) -> 4 -> Header#db_header{security_ptr = nil}; % [0.10 - 0.11) 5 -> Header#db_header{disk_version = ?LATEST_DISK_VERSION}; % pre 1.2 6 -> Header#db_header{disk_version = ?LATEST_DISK_VERSION}; % pre clustered purge + 7 -> Header#db_header{disk_version = ?LATEST_DISK_VERSION}; % pre partitioned dbs ?LATEST_DISK_VERSION -> Header; _ -> Reason = "Incorrect disk header version", @@ -422,12 +378,12 @@ upgrade_v3_test() -> -endif. -upgrade_v5_to_v7_test() -> +upgrade_v5_to_v8_test() -> Vsn5Header = mk_header(5), NewHeader = upgrade_disk_version(upgrade_tuple(Vsn5Header)), ?assert(is_record(NewHeader, db_header)), - ?assertEqual(7, disk_version(NewHeader)), + ?assertEqual(8, disk_version(NewHeader)), % Security ptr isn't changed for v5 headers ?assertEqual(bang, security_ptr(NewHeader)). diff --git a/src/couch/test/couch_bt_engine_partition_downgrade_tests.erl b/src/couch/test/couch_bt_engine_partition_downgrade_tests.erl deleted file mode 100644 index f0839a411..000000000 --- a/src/couch/test/couch_bt_engine_partition_downgrade_tests.erl +++ /dev/null @@ -1,86 +0,0 @@ -% Licensed under the Apache License, Version 2.0 (the "License"); you may not -% use this file except in compliance with the License. You may obtain a copy of -% the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -% License for the specific language governing permissions and limitations under -% the License. - --module(couch_bt_engine_partition_downgrade_tests). - --include_lib("couch/include/couch_eunit.hrl"). --include_lib("couch/include/couch_db.hrl"). - - -setup() -> - Ctx = test_util:start_couch(), - DbDir = config:get("couchdb", "database_dir"), - DbFileNames = [ - "db_non_partitioned.couch" - ], - NewPaths = lists:map(fun(DbFileName) -> - OldDbFilePath = filename:join([?FIXTURESDIR, DbFileName]), - NewDbFilePath = filename:join([DbDir, DbFileName]), - ok = filelib:ensure_dir(NewDbFilePath), - file:delete(NewDbFilePath), - {ok, _} = file:copy(OldDbFilePath, NewDbFilePath), - NewDbFilePath - end, DbFileNames), - {Ctx, NewPaths}. - - -teardown({Ctx, Paths}) -> - test_util:stop_couch(Ctx), - lists:foreach(fun(Path) -> - file:delete(Path) - end, Paths). - - -downgrade_test_() -> - { - "Couch Bt Engine partition downgrade tests", - { - setup, - fun setup/0, - fun teardown/1, - [ - t_downgrade_non_partitioned_db() - ] - } - }. - - -t_downgrade_non_partitioned_db() -> - ?_test(begin - % There are 13 documents in the fixture with 1 conflicted - DbName = <<"db_non_partitioned">>, - ?assertEqual(8, get_disk_version_from_header(DbName)), - - {ok, _} = save_doc(DbName, {[{<<"_id">>, <<"doc4">>}, {<<"v">>, 1}]}), - {ok, _} = save_doc(DbName, {[{<<"_id">>, <<"doc5">>}, {<<"v">>, 2}]}), - - couch_util:with_db(DbName, fun(Db) -> - ?assertEqual(7, couch_db_engine:get_disk_version(Db)) - end) - end). - - -get_disk_version_from_header(DbFileName) -> - DbDir = config:get("couchdb", "database_dir"), - DbFilePath = filename:join([DbDir, ?l2b(?b2l(DbFileName) ++ ".couch")]), - {ok, Fd} = couch_file:open(DbFilePath, []), - {ok, Header} = couch_file:read_header(Fd), - DiskVerison = couch_bt_engine_header:disk_version(Header), - couch_file:close(Fd), - DiskVerison. - - -save_doc(DbName, Json) -> - Doc = couch_doc:from_json_obj(Json), - couch_util:with_db(DbName, fun(Db) -> - couch_db:update_doc(Db, Doc, []) - end). diff --git a/src/couch/test/couch_bt_engine_upgrade_tests.erl b/src/couch/test/couch_bt_engine_upgrade_tests.erl index 8025528fb..3a516f8f7 100644 --- a/src/couch/test/couch_bt_engine_upgrade_tests.erl +++ b/src/couch/test/couch_bt_engine_upgrade_tests.erl @@ -16,14 +16,18 @@ -include_lib("couch/include/couch_db.hrl"). -setup() -> +setup(_) -> Ctx = test_util:start_couch(), DbDir = config:get("couchdb", "database_dir"), DbFileNames = [ - "db_without_purge_req.couch", - "db_with_1_purge_req.couch", - "db_with_2_purge_req.couch", - "db_with_1_purge_req_for_2_docs.couch" + "db_v6_without_purge_req.couch", + "db_v6_with_1_purge_req.couch", + "db_v6_with_2_purge_req.couch", + "db_v6_with_1_purge_req_for_2_docs.couch", + "db_v7_without_purge_req.couch", + "db_v7_with_1_purge_req.couch", + "db_v7_with_2_purge_req.couch", + "db_v7_with_1_purge_req_for_2_docs.couch" ], NewPaths = lists:map(fun(DbFileName) -> OldDbFilePath = filename:join([?FIXTURESDIR, DbFileName]), @@ -36,7 +40,7 @@ setup() -> {Ctx, NewPaths}. -teardown({Ctx, Paths}) -> +teardown(_, {Ctx, Paths}) -> test_util:stop_couch(Ctx), lists:foreach(fun(Path) -> file:delete(Path) @@ -44,35 +48,34 @@ teardown({Ctx, Paths}) -> upgrade_test_() -> + From = [6, 7], { "Couch Bt Engine Upgrade tests", { - setup, - fun setup/0, - fun teardown/1, - [ - t_upgrade_without_purge_req(), - t_upgrade_with_1_purge_req(), - t_upgrade_with_N_purge_req(), - t_upgrade_with_1_purge_req_for_2_docs() - ] + foreachx, + fun setup/1, fun teardown/2, + [{F, fun t_upgrade_without_purge_req/2} || F <- From] ++ + [{F, fun t_upgrade_with_1_purge_req/2} || F <- From] ++ + [{F, fun t_upgrade_with_N_purge_req/2} || F <- From] ++ + [{F, fun t_upgrade_with_1_purge_req_for_2_docs/2} || F <- From] } }. -t_upgrade_without_purge_req() -> +t_upgrade_without_purge_req(VersionFrom, {_Ctx, _NewPaths}) -> ?_test(begin % There are three documents in the fixture % db with zero purge entries - DbName = <<"db_without_purge_req">>, + DbName = ?l2b("db_v" ++ integer_to_list(VersionFrom) + ++ "_without_purge_req"), - ?assertEqual(6, get_disk_version_from_header(DbName)), + ?assertEqual(VersionFrom, get_disk_version_from_header(DbName)), {ok, UpgradedPurged} = couch_util:with_db(DbName, fun(Db) -> ?assertEqual(0, couch_db:get_purge_seq(Db)), couch_db:fold_purge_infos(Db, 0, fun fold_fun/2, []) end), ?assertEqual([], UpgradedPurged), - ?assertEqual(7, get_disk_version_from_header(DbName)), + ?assertEqual(8, get_disk_version_from_header(DbName)), {ok, Rev} = save_doc( DbName, {[{<<"_id">>, <<"doc4">>}, {<<"v">>, 1}]} ), @@ -99,18 +102,19 @@ t_upgrade_without_purge_req() -> end). -t_upgrade_with_1_purge_req() -> +t_upgrade_with_1_purge_req(VersionFrom, {_Ctx, _NewPaths}) -> ?_test(begin % There are two documents in the fixture database % with a single purge entry - DbName = <<"db_with_1_purge_req">>, + DbName = ?l2b("db_v" ++ integer_to_list(VersionFrom) + ++ "_with_1_purge_req"), - ?assertEqual(6, get_disk_version_from_header(DbName)), + ?assertEqual(VersionFrom, get_disk_version_from_header(DbName)), {ok, UpgradedPurged} = couch_util:with_db(DbName, fun(Db) -> ?assertEqual(1, couch_db:get_purge_seq(Db)), couch_db:fold_purge_infos(Db, 0, fun fold_fun/2, []) end), - ?assertEqual(7, get_disk_version_from_header(DbName)), + ?assertEqual(8, get_disk_version_from_header(DbName)), ?assertEqual([{1, <<"doc1">>}], UpgradedPurged), {ok, Rev} = save_doc( @@ -139,18 +143,19 @@ t_upgrade_with_1_purge_req() -> end). -t_upgrade_with_N_purge_req() -> +t_upgrade_with_N_purge_req(VersionFrom, {_Ctx, _NewPaths}) -> ?_test(begin % There is one document in the fixture database % with two docs that have been purged - DbName = <<"db_with_2_purge_req">>, + DbName = ?l2b("db_v" ++ integer_to_list(VersionFrom) + ++ "_with_2_purge_req"), - ?assertEqual(6, get_disk_version_from_header(DbName)), + ?assertEqual(VersionFrom, get_disk_version_from_header(DbName)), {ok, UpgradedPurged} = couch_util:with_db(DbName, fun(Db) -> ?assertEqual(2, couch_db:get_purge_seq(Db)), couch_db:fold_purge_infos(Db, 1, fun fold_fun/2, []) end), - ?assertEqual(7, get_disk_version_from_header(DbName)), + ?assertEqual(8, get_disk_version_from_header(DbName)), ?assertEqual([{2, <<"doc2">>}], UpgradedPurged), {ok, Rev} = save_doc(DbName, {[{<<"_id">>, <<"doc4">>}, {<<"v">>, 1}]}), @@ -177,19 +182,20 @@ t_upgrade_with_N_purge_req() -> end). -t_upgrade_with_1_purge_req_for_2_docs() -> +t_upgrade_with_1_purge_req_for_2_docs(VersionFrom, {_Ctx, _NewPaths}) -> ?_test(begin % There are two documents (Doc4 and Doc5) in the fixture database % with three docs (Doc1, Doc2 and Doc3) that have been purged, and % with one purge req for Doc1 and another purge req for Doc 2 and Doc3 - DbName = <<"db_with_1_purge_req_for_2_docs">>, + DbName = ?l2b("db_v" ++ integer_to_list(VersionFrom) + ++ "_with_1_purge_req_for_2_docs"), - ?assertEqual(6, get_disk_version_from_header(DbName)), + ?assertEqual(VersionFrom, get_disk_version_from_header(DbName)), {ok, UpgradedPurged} = couch_util:with_db(DbName, fun(Db) -> ?assertEqual(3, couch_db:get_purge_seq(Db)), couch_db:fold_purge_infos(Db, 1, fun fold_fun/2, []) end), - ?assertEqual(7, get_disk_version_from_header(DbName)), + ?assertEqual(8, get_disk_version_from_header(DbName)), ?assertEqual([{3,<<"doc2">>},{2,<<"doc3">>}], UpgradedPurged), {ok, Rev} = save_doc(DbName, {[{<<"_id">>, <<"doc6">>}, {<<"v">>, 1}]}), diff --git a/src/couch/test/fixtures/db_with_1_purge_req.couch b/src/couch/test/fixtures/db_v6_with_1_purge_req.couch Binary files differindex b0d39c9ec..b0d39c9ec 100644 --- a/src/couch/test/fixtures/db_with_1_purge_req.couch +++ b/src/couch/test/fixtures/db_v6_with_1_purge_req.couch diff --git a/src/couch/test/fixtures/db_with_1_purge_req_for_2_docs.couch b/src/couch/test/fixtures/db_v6_with_1_purge_req_for_2_docs.couch Binary files differindex b584fce31..b584fce31 100644 --- a/src/couch/test/fixtures/db_with_1_purge_req_for_2_docs.couch +++ b/src/couch/test/fixtures/db_v6_with_1_purge_req_for_2_docs.couch diff --git a/src/couch/test/fixtures/db_with_2_purge_req.couch b/src/couch/test/fixtures/db_v6_with_2_purge_req.couch Binary files differindex ee4e11b7f..ee4e11b7f 100644 --- a/src/couch/test/fixtures/db_with_2_purge_req.couch +++ b/src/couch/test/fixtures/db_v6_with_2_purge_req.couch diff --git a/src/couch/test/fixtures/db_without_purge_req.couch b/src/couch/test/fixtures/db_v6_without_purge_req.couch Binary files differindex 814feb8e1..814feb8e1 100644 --- a/src/couch/test/fixtures/db_without_purge_req.couch +++ b/src/couch/test/fixtures/db_v6_without_purge_req.couch diff --git a/src/couch/test/fixtures/db_v7_with_1_purge_req.couch b/src/couch/test/fixtures/db_v7_with_1_purge_req.couch Binary files differnew file mode 100644 index 000000000..cab8331db --- /dev/null +++ b/src/couch/test/fixtures/db_v7_with_1_purge_req.couch diff --git a/src/couch/test/fixtures/db_v7_with_1_purge_req_for_2_docs.couch b/src/couch/test/fixtures/db_v7_with_1_purge_req_for_2_docs.couch Binary files differnew file mode 100644 index 000000000..b613646b1 --- /dev/null +++ b/src/couch/test/fixtures/db_v7_with_1_purge_req_for_2_docs.couch diff --git a/src/couch/test/fixtures/db_v7_with_2_purge_req.couch b/src/couch/test/fixtures/db_v7_with_2_purge_req.couch Binary files differnew file mode 100644 index 000000000..126fc919e --- /dev/null +++ b/src/couch/test/fixtures/db_v7_with_2_purge_req.couch diff --git a/src/couch/test/fixtures/db_v7_without_purge_req.couch b/src/couch/test/fixtures/db_v7_without_purge_req.couch Binary files differnew file mode 100644 index 000000000..762dc8dad --- /dev/null +++ b/src/couch/test/fixtures/db_v7_without_purge_req.couch |